From 8877fb263a34c1d6f8c37535c240cdc6a4510d89 Mon Sep 17 00:00:00 2001 From: dave Date: Thu, 11 Jul 2019 19:26:22 -0700 Subject: [PATCH] use storage urls for thumbs too --- Dockerfile | 2 +- README.md | 3 ++- photoapp/daemon.py | 13 +++++++------ photoapp/storage.py | 1 + photoapp/thumb.py | 30 ++++++++++++++++++++---------- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index c215793..cefc8f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,6 @@ VOLUME /srv/cache VOLUME /srv/db USER app -ENV CACHE_PATH=/tmp/cache +ENV CACHE_URL=file://./tmp/cache ENTRYPOINT ["photoappd"] diff --git a/README.md b/README.md index 583580d..c17619d 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Arguments are as follows: * `--library file://./library` - file storage uri, in this case the relative path `./library` * `--database sqlite:///photos.db` - [Sqlalchemy](https://docs.sqlalchemy.org/en/13/core/engines.html) connection uri -* `--cache ./cache` - use this directory as a cache for things like thumbnails +* `--cache file://./cache` - storage uri to use as a cache for things like thumbnails. It can be the same as the library * `--port 8080` - listen on http on port 8080 Supported library uri schemes are: @@ -114,6 +114,7 @@ This would ingest all the files listed in `shas.txt` that aren't already in the Roadmap ------- +- Fix Dates and Stats under mysql - Flesh out CLI: - Relink function - make a photo a member of another photo - Config that is saved somewhere diff --git a/photoapp/daemon.py b/photoapp/daemon.py index 7f45426..22f24f0 100644 --- a/photoapp/daemon.py +++ b/photoapp/daemon.py @@ -261,9 +261,10 @@ class ThumbnailView(object): if not thumb_from: raise cherrypy.HTTPError(404) # TODO some lock around calls to this based on uuid - thumb_path = self.master.thumbtool.make_thumb(thumb_from, thumb_size) - if thumb_path: - return cherrypy.lib.static.serve_file(thumb_path, "image/jpeg") + thumb_fobj = self.master.thumbtool.make_thumb(thumb_from, thumb_size) + + if thumb_fobj: + return cherrypy.lib.static.serve_fileobj(thumb_fobj, "image/jpeg") else: return cherrypy.lib.static.serve_file(os.path.join(APPROOT, "assets/img/unknown.svg"), "image/svg+xml") @@ -426,7 +427,7 @@ def main(): parser.add_argument('-p', '--port', help="tcp port to listen on", default=int(os.environ.get("PHOTOLIB_PORT", 8080)), type=int) parser.add_argument('-l', '--library', default=os.environ.get("STORAGE_URL"), help="library path") - parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_PATH"), help="cache path") + parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_URL"), help="cache url") # https://docs.sqlalchemy.org/en/13/core/engines.html parser.add_argument('-s', '--database', help="sqlalchemy database connection uri", default=os.environ.get("DATABASE_URL")), @@ -445,7 +446,7 @@ def main(): parser.error("--library or STORAGE_URL is required") if not args.cache: - parser.error("--cache or CACHE_PATH is required") + parser.error("--cache or CACHE_URL is required") logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") @@ -460,7 +461,7 @@ def main(): # Create various internal tools library_storage = uri_to_storage(args.library) library_manager = LibraryManager(library_storage) - thumbnail_tool = ThumbGenerator(library_manager, args.cache) + thumbnail_tool = ThumbGenerator(library_manager, uri_to_storage(args.cache)) # Setup and mount web ui tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates" diff --git a/photoapp/storage.py b/photoapp/storage.py index 1f44163..44c29a8 100644 --- a/photoapp/storage.py +++ b/photoapp/storage.py @@ -19,6 +19,7 @@ def uri_to_storage(uri): class StorageAdapter(object): """ Abstract interface for working with photo file storage. All paths are relative to the storage adapter's root param. + TODO add __exit__, and maybe __enter__ if that matches normal files """ def exists(self, path): diff --git a/photoapp/thumb.py b/photoapp/thumb.py index 53ce463..266de3e 100644 --- a/photoapp/thumb.py +++ b/photoapp/thumb.py @@ -7,12 +7,14 @@ from multiprocessing import Process from PIL import Image, ImageOps import tempfile from shutil import copyfileobj +from contextlib import closing +import logging class ThumbGenerator(object): - def __init__(self, library, cache_path): + def __init__(self, library, storage): self.library = library - self.cache_path = cache_path + self.storage = storage self._failed_thumbs_cache = defaultdict(dict) def get_datedir_path(self, date): @@ -33,9 +35,10 @@ class ThumbGenerator(object): "feed": (250, 250, False), "preview": (1024, 768, True), "big": (2048, 1536, True)} - dest = os.path.join(self.cache_path, "thumbs", style, "{}.jpg".format(photo.uuid)) - if os.path.exists(dest): - return os.path.abspath(dest) + dest = os.path.join("thumbs", style, photo.uuid[0:2], "{}.jpg".format(photo.uuid)) + + if self.storage.exists(dest): + return self.storage.open(dest, "rb") if photo.width is None: # todo better detection of images that PIL can't open return None if photo.uuid in self._failed_thumbs_cache[style]: @@ -53,19 +56,26 @@ class ThumbGenerator(object): thumb_height = min(thumb_height, i_height if i_height > 0 else 999999999) # TODO this seems bad # TODO have the subprocess download the file + # TODO thundering herd with tempfile.TemporaryDirectory() as tmpdir: + fthumblocal = tempfile.NamedTemporaryFile(delete=True) fpath = os.path.join(tmpdir, "image") with self.library.storage.open(photo.path, 'rb') as fsrc: - with open(fpath, 'wb') as fdest: - copyfileobj(fsrc, fdest) + with open(fpath, 'wb') as ftmpdest: + copyfileobj(fsrc, ftmpdest) - p = Process(target=self.gen_thumb, args=(fpath, dest, thumb_width, thumb_height, photo.orientation)) + p = Process(target=self.gen_thumb, args=(fpath, fthumblocal.name, thumb_width, thumb_height, photo.orientation)) p.start() p.join() if p.exitcode != 0: self._failed_thumbs_cache[style][photo.uuid] = True # dont retry failed generations return None - return os.path.abspath(dest) + + with closing(self.storage.open(dest, 'wb')) as fdest: + copyfileobj(fthumblocal, fdest) + + fthumblocal.seek(0) + return fthumblocal @staticmethod def gen_thumb(src_img, dest_img, width, height, rotation): @@ -77,7 +87,7 @@ class ThumbGenerator(object): image = image.rotate(90 * rotation, expand=True) thumb = ImageOps.fit(image, (width, height), Image.ANTIALIAS) thumb.save(dest_img, 'JPEG') - print("Generated {} in {}s".format(dest_img, round(time() - start, 4))) + logging.info("Generated {} in {}s".format(dest_img, round(time() - start, 4))) except Exception: traceback.print_exc() if os.path.exists(dest_img):