use storage urls for thumbs too

Dockerfile

@@ -34,6 +34,6 @@ VOLUME /srv/cache
 VOLUME /srv/db
 
 USER app
-ENV CACHE_PATH=/tmp/cache
+ENV CACHE_URL=file://./tmp/cache
 
 ENTRYPOINT ["photoappd"]

README.md

@@ -48,7 +48,7 @@ Arguments are as follows:
 
 * `--library file://./library` - file storage uri, in this case the relative path `./library`
 * `--database sqlite:///photos.db` - [Sqlalchemy](https://docs.sqlalchemy.org/en/13/core/engines.html) connection uri
-* `--cache ./cache` - use this directory as a cache for things like thumbnails
+* `--cache file://./cache` - storage uri to use as a cache for things like thumbnails. It can be the same as the library
 * `--port 8080` - listen on http on port 8080
 
 Supported library uri schemes are:
@@ -114,6 +114,7 @@ This would ingest all the files listed in `shas.txt` that aren't already in the
 
 Roadmap
 -------
+- Fix Dates and Stats under mysql
 - Flesh out CLI:
     - Relink function - make a photo a member of another photo
     - Config that is saved somewhere

photoapp/daemon.py

@@ -261,9 +261,10 @@ class ThumbnailView(object):
         if not thumb_from:
             raise cherrypy.HTTPError(404)
         # TODO some lock around calls to this based on uuid
-        thumb_path = self.master.thumbtool.make_thumb(thumb_from, thumb_size)
-        if thumb_path:
-            return cherrypy.lib.static.serve_file(thumb_path, "image/jpeg")
+        thumb_fobj = self.master.thumbtool.make_thumb(thumb_from, thumb_size)
+
+        if thumb_fobj:
+            return cherrypy.lib.static.serve_fileobj(thumb_fobj, "image/jpeg")
         else:
             return cherrypy.lib.static.serve_file(os.path.join(APPROOT, "assets/img/unknown.svg"), "image/svg+xml")
 
@@ -426,7 +427,7 @@ def main():
     parser.add_argument('-p', '--port', help="tcp port to listen on",
                         default=int(os.environ.get("PHOTOLIB_PORT", 8080)), type=int)
     parser.add_argument('-l', '--library', default=os.environ.get("STORAGE_URL"), help="library path")
-    parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_PATH"), help="cache path")
+    parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_URL"), help="cache url")
     # https://docs.sqlalchemy.org/en/13/core/engines.html
     parser.add_argument('-s', '--database', help="sqlalchemy database connection uri",
                         default=os.environ.get("DATABASE_URL")),
@@ -445,7 +446,7 @@ def main():
         parser.error("--library or STORAGE_URL is required")
 
     if not args.cache:
-        parser.error("--cache or CACHE_PATH is required")
+        parser.error("--cache or CACHE_URL is required")
 
     logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
                         format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
@@ -460,7 +461,7 @@ def main():
     # Create various internal tools
     library_storage = uri_to_storage(args.library)
     library_manager = LibraryManager(library_storage)
-    thumbnail_tool = ThumbGenerator(library_manager, args.cache)
+    thumbnail_tool = ThumbGenerator(library_manager, uri_to_storage(args.cache))
 
     # Setup and mount web ui
     tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates"

photoapp/storage.py

@@ -19,6 +19,7 @@ def uri_to_storage(uri):
 class StorageAdapter(object):
     """
     Abstract interface for working with photo file storage. All paths are relative to the storage adapter's root param.
+    TODO add __exit__, and maybe __enter__ if that matches normal files
     """
 
     def exists(self, path):

photoapp/thumb.py

@@ -7,12 +7,14 @@ from multiprocessing import Process
 from PIL import Image, ImageOps
 import tempfile
 from shutil import copyfileobj
+from contextlib import closing
+import logging
 
 
 class ThumbGenerator(object):
-    def __init__(self, library, cache_path):
+    def __init__(self, library, storage):
         self.library = library
-        self.cache_path = cache_path
+        self.storage = storage
         self._failed_thumbs_cache = defaultdict(dict)
 
     def get_datedir_path(self, date):
@@ -33,9 +35,10 @@ class ThumbGenerator(object):
                   "feed": (250, 250, False),
                   "preview": (1024, 768, True),
                   "big": (2048, 1536, True)}
-        dest = os.path.join(self.cache_path, "thumbs", style, "{}.jpg".format(photo.uuid))
-        if os.path.exists(dest):
-            return os.path.abspath(dest)
+        dest = os.path.join("thumbs", style, photo.uuid[0:2], "{}.jpg".format(photo.uuid))
+
+        if self.storage.exists(dest):
+            return self.storage.open(dest, "rb")
         if photo.width is None:  # todo better detection of images that PIL can't open
             return None
         if photo.uuid in self._failed_thumbs_cache[style]:
@@ -53,19 +56,26 @@ class ThumbGenerator(object):
         thumb_height = min(thumb_height, i_height if i_height > 0 else 999999999)  # TODO this seems bad
 
         # TODO have the subprocess download the file
+        # TODO thundering herd
         with tempfile.TemporaryDirectory() as tmpdir:
+            fthumblocal = tempfile.NamedTemporaryFile(delete=True)
             fpath = os.path.join(tmpdir, "image")
             with self.library.storage.open(photo.path, 'rb') as fsrc:
-                with open(fpath, 'wb') as fdest:
-                    copyfileobj(fsrc, fdest)
+                with open(fpath, 'wb') as ftmpdest:
+                    copyfileobj(fsrc, ftmpdest)
 
-            p = Process(target=self.gen_thumb, args=(fpath, dest, thumb_width, thumb_height, photo.orientation))
+            p = Process(target=self.gen_thumb, args=(fpath, fthumblocal.name, thumb_width, thumb_height, photo.orientation))
             p.start()
             p.join()
             if p.exitcode != 0:
                 self._failed_thumbs_cache[style][photo.uuid] = True  # dont retry failed generations
                 return None
-            return os.path.abspath(dest)
+
+            with closing(self.storage.open(dest, 'wb')) as fdest:
+                copyfileobj(fthumblocal, fdest)
+
+        fthumblocal.seek(0)
+        return fthumblocal
 
     @staticmethod
     def gen_thumb(src_img, dest_img, width, height, rotation):
@@ -77,7 +87,7 @@ class ThumbGenerator(object):
             image = image.rotate(90 * rotation, expand=True)
             thumb = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
             thumb.save(dest_img, 'JPEG')
-            print("Generated {} in {}s".format(dest_img, round(time() - start, 4)))
+            logging.info("Generated {} in {}s".format(dest_img, round(time() - start, 4)))
         except Exception:
             traceback.print_exc()
             if os.path.exists(dest_img):