refactor image thumbnail generation code

photoapp/daemon.py

@@ -6,7 +6,7 @@ import cherrypy
 from collections import defaultdict
 from urllib.parse import urlparse
 from datetime import datetime, timedelta
-from photoapp.thumb import ThumbGenerator
+from photoapp.thumbtool import ThumbGenerator
 from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User, mime2ext, regular_mimes, video_mimes
 from photoapp.dbsession import DatabaseSession
 from photoapp.common import pwhash
@@ -258,33 +258,32 @@ class ThumbnailView(object):
             formats[photo.format].append(photo)
         formats = dict(formats)
 
+        thumb_from = None
+
         # prefer image files. If an image is available assume it is the target or intended to be the thumbnail.
         imtypes = regular_mimes.intersection(set(formats.keys()))
         if imtypes:
             # prefer making thumbs from jpeg to avoid loading large raws
             # jk we can't load raws anyway
-            thumb_from = formats.get("image/jpeg")
+            thumb_jpegs = formats.get("image/jpeg")
-            if thumb_from:
+            if thumb_jpegs:
-                thumb_from = thumb_from[0]  # TODO if we're thumbing a set this is an arbitrary image picked
+                thumb_from = thumb_jpegs[0]  # TODO if we're thumbing a set this is an arbitrary image picked
             else:
                 t = imtypes.pop()  # TODO arbitrary
                 thumb_from = formats[t][0]  # TODO arbitrary
 
-            thumb_fobj = self.master.thumbtool.make_photo_thumb(thumb_from, thumb_size)
+        vtypes = video_mimes.intersection(set(formats.keys()))
+        if vtypes and not thumb_from:
+            t = vtypes.pop()  # TODO arbitrary
+            thumb_from = formats[t][0]  # TODO arbitrary
 
-            if thumb_fobj:
+        thumb_fobj = self.master.thumbtool.make_thumb(thumb_from, thumb_size)
-                return cherrypy.lib.static.serve_fileobj(thumb_fobj, "image/jpeg")
-            else:
-                return cherrypy.lib.static.serve_file(os.path.join(APPROOT, "assets/img/unknown.svg"), "image/svg+xml")
 
-        elif video_mimes.intersection(set(formats.keys())):
+        if thumb_fobj:
-            # call out to the video thumb service
+            return cherrypy.lib.static.serve_fileobj(thumb_fobj, "image/jpeg")
-            #TODO call out - placeholder for now
+        else:
-            #TODO thumb service configuration option or show the ? (or a thumb that suggests its a video?)
+            cherrypy.response.headers["Cache-Control"] = "no-store, must-revalidate, max-age=0"
-            #TODO make this set no cache headers
+            cherrypy.response.headers["Age"] = "0"
-            return cherrypy.lib.static.serve_file(os.path.join(APPROOT, "assets/img/unknown.svg"), "image/svg+xml")
-
-        else:  # No convertable file types
             return cherrypy.lib.static.serve_file(os.path.join(APPROOT, "assets/img/unknown.svg"), "image/svg+xml")
 
 

photoapp/thumb.py

@@ -1,14 +1,8 @@
 import os
 import sys
 import traceback
-from time import time
-from collections import defaultdict
-from multiprocessing import Process
 from PIL import Image, ImageOps
-import tempfile
+from multiprocessing import Process
-from shutil import copyfileobj
-from contextlib import closing
-import logging
 
 
 # style tuples: max x, max y, rotate ok)
@@ -20,91 +14,60 @@ THUMB_STYLES = {"tiny": (80, 80, False),
                 "big": (2048, 1536, True)}
 
 
-def gen_thumb(src_img, dest_img, width, height, rotation):
-    start = time()
-    # TODO lock around the dir creation
-    os.makedirs(os.path.split(dest_img)[0], exist_ok=True)
-    image = Image.open(src_img)
-    if image.mode != "RGB":
-        image = image.convert("RGB")
-    image = image.rotate(90 * rotation, expand=True)
-    thumb = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
-    thumb.save(dest_img, 'JPEG')
-    logging.info("Generated {} in {}s".format(dest_img, round(time() - start, 4)))
-    return dest_img
-
-
-def gen_thumb_process(src_img, dest_img, width, height, rotation):
-    try:
-        return gen_thumb(src_img, dest_img, width, height, rotation)
-    except Exception:
-        traceback.print_exc()
-        if os.path.exists(dest_img):
-            os.unlink(dest_img)
-        sys.exit(1)
-
-
 def thumb_path(style_name, uuid):
     return os.path.join("thumbs", style_name, uuid[0:2], "{}.jpg".format(uuid))
 
 
-class ThumbGenerator(object):
+def image_style(image, style_name, orientation):
-    def __init__(self, library, storage):
+    """
-        self.library = library
+    Given an input PIL Image object, return a scaled/cropped image object of style_name
-        self.storage = storage
+    """
-        self._failed_thumbs_cache = defaultdict(dict)
+    thumb_width, thumb_height, flip_ok = THUMB_STYLES[style_name]
+    i_width, i_height = image.size
 
-    def make_photo_thumb(self, photo, style_name):
+    im_is_rotated = orientation % 2 != 0 or i_height > i_width
-        """
-        Create a thumbnail of the given photo, scaled/cropped to the given named style
-        :return: local path to thumbnail file or None if creation failed or was blocked
-        """
 
-        dest = thumb_path(style_name, photo.uuid)
+    if im_is_rotated and flip_ok:
+        thumb_width, thumb_height = thumb_height, thumb_width
+
+    # prevents scale-up (why do i want this?)
+    thumb_width = min(thumb_width, i_width) if i_width > 0 else thumb_width
+    thumb_height = min(thumb_height, i_height) if i_height > 0 else thumb_height
+
+    image = image.rotate(90 * orientation, expand=True)
+    return ImageOps.fit(image, (thumb_width, thumb_height), Image.ANTIALIAS)
 
 
-        if self.storage.exists(dest):
+def image_file_style(src_path, output_path, style_name, orientation):
-            return self.storage.open(dest, "rb")
+    """
+    Given an input and output image paths, create a scaled/cropped image file of style_name
+    """
+    image = Image.open(src_path)
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+    output = image_style(image, style_name, orientation)
+    output.save(output_path, 'JPEG')
 
 
-        if photo.uuid in self._failed_thumbs_cache[style_name]:
+def _image_file_style_process_worker(src_path, output_path, style_name, orientation):
-            return None
+    try:
+        image_file_style(src_path, output_path, style_name, orientation)
+    except Exception:
+        traceback.print_exc()
+        if os.path.exists(output_path):
+            os.unlink(output_path)
+        sys.exit(1)
 
 
-        # if photo.width is None:  # todo better detection of images that PIL can't open
+def image_file_style_process(src_path, output_path, style_name, orientation):
-        #     return None
+    """
+    Same as image_file_style but the image manipulation is done in a background process
+    """
+    p = Process(target=_image_file_style_process_worker,
+                args=(src_path, output_path, style_name, orientation))
+    p.start()
+    p.join()
+    if p.exitcode != 0:
+        return False
 
-        thumb_width, thumb_height, flip_ok = THUMB_STYLES[style_name]
+    return True
-        i_width = photo.width
-        i_height = photo.height
-        im_is_rotated = photo.orientation % 2 != 0 or i_height > i_width
-
-        if im_is_rotated and flip_ok:
-            thumb_width, thumb_height = thumb_height, thumb_width
-
-        thumb_width = min(thumb_width, i_width if i_width > 0 else 999999999)
-        thumb_height = min(thumb_height, i_height if i_height > 0 else 999999999)
-
-
-        # TODO have the subprocess download the file
-        # TODO thundering herd
-        with tempfile.TemporaryDirectory() as tmpdir:
-            fthumblocal = tempfile.NamedTemporaryFile(delete=True)
-            fpath = os.path.join(tmpdir, "image")
-            with self.library.storage.open(photo.path, 'rb') as fsrc:
-                with open(fpath, 'wb') as ftmpdest:
-                    copyfileobj(fsrc, ftmpdest)
-
-            p = Process(target=gen_thumb_process,
-                        args=(fpath, fthumblocal.name, thumb_width, thumb_height, photo.orientation))
-            p.start()
-            p.join()
-            if p.exitcode != 0:
-                self._failed_thumbs_cache[style_name][photo.uuid] = True  # dont retry failed generations
-                return None
-
-            with closing(self.storage.open(dest, 'wb')) as fdest:
-                copyfileobj(fthumblocal, fdest)
-
-        fthumblocal.seek(0)
-        return fthumblocal

photoapp/thumbtool.py

@@ -0,0 +1,48 @@
+import os
+from collections import defaultdict
+import tempfile
+from shutil import copyfileobj
+from contextlib import closing
+from photoapp.thumb import thumb_path, image_file_style_process
+
+
+class ThumbGenerator(object):
+    def __init__(self, library, storage):
+        self.library = library
+        self.storage = storage
+        self._failed_thumbs_cache = defaultdict(dict)
+
+    def make_thumb(self, photo, style_name):
+        """
+        Create a thumbnail of the given photo, scaled/cropped to the given named style
+        :return: local path to thumbnail file or None if creation failed or was blocked
+        """
+        dest = thumb_path(style_name, photo.uuid)
+
+        if self.storage.exists(dest):
+            return self.storage.open(dest, "rb")
+
+        if photo.uuid in self._failed_thumbs_cache[style_name]:
+            return None
+
+        # if photo.width is None:  # todo better detection of images that PIL can't open
+        #     return None
+
+        # TODO have the subprocess download the file
+        # TODO thundering herd
+        with tempfile.TemporaryDirectory() as tmpdir:
+            fthumblocal = tempfile.NamedTemporaryFile(delete=True)
+            fpath = os.path.join(tmpdir, "image")
+            with self.library.storage.open(photo.path, 'rb') as fsrc:
+                with open(fpath, 'wb') as ftmpdest:
+                    copyfileobj(fsrc, ftmpdest)
+
+            if not image_file_style_process(fpath, fthumblocal.name, style_name, photo.orientation):
+                self._failed_thumbs_cache[style_name][photo.uuid] = True  # dont retry failed generations
+                return
+
+            with closing(self.storage.open(dest, 'wb')) as fdest:
+                copyfileobj(fthumblocal, fdest)
+
+        fthumblocal.seek(0)
+        return fthumblocal