standardize photo file and db access

README.md

@@ -51,8 +51,8 @@ Arguments are as follows:
 * `--cache ./cache` - use this directory as a cache for things like thumbnails
 * `--port 8080` - listen on http on port 8080
 
-Depending on your setup, the `photousers` command can be used to create a user account. Login information is necessary
+Next, the `photousers` command can be used to create a user account. Login information is necessary to see images marked
-to see images marked as private or upload images.
+as private or upload images.
 
 * `photousers create -u dave -p mypassword`
 
@@ -107,12 +107,13 @@ This would ingest all the files listed in `shas.txt` that aren't already in the
 Roadmap
 -------
 - Stateless aka docker support
-    - Photo storage
+    - ~~Photo storage~~ done
         - ~~Abstract the storage api~~ done
         - ~~Standardize on API ingest~~ done
-        - Display and retrieval of images from the abstracted image store
+        - ~~Display and retrieval of images from the abstracted image store~~ done
+            - ~~Thumbnail gen~~ done
     - Database support
-        - Get the web UI code (daemon.py) using the same db access method as the api
+        - ~~Get the web UI code (daemon.py) using the same db access method as the api~~
         - Support any connection URI sqlalchemy is happy with
         - Tune certain databases if their uri is detected (sqlite and threads lol)
     - ~~Cache~~ done
@@ -127,3 +128,4 @@ Roadmap
 
 - Longer term ideas:
     - "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest.
+    - Dynamic svg placeholder for images we can't open

photoapp/daemon.py

@@ -1,27 +1,32 @@
 import os
-import cherrypy
+import math
 import logging
+import cherrypy
+from urllib.parse import urlparse
 from datetime import datetime, timedelta
-from photoapp.library import PhotoLibrary
+from photoapp.thumb import ThumbGenerator
 from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
-from jinja2 import Environment, FileSystemLoader, select_autoescape
-from sqlalchemy import desc
-from sqlalchemy import func, and_, or_
 from photoapp.common import pwhash
 from photoapp.api import PhotosApi, LibraryManager, FilesystemAdapter
-from photoapp.dbutils import SAEnginePlugin, SATool
+from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine
-import math
+from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify
-from urllib.parse import urlparse
+from jinja2 import Environment, FileSystemLoader, select_autoescape
-from photoapp.utils import mime2ext, auth, require_auth, photo_auth_filter, slugify
+from sqlalchemy import desc, func, and_, or_
-from photoapp.dbutils import db
 
 
 APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
 
 
+def validate_password(realm, username, password):
+    if db.query(User).filter(User.name == username, User.password == pwhash(password)).first():
+        return True
+    return False
+
+
 class PhotosWeb(object):
-    def __init__(self, library, template_dir):
+    def __init__(self, library, thumbtool, template_dir):
         self.library = library
+        self.thumbtool = thumbtool
         self.tpl = Environment(loader=FileSystemLoader(template_dir),
                                autoescape=select_autoescape(['html', 'xml']))
         self.tpl.filters.update(mime2ext=mime2ext,
@@ -79,8 +84,8 @@ class PhotosWeb(object):
         /feed - main photo feed - show photos sorted by date, newest first
         """
         page, pgsize = int(page), int(pgsize)
-        total_sets = photo_auth_filter(db.query(func.count(PhotoSet.id))).first()[0]
+        total_sets = photoset_auth_filter(db.query(func.count(PhotoSet.id))).first()[0]
-        images = photo_auth_filter(db.query(PhotoSet)).order_by(PhotoSet.date.desc()). \
+        images = photoset_auth_filter(db.query(PhotoSet)).order_by(PhotoSet.date.desc()). \
             offset(pgsize * page).limit(pgsize).all()
         yield self.render("feed.html", images=[i for i in images], page=page, pgsize=int(pgsize), total_sets=total_sets)
 
@@ -89,11 +94,11 @@ class PhotosWeb(object):
         """
         /stats - show server statistics
         """
-        images = photo_auth_filter(db.query(func.count(PhotoSet.uuid),
+        images = photoset_auth_filter(db.query(func.count(PhotoSet.uuid),
-                                            func.strftime('%Y', PhotoSet.date).label('year'),
+                                               func.strftime('%Y', PhotoSet.date).label('year'),
-                                            func.strftime('%m', PhotoSet.date).label('month'))). \
+                                               func.strftime('%m', PhotoSet.date).label('month'))). \
             group_by('year', 'month').order_by(desc('year'), desc('month')).all()
-        tsize = photo_auth_filter(db.query(func.sum(Photo.size)).join(PhotoSet)).scalar()  # pragma: manual auth
+        tsize = photoset_auth_filter(db.query(func.sum(Photo.size)).join(PhotoSet)).scalar()  # pragma: manual auth
         yield self.render("monthly.html", images=images, tsize=tsize)
 
     @cherrypy.expose
@@ -103,7 +108,7 @@ class PhotosWeb(object):
             the given tag.
         TODO using so many coordinates is slow in the browser. dedupe them somehow.
         """
-        query = photo_auth_filter(db.query(PhotoSet)).filter(PhotoSet.lat != 0, PhotoSet.lon != 0)
+        query = photoset_auth_filter(db.query(PhotoSet)).filter(PhotoSet.lat != 0, PhotoSet.lon != 0)
         if a:
             query = query.join(TagItem).join(Tag).filter(Tag.uuid == a)
         if i:
@@ -167,6 +172,7 @@ class PhotosWeb(object):
         /login - enable super features by logging into the app
         """
         cherrypy.session['authed'] = cherrypy.request.login
+        print("Authed as", cherrypy.session['authed'])
         dest = "/feed" if "Referer" not in cherrypy.request.headers \
             else urlparse(cherrypy.request.headers["Referer"]).path
         raise cherrypy.HTTPRedirect(dest, 302)
@@ -201,20 +207,20 @@ class DateView(object):
             pgsize = 100
             dt = datetime.strptime(date, "%Y-%m-%d")
             dt_end = dt + timedelta(days=1)
-            total_sets = photo_auth_filter(db.query(func.count(PhotoSet.id))). \
+            total_sets = photoset_auth_filter(db.query(func.count(PhotoSet.id))). \
                 filter(and_(PhotoSet.date >= dt, PhotoSet.date < dt_end)).first()[0]
-            images = photo_auth_filter(db.query(PhotoSet)).filter(and_(PhotoSet.date >= dt,
+            images = photoset_auth_filter(db.query(PhotoSet)).filter(and_(PhotoSet.date >= dt,
-                                                                  PhotoSet.date < dt_end)).order_by(PhotoSet.date). \
+                                                                     PhotoSet.date < dt_end)).order_by(PhotoSet.date). \
                 offset(page * pgsize).limit(pgsize).all()
             yield self.master.render("date.html", page=page, pgsize=pgsize, total_sets=total_sets,
                                      images=[i for i in images], date=dt)
             return
-        images = photo_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
+        images = photoset_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
-                                   PhotoSet.date).label('gdate'),
+                                      PhotoSet.date).label('gdate'),
-                                   func.count('photos.id'),
+                                      func.count('photos.id'),
-                                   func.strftime('%Y', PhotoSet.date).label('year'),
+                                      func.strftime('%Y', PhotoSet.date).label('year'),
-                                   func.strftime('%m', PhotoSet.date).label('month'),
+                                      func.strftime('%m', PhotoSet.date).label('month'),
-                                   func.strftime('%d', PhotoSet.date).label('day'))). \
+                                      func.strftime('%d', PhotoSet.date).label('day'))). \
             group_by('gdate').order_by(desc('year'), 'month', 'day').all()
         yield self.master.render("dates.html", images=images)
 
@@ -230,7 +236,7 @@ class ThumbnailView(object):
     @cherrypy.expose
     def index(self, item_type, thumb_size, uuid):
         uuid = uuid.split(".")[0]
-        query = photo_auth_filter(db.query(Photo).join(PhotoSet))
+        query = photoset_auth_filter(db.query(Photo).join(PhotoSet))
 
         query = query.filter(Photo.set.has(uuid=uuid)) if item_type == "set" \
             else query.filter(Photo.uuid == uuid) if item_type == "one" \
@@ -252,7 +258,7 @@ class ThumbnailView(object):
         if not thumb_from:
             raise cherrypy.HTTPError(404)
         # TODO some lock around calls to this based on uuid
-        thumb_path = self.master.library.make_thumb(thumb_from, thumb_size)
+        thumb_path = self.master.thumbtool.make_thumb(thumb_from, thumb_size)
         if thumb_path:
             return cherrypy.lib.static.serve_file(thumb_path, "image/jpeg")
         else:
@@ -271,7 +277,7 @@ class DownloadView(object):
     def index(self, item_type, uuid, preview=False):
         uuid = uuid.split(".")[0]
         query = None if item_type == "set" \
-            else photo_auth_filter(db.query(Photo)).filter(Photo.uuid == uuid) if item_type == "one" \
+            else photoset_auth_filter(db.query(Photo).join(PhotoSet)).filter(Photo.uuid == uuid) if item_type == "one" \
             else None  # TODO set download query
 
         item = query.first()
@@ -280,8 +286,8 @@ class DownloadView(object):
         extra = {}
         if not preview:
             extra.update(disposition="attachement", name=os.path.basename(item.path))
-        return cherrypy.lib.static.serve_file(os.path.abspath(os.path.join(self.master.library.path, item.path)),
+        return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'),
-                                              content_type=item.format, **extra)
+                                                 content_type=item.format, **extra)
 
 
 @cherrypy.popargs('uuid')
@@ -295,7 +301,8 @@ class PhotoView(object):
     @cherrypy.expose
     def index(self, uuid):
         # uuid = uuid.split(".")[0]
-        photo = photo_auth_filter(db.query(PhotoSet)).filter(or_(PhotoSet.uuid == uuid, PhotoSet.slug == uuid)).first()
+        photo = photoset_auth_filter(db.query(PhotoSet)).filter(or_(PhotoSet.uuid == uuid,
+                                                                    PhotoSet.slug == uuid)).first()
         if not photo:
             raise cherrypy.HTTPError(404)
         yield self.master.render("photo.html", image=photo)
@@ -326,7 +333,7 @@ class PhotoView(object):
     @cherrypy.expose
     @require_auth
     def edit(self, uuid):
-        photo = photo_auth_filter(db.query(PhotoSet)).filter(PhotoSet.uuid == uuid).first()
+        photo = photoset_auth_filter(db.query(PhotoSet)).filter(PhotoSet.uuid == uuid).first()
         yield self.master.render("photo_edit.html", image=photo)
 
 
@@ -343,17 +350,17 @@ class TagView(object):
         page = int(page)
         pgsize = 100
         if uuid == "untagged":
-            numphotos = photo_auth_filter(db.query(func.count(PhotoSet.id))). \
+            numphotos = photoset_auth_filter(db.query(func.count(PhotoSet.id))). \
                 filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).scalar()
-            photos = photo_auth_filter(db.query(PhotoSet)).filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).\
+            photos = photoset_auth_filter(db.query(PhotoSet)).filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).\
                 offset(page * pgsize). \
                 limit(pgsize).all()
             yield self.master.render("untagged.html", images=photos, total_items=numphotos, pgsize=pgsize, page=page)
         else:
             tag = db.query(Tag).filter(or_(Tag.uuid == uuid, Tag.slug == uuid)).first()
-            numphotos = photo_auth_filter(db.query(func.count(Tag.id)).join(TagItem).join(PhotoSet)). \
+            numphotos = photoset_auth_filter(db.query(func.count(Tag.id)).join(TagItem).join(PhotoSet)). \
                 filter(Tag.id == tag.id).scalar()
-            photos = photo_auth_filter(db.query(PhotoSet)).join(TagItem).join(Tag). \
+            photos = photoset_auth_filter(db.query(PhotoSet)).join(TagItem).join(Tag). \
                 filter(Tag.id == tag.id). \
                 order_by(PhotoSet.date.desc()). \
                 offset(page * pgsize). \
@@ -423,17 +430,21 @@ def main():
     logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
                         format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
 
-    library = PhotoLibrary(args.database, args.library, args.cache)
+    # Get database connection
+    engine = get_db_engine(args.database)
 
+    # Setup database in web framework
+    cherrypy.tools.db = SATool()
+    SAEnginePlugin(cherrypy.engine, engine).subscribe()
+
+    # Create various internal tools
+    library_storage = FilesystemAdapter(args.library)
+    library_manager = LibraryManager(library_storage)
+    thumbnail_tool = ThumbGenerator(library_manager, args.cache)
+
+    # Setup and mount web ui
     tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates"
-
+    web = PhotosWeb(library_manager, thumbnail_tool, tpl_dir)
-    web = PhotosWeb(library, tpl_dir)
-
-    def validate_password(realm, username, password):
-        if db.query(User).filter(User.name == username, User.password == pwhash(password)).first():
-            return True
-        return False
-
     cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
                                          'tools.db.on': True,
                                          'error_page.403': web.error,
@@ -445,12 +456,7 @@ def main():
                                               'tools.auth_basic.realm': 'photolib',
                                               'tools.auth_basic.checkpassword': validate_password}})
 
-    cherrypy.tools.db = SATool()
+    # Setup and mount API
-    SAEnginePlugin(cherrypy.engine, library.engine).subscribe()
-
-    library_storage = FilesystemAdapter(args.library)
-    library_manager = LibraryManager(library_storage)
-
     api = PhotosApi(library_manager)
     cherrypy.tree.mount(api, '/api', {'/': {'tools.trailing_slash.on': False,
                                             'tools.auth_basic.on': True,
@@ -458,6 +464,7 @@ def main():
                                             'tools.auth_basic.checkpassword': validate_password,
                                             'tools.db.on': True}})
 
+    # General config options
     cherrypy.config.update({
         'tools.sessions.on': True,
         'tools.sessions.locking': 'explicit',
@@ -471,6 +478,7 @@ def main():
         'engine.autoreload.on': args.debug
     })
 
+    # Setup signal handling and run it.
     def signal_handler(signum, stack):
         logging.critical('Got sig {}, exiting...'.format(signum))
         cherrypy.engine.exit()

photoapp/dbutils.py

@@ -2,11 +2,28 @@ import sqlalchemy
 import cherrypy
 from cherrypy.process import plugins
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.pool import StaticPool, AssertionPool, NullPool
+from sqlalchemy.orm import sessionmaker
 
 
 Base = declarative_base()
 
 
+def get_db_engine(uri):
+    # TODO handle more uris
+    engine = sqlalchemy.create_engine('sqlite:///{}'.format(uri),
+                                      connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True)
+    Base.metadata.create_all(engine)
+    return engine
+
+
+def get_db_session(uri):
+    engine = get_db_engine(uri)
+    session = sessionmaker()
+    session.configure(bind=engine)
+    return session
+
+
 class DbAlias(object):
     """
     This provides a shorter alias for the cherrypy.request.db object, which is a database session created bound to the

photoapp/ingest.py

@@ -1,6 +1,5 @@
 import argparse
 import traceback
-from photoapp.library import PhotoLibrary
 from photoapp.image import get_jpg_info, get_hash, get_mtime, special_magic
 from itertools import chain
 from photoapp.types import Photo, PhotoSet, known_extensions, regular_images, files_raw, files_video, map_extension

photoapp/thumb.py

@@ -2,66 +2,19 @@ import os
 import sys
 import traceback
 from time import time
-from sqlalchemy import create_engine
-from sqlalchemy.pool import StaticPool, AssertionPool, NullPool  # NOQA
-from sqlalchemy.orm import sessionmaker
-from photoapp.types import Base, Photo, PhotoSet  # NOQA need to be loaded for orm setup
-from sqlalchemy.exc import IntegrityError
 from collections import defaultdict
 from multiprocessing import Process
 from PIL import Image, ImageOps
+import tempfile
+from shutil import copyfileobj
 
 
-class PhotoLibrary(object):
+class ThumbGenerator(object):
-    def __init__(self, db_path, lib_path, cache_path):
+    def __init__(self, library, cache_path):
-        self.path = lib_path
+        self.library = library
         self.cache_path = cache_path
-        # TODO configure from env var
-        # TODO use the right pool and connection args depending on the url
-        # https://docs.sqlalchemy.org/en/13/core/pooling.html
-        self.engine = create_engine('sqlite:///{}'.format(db_path),
-                                    connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True)
-        Base.metadata.create_all(self.engine)
-        self.session = sessionmaker()
-        self.session.configure(bind=self.engine)
         self._failed_thumbs_cache = defaultdict(dict)
 
-    def add_photoset(self, photoset):
-        """
-        Commit a populated photoset object to the library. The paths in the photoset's file list entries will be updated
-        as the file is moved to the library path.
-        """
-
-        # Create target directory
-        path = os.path.join(self.path, self.get_datedir_path(photoset.date))
-        os.makedirs(path, exist_ok=True)
-
-        moves = []  # Track files moved. If the sql transaction files, we'll undo these
-
-        for file in photoset.files:
-            dest = os.path.join(path, os.path.basename(file.path))
-
-            # Check if the name is already in use, rename new file if needed
-            dupe_rename = 1
-            while os.path.exists(dest):
-                fname = os.path.basename(file.path).split(".")
-                fname[-2] += "_{}".format(dupe_rename)
-                dest = os.path.join(path, '.'.join(fname))
-                dupe_rename += 1
-            os.rename(file.path, dest)
-            moves.append((file.path, dest))
-            file.path = dest.lstrip(self.path)
-
-        s = self.session()
-        s.add(photoset)
-        try:
-            s.commit()
-        except IntegrityError:
-            # Commit failed, undo the moves
-            for move in moves:
-                os.rename(move[1], move[0])
-            raise
-
     def get_datedir_path(self, date):
         """
         Return a path like 2018/3/31 given a datetime object representing the same date
@@ -97,13 +50,19 @@ class PhotoLibrary(object):
             thumb_width = min(thumb_width, i_width if i_width > 0 else 999999999)  # TODO do we even have photo.width if PIL can't read the image?
             thumb_height = min(thumb_height, i_height if i_height > 0 else 999999999)  # TODO this seems bad
 
-            p = Process(target=self.gen_thumb, args=(os.path.join(self.path, photo.path), dest, thumb_width, thumb_height, photo.orientation))
+            # TODO have the subprocess download the file
-            p.start()
+            with tempfile.TemporaryDirectory() as tmpdir:
-            p.join()
+                fpath = os.path.join(tmpdir, "image")
-            if p.exitcode != 0:
+                with self.library.storage.open(photo.path, 'rb') as fsrc, open(fpath, 'wb') as fdest:
-                self._failed_thumbs_cache[style][photo.uuid] = True  # dont retry failed generations
+                    copyfileobj(fsrc, fdest)
-                return None
+
-            return os.path.abspath(dest)
+                p = Process(target=self.gen_thumb, args=(fpath, dest, thumb_width, thumb_height, photo.orientation))
+                p.start()
+                p.join()
+                if p.exitcode != 0:
+                    self._failed_thumbs_cache[style][photo.uuid] = True  # dont retry failed generations
+                    return None
+                return os.path.abspath(dest)
         return None
 
     @staticmethod
@@ -117,7 +76,7 @@ class PhotoLibrary(object):
             thumb = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
             thumb.save(dest_img, 'JPEG')
             print("Generated {} in {}s".format(dest_img, round(time() - start, 4)))
-        except:
+        except Exception:
             traceback.print_exc()
             if os.path.exists(dest_img):
                 os.unlink(dest_img)

photoapp/users.py

@@ -1,24 +1,22 @@
 import argparse
-from photoapp.library import PhotoLibrary
 from photoapp.types import User
 from photoapp.common import pwhash
+from photoapp.dbutils import get_db_session
+from tabulate import tabulate
 
 
-def create_user(library, username, password):
+def create_user(s, username, password):
-    s = library.session()
     s.add(User(name=username, password=pwhash(password)))
     s.commit()
 
 
-def list_users(library):
+def list_users(s):
-    s = library.session()
+    print(tabulate([[user.id,
-    print("id\tname")
+                     user.name] for user in s.query(User).order_by(User.name).all()],
-    for user in s.query(User).order_by(User.name).all():
+                   headers=["id", "username"]))
-        print("{}\t{}".format(user.id, user.name))
 
 
-def delete_user(library, username):
+def delete_user(s, username):
-    s = library.session()
     u = s.query(User).filter(User.name == username).first()
     s.delete(u)
     s.commit()
@@ -27,6 +25,8 @@ def delete_user(library, username):
 
 def main():
     parser = argparse.ArgumentParser(description="User manipulation tool")
+    parser.add_argument("-d", "--database", help="database uri")
+
     p_mode = parser.add_subparsers(dest='action', help='action to take')
 
     p_create = p_mode.add_parser('create', help='create user')
@@ -40,14 +40,14 @@ def main():
 
     args = parser.parse_args()
 
-    library = PhotoLibrary("photos.db", "./library/", "./cache/")
+    session = get_db_session(args.database)()
 
     if args.action == "create":
-        create_user(library, args.username, args.password)
+        create_user(session, args.username, args.password)
     elif args.action == "list":
-        list_users(library)
+        list_users(session)
     elif args.action == "delete":
-        delete_user(library, args.username)
+        delete_user(session, args.username)
     else:
         parser.print_help()
 

photoapp/utils.py

@@ -63,7 +63,7 @@ def require_auth(func):
     return wrapped
 
 
-def photo_auth_filter(query):
+def photoset_auth_filter(query):
     """
     Sqlalchemy helper: filter the given PhotoSet query to items that match the authorized user's PhotoStatus access
     level. Currently, authed users can access ALL photos, and unauthed users can access only PhotoStatus.public

setup.py

@@ -17,12 +17,12 @@ setup(name='photoapp',
       entry_points={
           "console_scripts": [
               "photoappd = photoapp.daemon:main",
-              "photoimport = photoapp.ingest:main",
-              "photovalidate = photoapp.validate:main",
               "photoinfo = photoapp.image:main",
-              "photooffset = photoapp.dateoffset:main",
               "photousers = photoapp.users:main",
               "photocli = photoapp.cli:main",
+              # "photoimport = photoapp.ingest:main",  # broken for now
+              # "photovalidate = photoapp.validate:main",
+              #"photooffset = photoapp.dateoffset:main",
           ]
       },
       include_package_data=True,