standardize photo file and db access

This commit is contained in:
dave 2019-07-04 18:41:57 -07:00
parent 6589f71052
commit 043ffbc543
8 changed files with 122 additions and 137 deletions

View File

@ -51,8 +51,8 @@ Arguments are as follows:
* `--cache ./cache` - use this directory as a cache for things like thumbnails
* `--port 8080` - listen on http on port 8080
Depending on your setup, the `photousers` command can be used to create a user account. Login information is necessary
to see images marked as private or upload images.
Next, the `photousers` command can be used to create a user account. Login information is necessary to see images marked
as private or upload images.
* `photousers create -u dave -p mypassword`
@ -107,12 +107,13 @@ This would ingest all the files listed in `shas.txt` that aren't already in the
Roadmap
-------
- Stateless aka docker support
- Photo storage
- ~~Photo storage~~ done
- ~~Abstract the storage api~~ done
- ~~Standardize on API ingest~~ done
- Display and retrieval of images from the abstracted image store
- ~~Display and retrieval of images from the abstracted image store~~ done
- ~~Thumbnail gen~~ done
- Database support
- Get the web UI code (daemon.py) using the same db access method as the api
- ~~Get the web UI code (daemon.py) using the same db access method as the api~~
- Support any connection URI sqlalchemy is happy with
- Tune certain databases if their uri is detected (sqlite and threads lol)
- ~~Cache~~ done
@ -127,3 +128,4 @@ Roadmap
- Longer term ideas:
- "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest.
- Dynamic svg placeholder for images we can't open

View File

@ -1,27 +1,32 @@
import os
import cherrypy
import math
import logging
import cherrypy
from urllib.parse import urlparse
from datetime import datetime, timedelta
from photoapp.library import PhotoLibrary
from photoapp.thumb import ThumbGenerator
from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import desc
from sqlalchemy import func, and_, or_
from photoapp.common import pwhash
from photoapp.api import PhotosApi, LibraryManager, FilesystemAdapter
from photoapp.dbutils import SAEnginePlugin, SATool
import math
from urllib.parse import urlparse
from photoapp.utils import mime2ext, auth, require_auth, photo_auth_filter, slugify
from photoapp.dbutils import db
from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine
from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import desc, func, and_, or_
APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
def validate_password(realm, username, password):
if db.query(User).filter(User.name == username, User.password == pwhash(password)).first():
return True
return False
class PhotosWeb(object):
def __init__(self, library, template_dir):
def __init__(self, library, thumbtool, template_dir):
self.library = library
self.thumbtool = thumbtool
self.tpl = Environment(loader=FileSystemLoader(template_dir),
autoescape=select_autoescape(['html', 'xml']))
self.tpl.filters.update(mime2ext=mime2ext,
@ -79,8 +84,8 @@ class PhotosWeb(object):
/feed - main photo feed - show photos sorted by date, newest first
"""
page, pgsize = int(page), int(pgsize)
total_sets = photo_auth_filter(db.query(func.count(PhotoSet.id))).first()[0]
images = photo_auth_filter(db.query(PhotoSet)).order_by(PhotoSet.date.desc()). \
total_sets = photoset_auth_filter(db.query(func.count(PhotoSet.id))).first()[0]
images = photoset_auth_filter(db.query(PhotoSet)).order_by(PhotoSet.date.desc()). \
offset(pgsize * page).limit(pgsize).all()
yield self.render("feed.html", images=[i for i in images], page=page, pgsize=int(pgsize), total_sets=total_sets)
@ -89,11 +94,11 @@ class PhotosWeb(object):
"""
/stats - show server statistics
"""
images = photo_auth_filter(db.query(func.count(PhotoSet.uuid),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'))). \
images = photoset_auth_filter(db.query(func.count(PhotoSet.uuid),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'))). \
group_by('year', 'month').order_by(desc('year'), desc('month')).all()
tsize = photo_auth_filter(db.query(func.sum(Photo.size)).join(PhotoSet)).scalar() # pragma: manual auth
tsize = photoset_auth_filter(db.query(func.sum(Photo.size)).join(PhotoSet)).scalar() # pragma: manual auth
yield self.render("monthly.html", images=images, tsize=tsize)
@cherrypy.expose
@ -103,7 +108,7 @@ class PhotosWeb(object):
the given tag.
TODO using so many coordinates is slow in the browser. dedupe them somehow.
"""
query = photo_auth_filter(db.query(PhotoSet)).filter(PhotoSet.lat != 0, PhotoSet.lon != 0)
query = photoset_auth_filter(db.query(PhotoSet)).filter(PhotoSet.lat != 0, PhotoSet.lon != 0)
if a:
query = query.join(TagItem).join(Tag).filter(Tag.uuid == a)
if i:
@ -167,6 +172,7 @@ class PhotosWeb(object):
/login - enable super features by logging into the app
"""
cherrypy.session['authed'] = cherrypy.request.login
print("Authed as", cherrypy.session['authed'])
dest = "/feed" if "Referer" not in cherrypy.request.headers \
else urlparse(cherrypy.request.headers["Referer"]).path
raise cherrypy.HTTPRedirect(dest, 302)
@ -201,20 +207,20 @@ class DateView(object):
pgsize = 100
dt = datetime.strptime(date, "%Y-%m-%d")
dt_end = dt + timedelta(days=1)
total_sets = photo_auth_filter(db.query(func.count(PhotoSet.id))). \
total_sets = photoset_auth_filter(db.query(func.count(PhotoSet.id))). \
filter(and_(PhotoSet.date >= dt, PhotoSet.date < dt_end)).first()[0]
images = photo_auth_filter(db.query(PhotoSet)).filter(and_(PhotoSet.date >= dt,
PhotoSet.date < dt_end)).order_by(PhotoSet.date). \
images = photoset_auth_filter(db.query(PhotoSet)).filter(and_(PhotoSet.date >= dt,
PhotoSet.date < dt_end)).order_by(PhotoSet.date). \
offset(page * pgsize).limit(pgsize).all()
yield self.master.render("date.html", page=page, pgsize=pgsize, total_sets=total_sets,
images=[i for i in images], date=dt)
return
images = photo_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
PhotoSet.date).label('gdate'),
func.count('photos.id'),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'),
func.strftime('%d', PhotoSet.date).label('day'))). \
images = photoset_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
PhotoSet.date).label('gdate'),
func.count('photos.id'),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'),
func.strftime('%d', PhotoSet.date).label('day'))). \
group_by('gdate').order_by(desc('year'), 'month', 'day').all()
yield self.master.render("dates.html", images=images)
@ -230,7 +236,7 @@ class ThumbnailView(object):
@cherrypy.expose
def index(self, item_type, thumb_size, uuid):
uuid = uuid.split(".")[0]
query = photo_auth_filter(db.query(Photo).join(PhotoSet))
query = photoset_auth_filter(db.query(Photo).join(PhotoSet))
query = query.filter(Photo.set.has(uuid=uuid)) if item_type == "set" \
else query.filter(Photo.uuid == uuid) if item_type == "one" \
@ -252,7 +258,7 @@ class ThumbnailView(object):
if not thumb_from:
raise cherrypy.HTTPError(404)
# TODO some lock around calls to this based on uuid
thumb_path = self.master.library.make_thumb(thumb_from, thumb_size)
thumb_path = self.master.thumbtool.make_thumb(thumb_from, thumb_size)
if thumb_path:
return cherrypy.lib.static.serve_file(thumb_path, "image/jpeg")
else:
@ -271,7 +277,7 @@ class DownloadView(object):
def index(self, item_type, uuid, preview=False):
uuid = uuid.split(".")[0]
query = None if item_type == "set" \
else photo_auth_filter(db.query(Photo)).filter(Photo.uuid == uuid) if item_type == "one" \
else photoset_auth_filter(db.query(Photo).join(PhotoSet)).filter(Photo.uuid == uuid) if item_type == "one" \
else None # TODO set download query
item = query.first()
@ -280,8 +286,8 @@ class DownloadView(object):
extra = {}
if not preview:
extra.update(disposition="attachement", name=os.path.basename(item.path))
return cherrypy.lib.static.serve_file(os.path.abspath(os.path.join(self.master.library.path, item.path)),
content_type=item.format, **extra)
return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'),
content_type=item.format, **extra)
@cherrypy.popargs('uuid')
@ -295,7 +301,8 @@ class PhotoView(object):
@cherrypy.expose
def index(self, uuid):
# uuid = uuid.split(".")[0]
photo = photo_auth_filter(db.query(PhotoSet)).filter(or_(PhotoSet.uuid == uuid, PhotoSet.slug == uuid)).first()
photo = photoset_auth_filter(db.query(PhotoSet)).filter(or_(PhotoSet.uuid == uuid,
PhotoSet.slug == uuid)).first()
if not photo:
raise cherrypy.HTTPError(404)
yield self.master.render("photo.html", image=photo)
@ -326,7 +333,7 @@ class PhotoView(object):
@cherrypy.expose
@require_auth
def edit(self, uuid):
photo = photo_auth_filter(db.query(PhotoSet)).filter(PhotoSet.uuid == uuid).first()
photo = photoset_auth_filter(db.query(PhotoSet)).filter(PhotoSet.uuid == uuid).first()
yield self.master.render("photo_edit.html", image=photo)
@ -343,17 +350,17 @@ class TagView(object):
page = int(page)
pgsize = 100
if uuid == "untagged":
numphotos = photo_auth_filter(db.query(func.count(PhotoSet.id))). \
numphotos = photoset_auth_filter(db.query(func.count(PhotoSet.id))). \
filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).scalar()
photos = photo_auth_filter(db.query(PhotoSet)).filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).\
photos = photoset_auth_filter(db.query(PhotoSet)).filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).\
offset(page * pgsize). \
limit(pgsize).all()
yield self.master.render("untagged.html", images=photos, total_items=numphotos, pgsize=pgsize, page=page)
else:
tag = db.query(Tag).filter(or_(Tag.uuid == uuid, Tag.slug == uuid)).first()
numphotos = photo_auth_filter(db.query(func.count(Tag.id)).join(TagItem).join(PhotoSet)). \
numphotos = photoset_auth_filter(db.query(func.count(Tag.id)).join(TagItem).join(PhotoSet)). \
filter(Tag.id == tag.id).scalar()
photos = photo_auth_filter(db.query(PhotoSet)).join(TagItem).join(Tag). \
photos = photoset_auth_filter(db.query(PhotoSet)).join(TagItem).join(Tag). \
filter(Tag.id == tag.id). \
order_by(PhotoSet.date.desc()). \
offset(page * pgsize). \
@ -423,17 +430,21 @@ def main():
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
library = PhotoLibrary(args.database, args.library, args.cache)
# Get database connection
engine = get_db_engine(args.database)
# Setup database in web framework
cherrypy.tools.db = SATool()
SAEnginePlugin(cherrypy.engine, engine).subscribe()
# Create various internal tools
library_storage = FilesystemAdapter(args.library)
library_manager = LibraryManager(library_storage)
thumbnail_tool = ThumbGenerator(library_manager, args.cache)
# Setup and mount web ui
tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates"
web = PhotosWeb(library, tpl_dir)
def validate_password(realm, username, password):
if db.query(User).filter(User.name == username, User.password == pwhash(password)).first():
return True
return False
web = PhotosWeb(library_manager, thumbnail_tool, tpl_dir)
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True,
'error_page.403': web.error,
@ -445,12 +456,7 @@ def main():
'tools.auth_basic.realm': 'photolib',
'tools.auth_basic.checkpassword': validate_password}})
cherrypy.tools.db = SATool()
SAEnginePlugin(cherrypy.engine, library.engine).subscribe()
library_storage = FilesystemAdapter(args.library)
library_manager = LibraryManager(library_storage)
# Setup and mount API
api = PhotosApi(library_manager)
cherrypy.tree.mount(api, '/api', {'/': {'tools.trailing_slash.on': False,
'tools.auth_basic.on': True,
@ -458,6 +464,7 @@ def main():
'tools.auth_basic.checkpassword': validate_password,
'tools.db.on': True}})
# General config options
cherrypy.config.update({
'tools.sessions.on': True,
'tools.sessions.locking': 'explicit',
@ -471,6 +478,7 @@ def main():
'engine.autoreload.on': args.debug
})
# Setup signal handling and run it.
def signal_handler(signum, stack):
logging.critical('Got sig {}, exiting...'.format(signum))
cherrypy.engine.exit()

View File

@ -2,11 +2,28 @@ import sqlalchemy
import cherrypy
from cherrypy.process import plugins
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.pool import StaticPool, AssertionPool, NullPool
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
def get_db_engine(uri):
# TODO handle more uris
engine = sqlalchemy.create_engine('sqlite:///{}'.format(uri),
connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True)
Base.metadata.create_all(engine)
return engine
def get_db_session(uri):
engine = get_db_engine(uri)
session = sessionmaker()
session.configure(bind=engine)
return session
class DbAlias(object):
"""
This provides a shorter alias for the cherrypy.request.db object, which is a database session created bound to the

View File

@ -1,6 +1,5 @@
import argparse
import traceback
from photoapp.library import PhotoLibrary
from photoapp.image import get_jpg_info, get_hash, get_mtime, special_magic
from itertools import chain
from photoapp.types import Photo, PhotoSet, known_extensions, regular_images, files_raw, files_video, map_extension

View File

@ -2,66 +2,19 @@ import os
import sys
import traceback
from time import time
from sqlalchemy import create_engine
from sqlalchemy.pool import StaticPool, AssertionPool, NullPool # NOQA
from sqlalchemy.orm import sessionmaker
from photoapp.types import Base, Photo, PhotoSet # NOQA need to be loaded for orm setup
from sqlalchemy.exc import IntegrityError
from collections import defaultdict
from multiprocessing import Process
from PIL import Image, ImageOps
import tempfile
from shutil import copyfileobj
class PhotoLibrary(object):
def __init__(self, db_path, lib_path, cache_path):
self.path = lib_path
class ThumbGenerator(object):
def __init__(self, library, cache_path):
self.library = library
self.cache_path = cache_path
# TODO configure from env var
# TODO use the right pool and connection args depending on the url
# https://docs.sqlalchemy.org/en/13/core/pooling.html
self.engine = create_engine('sqlite:///{}'.format(db_path),
connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True)
Base.metadata.create_all(self.engine)
self.session = sessionmaker()
self.session.configure(bind=self.engine)
self._failed_thumbs_cache = defaultdict(dict)
def add_photoset(self, photoset):
"""
Commit a populated photoset object to the library. The paths in the photoset's file list entries will be updated
as the file is moved to the library path.
"""
# Create target directory
path = os.path.join(self.path, self.get_datedir_path(photoset.date))
os.makedirs(path, exist_ok=True)
moves = [] # Track files moved. If the sql transaction files, we'll undo these
for file in photoset.files:
dest = os.path.join(path, os.path.basename(file.path))
# Check if the name is already in use, rename new file if needed
dupe_rename = 1
while os.path.exists(dest):
fname = os.path.basename(file.path).split(".")
fname[-2] += "_{}".format(dupe_rename)
dest = os.path.join(path, '.'.join(fname))
dupe_rename += 1
os.rename(file.path, dest)
moves.append((file.path, dest))
file.path = dest.lstrip(self.path)
s = self.session()
s.add(photoset)
try:
s.commit()
except IntegrityError:
# Commit failed, undo the moves
for move in moves:
os.rename(move[1], move[0])
raise
def get_datedir_path(self, date):
"""
Return a path like 2018/3/31 given a datetime object representing the same date
@ -97,13 +50,19 @@ class PhotoLibrary(object):
thumb_width = min(thumb_width, i_width if i_width > 0 else 999999999) # TODO do we even have photo.width if PIL can't read the image?
thumb_height = min(thumb_height, i_height if i_height > 0 else 999999999) # TODO this seems bad
p = Process(target=self.gen_thumb, args=(os.path.join(self.path, photo.path), dest, thumb_width, thumb_height, photo.orientation))
p.start()
p.join()
if p.exitcode != 0:
self._failed_thumbs_cache[style][photo.uuid] = True # dont retry failed generations
return None
return os.path.abspath(dest)
# TODO have the subprocess download the file
with tempfile.TemporaryDirectory() as tmpdir:
fpath = os.path.join(tmpdir, "image")
with self.library.storage.open(photo.path, 'rb') as fsrc, open(fpath, 'wb') as fdest:
copyfileobj(fsrc, fdest)
p = Process(target=self.gen_thumb, args=(fpath, dest, thumb_width, thumb_height, photo.orientation))
p.start()
p.join()
if p.exitcode != 0:
self._failed_thumbs_cache[style][photo.uuid] = True # dont retry failed generations
return None
return os.path.abspath(dest)
return None
@staticmethod
@ -117,7 +76,7 @@ class PhotoLibrary(object):
thumb = ImageOps.fit(image, (width, height), Image.ANTIALIAS)
thumb.save(dest_img, 'JPEG')
print("Generated {} in {}s".format(dest_img, round(time() - start, 4)))
except:
except Exception:
traceback.print_exc()
if os.path.exists(dest_img):
os.unlink(dest_img)

View File

@ -1,24 +1,22 @@
import argparse
from photoapp.library import PhotoLibrary
from photoapp.types import User
from photoapp.common import pwhash
from photoapp.dbutils import get_db_session
from tabulate import tabulate
def create_user(library, username, password):
s = library.session()
def create_user(s, username, password):
s.add(User(name=username, password=pwhash(password)))
s.commit()
def list_users(library):
s = library.session()
print("id\tname")
for user in s.query(User).order_by(User.name).all():
print("{}\t{}".format(user.id, user.name))
def list_users(s):
print(tabulate([[user.id,
user.name] for user in s.query(User).order_by(User.name).all()],
headers=["id", "username"]))
def delete_user(library, username):
s = library.session()
def delete_user(s, username):
u = s.query(User).filter(User.name == username).first()
s.delete(u)
s.commit()
@ -27,6 +25,8 @@ def delete_user(library, username):
def main():
parser = argparse.ArgumentParser(description="User manipulation tool")
parser.add_argument("-d", "--database", help="database uri")
p_mode = parser.add_subparsers(dest='action', help='action to take')
p_create = p_mode.add_parser('create', help='create user')
@ -40,14 +40,14 @@ def main():
args = parser.parse_args()
library = PhotoLibrary("photos.db", "./library/", "./cache/")
session = get_db_session(args.database)()
if args.action == "create":
create_user(library, args.username, args.password)
create_user(session, args.username, args.password)
elif args.action == "list":
list_users(library)
list_users(session)
elif args.action == "delete":
delete_user(library, args.username)
delete_user(session, args.username)
else:
parser.print_help()

View File

@ -63,7 +63,7 @@ def require_auth(func):
return wrapped
def photo_auth_filter(query):
def photoset_auth_filter(query):
"""
Sqlalchemy helper: filter the given PhotoSet query to items that match the authorized user's PhotoStatus access
level. Currently, authed users can access ALL photos, and unauthed users can access only PhotoStatus.public

View File

@ -17,12 +17,12 @@ setup(name='photoapp',
entry_points={
"console_scripts": [
"photoappd = photoapp.daemon:main",
"photoimport = photoapp.ingest:main",
"photovalidate = photoapp.validate:main",
"photoinfo = photoapp.image:main",
"photooffset = photoapp.dateoffset:main",
"photousers = photoapp.users:main",
"photocli = photoapp.cli:main",
# "photoimport = photoapp.ingest:main", # broken for now
# "photovalidate = photoapp.validate:main",
#"photooffset = photoapp.dateoffset:main",
]
},
include_package_data=True,