Browse Source

standardize photo file and db access

api
dave 2 years ago
parent
commit
043ffbc543
  1. 12
      README.md
  2. 112
      photoapp/daemon.py
  3. 17
      photoapp/dbutils.py
  4. 1
      photoapp/ingest.py
  5. 79
      photoapp/thumb.py
  6. 28
      photoapp/users.py
  7. 2
      photoapp/utils.py
  8. 6
      setup.py

12
README.md

@ -51,8 +51,8 @@ Arguments are as follows:
* `--cache ./cache` - use this directory as a cache for things like thumbnails
* `--port 8080` - listen on http on port 8080
Depending on your setup, the `photousers` command can be used to create a user account. Login information is necessary
to see images marked as private or upload images.
Next, the `photousers` command can be used to create a user account. Login information is necessary to see images marked
as private or upload images.
* `photousers create -u dave -p mypassword`
@ -107,12 +107,13 @@ This would ingest all the files listed in `shas.txt` that aren't already in the
Roadmap
-------
- Stateless aka docker support
- Photo storage
- ~~Photo storage~~ done
- ~~Abstract the storage api~~ done
- ~~Standardize on API ingest~~ done
- Display and retrieval of images from the abstracted image store
- ~~Display and retrieval of images from the abstracted image store~~ done
- ~~Thumbnail gen~~ done
- Database support
- Get the web UI code (daemon.py) using the same db access method as the api
- ~~Get the web UI code (daemon.py) using the same db access method as the api~~
- Support any connection URI sqlalchemy is happy with
- Tune certain databases if their uri is detected (sqlite and threads lol)
- ~~Cache~~ done
@ -127,3 +128,4 @@ Roadmap
- Longer term ideas:
- "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest.
- Dynamic svg placeholder for images we can't open

112
photoapp/daemon.py

@ -1,27 +1,32 @@
import os
import cherrypy
import math
import logging
import cherrypy
from urllib.parse import urlparse
from datetime import datetime, timedelta
from photoapp.library import PhotoLibrary
from photoapp.thumb import ThumbGenerator
from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import desc
from sqlalchemy import func, and_, or_
from photoapp.common import pwhash
from photoapp.api import PhotosApi, LibraryManager, FilesystemAdapter
from photoapp.dbutils import SAEnginePlugin, SATool
import math
from urllib.parse import urlparse
from photoapp.utils import mime2ext, auth, require_auth, photo_auth_filter, slugify
from photoapp.dbutils import db
from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine
from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import desc, func, and_, or_
APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
def validate_password(realm, username, password):
if db.query(User).filter(User.name == username, User.password == pwhash(password)).first():
return True
return False
class PhotosWeb(object):
def __init__(self, library, template_dir):
def __init__(self, library, thumbtool, template_dir):
self.library = library
self.thumbtool = thumbtool
self.tpl = Environment(loader=FileSystemLoader(template_dir),
autoescape=select_autoescape(['html', 'xml']))
self.tpl.filters.update(mime2ext=mime2ext,
@ -79,8 +84,8 @@ class PhotosWeb(object):
/feed - main photo feed - show photos sorted by date, newest first
"""
page, pgsize = int(page), int(pgsize)
total_sets = photo_auth_filter(db.query(func.count(PhotoSet.id))).first()[0]
images = photo_auth_filter(db.query(PhotoSet)).order_by(PhotoSet.date.desc()). \
total_sets = photoset_auth_filter(db.query(func.count(PhotoSet.id))).first()[0]
images = photoset_auth_filter(db.query(PhotoSet)).order_by(PhotoSet.date.desc()). \
offset(pgsize * page).limit(pgsize).all()
yield self.render("feed.html", images=[i for i in images], page=page, pgsize=int(pgsize), total_sets=total_sets)
@ -89,11 +94,11 @@ class PhotosWeb(object):
"""
/stats - show server statistics
"""
images = photo_auth_filter(db.query(func.count(PhotoSet.uuid),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'))). \
images = photoset_auth_filter(db.query(func.count(PhotoSet.uuid),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'))). \
group_by('year', 'month').order_by(desc('year'), desc('month')).all()
tsize = photo_auth_filter(db.query(func.sum(Photo.size)).join(PhotoSet)).scalar() # pragma: manual auth
tsize = photoset_auth_filter(db.query(func.sum(Photo.size)).join(PhotoSet)).scalar() # pragma: manual auth
yield self.render("monthly.html", images=images, tsize=tsize)
@cherrypy.expose
@ -103,7 +108,7 @@ class PhotosWeb(object):
the given tag.
TODO using so many coordinates is slow in the browser. dedupe them somehow.
"""
query = photo_auth_filter(db.query(PhotoSet)).filter(PhotoSet.lat != 0, PhotoSet.lon != 0)
query = photoset_auth_filter(db.query(PhotoSet)).filter(PhotoSet.lat != 0, PhotoSet.lon != 0)
if a:
query = query.join(TagItem).join(Tag).filter(Tag.uuid == a)
if i:
@ -167,6 +172,7 @@ class PhotosWeb(object):
/login - enable super features by logging into the app
"""
cherrypy.session['authed'] = cherrypy.request.login
print("Authed as", cherrypy.session['authed'])
dest = "/feed" if "Referer" not in cherrypy.request.headers \
else urlparse(cherrypy.request.headers["Referer"]).path
raise cherrypy.HTTPRedirect(dest, 302)
@ -201,20 +207,20 @@ class DateView(object):
pgsize = 100
dt = datetime.strptime(date, "%Y-%m-%d")
dt_end = dt + timedelta(days=1)
total_sets = photo_auth_filter(db.query(func.count(PhotoSet.id))). \
total_sets = photoset_auth_filter(db.query(func.count(PhotoSet.id))). \
filter(and_(PhotoSet.date >= dt, PhotoSet.date < dt_end)).first()[0]
images = photo_auth_filter(db.query(PhotoSet)).filter(and_(PhotoSet.date >= dt,
PhotoSet.date < dt_end)).order_by(PhotoSet.date). \
images = photoset_auth_filter(db.query(PhotoSet)).filter(and_(PhotoSet.date >= dt,
PhotoSet.date < dt_end)).order_by(PhotoSet.date). \
offset(page * pgsize).limit(pgsize).all()
yield self.master.render("date.html", page=page, pgsize=pgsize, total_sets=total_sets,
images=[i for i in images], date=dt)
return
images = photo_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
PhotoSet.date).label('gdate'),
func.count('photos.id'),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'),
func.strftime('%d', PhotoSet.date).label('day'))). \
images = photoset_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
PhotoSet.date).label('gdate'),
func.count('photos.id'),
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'),
func.strftime('%d', PhotoSet.date).label('day'))). \
group_by('gdate').order_by(desc('year'), 'month', 'day').all()
yield self.master.render("dates.html", images=images)
@ -230,7 +236,7 @@ class ThumbnailView(object):
@cherrypy.expose
def index(self, item_type, thumb_size, uuid):
uuid = uuid.split(".")[0]
query = photo_auth_filter(db.query(Photo).join(PhotoSet))
query = photoset_auth_filter(db.query(Photo).join(PhotoSet))
query = query.filter(Photo.set.has(uuid=uuid)) if item_type == "set" \
else query.filter(Photo.uuid == uuid) if item_type == "one" \
@ -252,7 +258,7 @@ class ThumbnailView(object):
if not thumb_from:
raise cherrypy.HTTPError(404)
# TODO some lock around calls to this based on uuid
thumb_path = self.master.library.make_thumb(thumb_from, thumb_size)
thumb_path = self.master.thumbtool.make_thumb(thumb_from, thumb_size)
if thumb_path:
return cherrypy.lib.static.serve_file(thumb_path, "image/jpeg")
else:
@ -271,7 +277,7 @@ class DownloadView(object):
def index(self, item_type, uuid, preview=False):
uuid = uuid.split(".")[0]
query = None if item_type == "set" \
else photo_auth_filter(db.query(Photo)).filter(Photo.uuid == uuid) if item_type == "one" \
else photoset_auth_filter(db.query(Photo).join(PhotoSet)).filter(Photo.uuid == uuid) if item_type == "one" \
else None # TODO set download query
item = query.first()
@ -280,8 +286,8 @@ class DownloadView(object):
extra = {}
if not preview:
extra.update(disposition="attachement", name=os.path.basename(item.path))
return cherrypy.lib.static.serve_file(os.path.abspath(os.path.join(self.master.library.path, item.path)),
content_type=item.format, **extra)
return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'),
content_type=item.format, **extra)
@cherrypy.popargs('uuid')
@ -295,7 +301,8 @@ class PhotoView(object):
@cherrypy.expose
def index(self, uuid):
# uuid = uuid.split(".")[0]
photo = photo_auth_filter(db.query(PhotoSet)).filter(or_(PhotoSet.uuid == uuid, PhotoSet.slug == uuid)).first()
photo = photoset_auth_filter(db.query(PhotoSet)).filter(or_(PhotoSet.uuid == uuid,
PhotoSet.slug == uuid)).first()
if not photo:
raise cherrypy.HTTPError(404)
yield self.master.render("photo.html", image=photo)
@ -326,7 +333,7 @@ class PhotoView(object):
@cherrypy.expose
@require_auth
def edit(self, uuid):
photo = photo_auth_filter(db.query(PhotoSet)).filter(PhotoSet.uuid == uuid).first()
photo = photoset_auth_filter(db.query(PhotoSet)).filter(PhotoSet.uuid == uuid).first()
yield self.master.render("photo_edit.html", image=photo)
@ -343,17 +350,17 @@ class TagView(object):
page = int(page)
pgsize = 100
if uuid == "untagged":
numphotos = photo_auth_filter(db.query(func.count(PhotoSet.id))). \
numphotos = photoset_auth_filter(db.query(func.count(PhotoSet.id))). \
filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).scalar()
photos = photo_auth_filter(db.query(PhotoSet)).filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).\
photos = photoset_auth_filter(db.query(PhotoSet)).filter(~PhotoSet.id.in_(db.query(TagItem.set_id))).\
offset(page * pgsize). \
limit(pgsize).all()
yield self.master.render("untagged.html", images=photos, total_items=numphotos, pgsize=pgsize, page=page)
else:
tag = db.query(Tag).filter(or_(Tag.uuid == uuid, Tag.slug == uuid)).first()
numphotos = photo_auth_filter(db.query(func.count(Tag.id)).join(TagItem).join(PhotoSet)). \
numphotos = photoset_auth_filter(db.query(func.count(Tag.id)).join(TagItem).join(PhotoSet)). \
filter(Tag.id == tag.id).scalar()
photos = photo_auth_filter(db.query(PhotoSet)).join(TagItem).join(Tag). \
photos = photoset_auth_filter(db.query(PhotoSet)).join(TagItem).join(Tag). \
filter(Tag.id == tag.id). \
order_by(PhotoSet.date.desc()). \
offset(page * pgsize). \
@ -423,17 +430,21 @@ def main():
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
library = PhotoLibrary(args.database, args.library, args.cache)
# Get database connection
engine = get_db_engine(args.database)
tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates"
web = PhotosWeb(library, tpl_dir)
# Setup database in web framework
cherrypy.tools.db = SATool()
SAEnginePlugin(cherrypy.engine, engine).subscribe()
def validate_password(realm, username, password):
if db.query(User).filter(User.name == username, User.password == pwhash(password)).first():
return True
return False
# Create various internal tools
library_storage = FilesystemAdapter(args.library)
library_manager = LibraryManager(library_storage)
thumbnail_tool = ThumbGenerator(library_manager, args.cache)
# Setup and mount web ui
tpl_dir = os.path.join(APPROOT, "templates") if not args.debug else "templates"
web = PhotosWeb(library_manager, thumbnail_tool, tpl_dir)
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True,
'error_page.403': web.error,
@ -445,12 +456,7 @@ def main():
'tools.auth_basic.realm': 'photolib',
'tools.auth_basic.checkpassword': validate_password}})
cherrypy.tools.db = SATool()
SAEnginePlugin(cherrypy.engine, library.engine).subscribe()
library_storage = FilesystemAdapter(args.library)
library_manager = LibraryManager(library_storage)
# Setup and mount API
api = PhotosApi(library_manager)
cherrypy.tree.mount(api, '/api', {'/': {'tools.trailing_slash.on': False,
'tools.auth_basic.on': True,
@ -458,6 +464,7 @@ def main():
'tools.auth_basic.checkpassword': validate_password,
'tools.db.on': True}})
# General config options
cherrypy.config.update({
'tools.sessions.on': True,
'tools.sessions.locking': 'explicit',
@ -471,6 +478,7 @@ def main():
'engine.autoreload.on': args.debug
})
# Setup signal handling and run it.
def signal_handler(signum, stack):
logging.critical('Got sig {}, exiting...'.format(signum))
cherrypy.engine.exit()

17
photoapp/dbutils.py

@ -2,11 +2,28 @@ import sqlalchemy
import cherrypy
from cherrypy.process import plugins
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.pool import StaticPool, AssertionPool, NullPool
from sqlalchemy.orm import sessionmaker
Base = declarative_base()
def get_db_engine(uri):
# TODO handle more uris
engine = sqlalchemy.create_engine('sqlite:///{}'.format(uri),
connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True)
Base.metadata.create_all(engine)
return engine
def get_db_session(uri):
engine = get_db_engine(uri)
session = sessionmaker()
session.configure(bind=engine)
return session
class DbAlias(object):
"""
This provides a shorter alias for the cherrypy.request.db object, which is a database session created bound to the

1
photoapp/ingest.py

@ -1,6 +1,5 @@
import argparse
import traceback
from photoapp.library import PhotoLibrary
from photoapp.image import get_jpg_info, get_hash, get_mtime, special_magic
from itertools import chain
from photoapp.types import Photo, PhotoSet, known_extensions, regular_images, files_raw, files_video, map_extension

photoapp/library.py → photoapp/thumb.py

28
photoapp/users.py

@ -1,24 +1,22 @@
import argparse
from photoapp.library import PhotoLibrary
from photoapp.types import User
from photoapp.common import pwhash
from photoapp.dbutils import get_db_session
from tabulate import tabulate
def create_user(library, username, password):
s = library.session()
def create_user(s, username, password):
s.add(User(name=username, password=pwhash(password)))
s.commit()
def list_users(library):
s = library.session()
print("id\tname")
for user in s.query(User).order_by(User.name).all():
print("{}\t{}".format(user.id, user.name))
def list_users(s):
print(tabulate([[user.id,
user.name] for user in s.query(User).order_by(User.name).all()],
headers=["id", "username"]))
def delete_user(library, username):
s = library.session()
def delete_user(s, username):
u = s.query(User).filter(User.name == username).first()
s.delete(u)
s.commit()
@ -27,6 +25,8 @@ def delete_user(library, username):
def main():
parser = argparse.ArgumentParser(description="User manipulation tool")
parser.add_argument("-d", "--database", help="database uri")
p_mode = parser.add_subparsers(dest='action', help='action to take')
p_create = p_mode.add_parser('create', help='create user')
@ -40,14 +40,14 @@ def main():
args = parser.parse_args()
library = PhotoLibrary("photos.db", "./library/", "./cache/")
session = get_db_session(args.database)()
if args.action == "create":
create_user(library, args.username, args.password)
create_user(session, args.username, args.password)
elif args.action == "list":
list_users(library)
list_users(session)
elif args.action == "delete":
delete_user(library, args.username)
delete_user(session, args.username)
else:
parser.print_help()

2
photoapp/utils.py

@ -63,7 +63,7 @@ def require_auth(func):
return wrapped
def photo_auth_filter(query):
def photoset_auth_filter(query):
"""
Sqlalchemy helper: filter the given PhotoSet query to items that match the authorized user's PhotoStatus access
level. Currently, authed users can access ALL photos, and unauthed users can access only PhotoStatus.public

6
setup.py

@ -17,12 +17,12 @@ setup(name='photoapp',
entry_points={
"console_scripts": [
"photoappd = photoapp.daemon:main",
"photoimport = photoapp.ingest:main",
"photovalidate = photoapp.validate:main",
"photoinfo = photoapp.image:main",
"photooffset = photoapp.dateoffset:main",
"photousers = photoapp.users:main",
"photocli = photoapp.cli:main",
# "photoimport = photoapp.ingest:main", # broken for now
# "photovalidate = photoapp.validate:main",
#"photooffset = photoapp.dateoffset:main",
]
},
include_package_data=True,

Loading…
Cancel
Save