From 06dea40d24fe891d2df221ad1962b9f086d5386b Mon Sep 17 00:00:00 2001 From: dave Date: Thu, 14 Nov 2019 21:28:32 -0800 Subject: [PATCH] fix sqlite support --- .dockerignore | 1 + Dockerfile | 3 +-- photoapp/api.py | 8 +++++--- photoapp/cli.py | 6 +++++- photoapp/daemon.py | 19 +++++++++++-------- photoapp/dbutils.py | 21 +++++++++++++++++++++ photoapp/image.py | 4 +--- templates/dates.html | 15 +++++++-------- 8 files changed, 52 insertions(+), 25 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2658560..2b0554e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ source/ source_copy/ raws/ *.sql +vidtest/ diff --git a/Dockerfile b/Dockerfile index cefc8f4..29aaa2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,7 @@ RUN apt-get update && \ COPY --from=frontend /tmp/code/styles/dist/style.css /tmp/code/styles/dist/style.css -RUN pip3 install -U pip && \ - cd /tmp/code && \ +RUN cd /tmp/code && \ pip3 install -r requirements.txt && \ python3 setup.py install && \ useradd --uid 1000 app && \ diff --git a/photoapp/api.py b/photoapp/api.py index 1a5a552..f350483 100644 --- a/photoapp/api.py +++ b/photoapp/api.py @@ -40,7 +40,8 @@ class PhotosApiV1(object): """ upload accepts one photoset (multiple images) """ - # load and verify metadata + # TODO stage files in tmp storage since this can DEFINITELY lead to data loss + # TODO refactor function stored_files = [] def abort_upload(reason): @@ -72,9 +73,10 @@ class PhotosApiV1(object): ext = get_extension(file.filename) assert ext in known_extensions photo_path = generate_storage_path(photo_date, photo_meta['hash'][0:8], ext) - # shouldn't we just trust the database? if self.library.storage.exists(photo_path): - return abort_upload(f"file already in library: {photo_path}") + pass + # we trust that the database has enforced uniqueness of this image + # return abort_upload(f"file already in library: {photo_path}") # write file to the path (and copy sha while in flight) with closing(self.library.storage.open(photo_path, 'wb')) as f: diff --git a/photoapp/cli.py b/photoapp/cli.py index 56f80e8..f73753b 100644 --- a/photoapp/cli.py +++ b/photoapp/cli.py @@ -4,7 +4,7 @@ import argparse import requests from requests.exceptions import HTTPError from photoapp.utils import get_extension -from photoapp.types import known_extensions, PhotoStatus, Tag +from photoapp.types import known_extensions, PhotoStatus from photoapp.common import pwhash from photoapp.ingest import get_photosets from urllib.parse import urlparse @@ -233,6 +233,10 @@ def main(): raise elif args.action == "ingest": + if args.copy_of and args.tag: + raise NotImplementedError("--copy-of not allowed with --tag") + + # TODO refactor this to be less ugly tag = None if args.tag: tag = args.tag.lower() diff --git a/photoapp/daemon.py b/photoapp/daemon.py index 2c7b726..dff4285 100644 --- a/photoapp/daemon.py +++ b/photoapp/daemon.py @@ -9,7 +9,7 @@ from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User from photoapp.dbsession import DatabaseSession from photoapp.common import pwhash from photoapp.api import PhotosApi, LibraryManager -from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine +from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine, driver_statement from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify from photoapp.storage import uri_to_storage from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -218,13 +218,16 @@ class DateView(object): yield self.master.render("date.html", page=page, pgsize=pgsize, total_sets=total_sets, images=[i for i in images], date=dt) return - images = photoset_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d', - PhotoSet.date).label('gdate'), - func.count('photos.id'), - func.strftime('%Y', PhotoSet.date).label('year'), - func.strftime('%m', PhotoSet.date).label('month'), - func.strftime('%d', PhotoSet.date).label('day'))). \ - group_by('gdate').order_by(desc('year'), 'month', 'day').all() + + date_format = driver_statement({"sqlite": lambda date_format, value: func.strftime(date_format, value), + "mysql": lambda date_format, value: func.date_format(value, date_format)}) + + images = photoset_auth_filter(db.query( + func.count(PhotoSet.id), + date_format('%Y', PhotoSet.date).label('year'), + date_format('%m', PhotoSet.date).label('month'), + date_format('%d', PhotoSet.date).label('day'))). \ + group_by('year', 'month', 'day').order_by(desc('year'), 'month', 'day').all() yield self.master.render("dates.html", images=images) diff --git a/photoapp/dbutils.py b/photoapp/dbutils.py index 9951b05..d96ff95 100644 --- a/photoapp/dbutils.py +++ b/photoapp/dbutils.py @@ -35,6 +35,27 @@ def get_db_session(uri): return session +def driver_statement(statements): + """ + Select a value from the passed dict based on the sql driver in use. Must be used in request context. For example: + Sqlite and mysql use different date functions. This function can be used to build queries supporting either: + + date_format = driver_statement({"sqlite": lambda date_format, value: func.strftime(date_format, value), + "mysql": lambda date_format, value: func.date_format(value, date_format)}) + + rows = db.query(PhotoSet.id, date_format('%Y-%m=%d', PhotoSet.date).label('year')).all() + + :param statements: dict of driver_type->value. since sqlalchemy drivers vary per language (e.g. pymysql, pysqlite), + it is checked if the driver_type is a substring of sqlalchemy's driver name. + :type statements: dict + """ + driver = cherrypy.request.db.connection().engine.driver + for key, lambda_ in statements.items(): + if key in driver: + return lambda_ + raise Exception(f"Statement not supported for driver {driver}") + + class DbAlias(object): """ This provides a shorter alias for the cherrypy.request.db object, which is a database session created bound to the diff --git a/photoapp/image.py b/photoapp/image.py index fe9d5e2..eba0eed 100644 --- a/photoapp/image.py +++ b/photoapp/image.py @@ -14,9 +14,7 @@ def get_jpg_info(fpath): date, gps, dimensions, orientation = get_exif_data(fpath) if date is None: - import pdb - pdb.set_trace() - raise Exception("fuk") + raise Exception("No date found, panicing for unknown reasons!") # gps is set to 0,0 if unavailable lat, lon = gps or [None, None] diff --git a/templates/dates.html b/templates/dates.html index dcac9d5..95a1467 100644 --- a/templates/dates.html +++ b/templates/dates.html @@ -13,16 +13,15 @@
{% set locals.year = "" %} {% set locals.month = "" %} - {% for item, date, count, _, _, _ in images %} - {% if item.date.year != locals.year %} - {% set locals.year = item.date.year %} -

{{ item.date.year }}

+ {% for count, year, month, day in images %} + {% if year != locals.year %} + {% set locals.year = year %} +

{{ year }}

{% endif %} - {% if item.date.month != locals.month %} - {% set locals.month = item.date.month %} -

{{ item.date.strftime("%B") }}

+ {% if month != locals.month %} + {% set locals.month = month %} {% endif %} - {{ date }} ({{ count }}) + {{ month }}-{{ day }} ({{ count }}) {% endfor %}