fix sqlite support

This commit is contained in:
dave 2019-11-14 21:28:32 -08:00
parent 72346c59fb
commit 06dea40d24
8 changed files with 52 additions and 25 deletions

View File

@ -14,3 +14,4 @@ source/
source_copy/ source_copy/
raws/ raws/
*.sql *.sql
vidtest/

View File

@ -22,8 +22,7 @@ RUN apt-get update && \
COPY --from=frontend /tmp/code/styles/dist/style.css /tmp/code/styles/dist/style.css COPY --from=frontend /tmp/code/styles/dist/style.css /tmp/code/styles/dist/style.css
RUN pip3 install -U pip && \ RUN cd /tmp/code && \
cd /tmp/code && \
pip3 install -r requirements.txt && \ pip3 install -r requirements.txt && \
python3 setup.py install && \ python3 setup.py install && \
useradd --uid 1000 app && \ useradd --uid 1000 app && \

View File

@ -40,7 +40,8 @@ class PhotosApiV1(object):
""" """
upload accepts one photoset (multiple images) upload accepts one photoset (multiple images)
""" """
# load and verify metadata # TODO stage files in tmp storage since this can DEFINITELY lead to data loss
# TODO refactor function
stored_files = [] stored_files = []
def abort_upload(reason): def abort_upload(reason):
@ -72,9 +73,10 @@ class PhotosApiV1(object):
ext = get_extension(file.filename) ext = get_extension(file.filename)
assert ext in known_extensions assert ext in known_extensions
photo_path = generate_storage_path(photo_date, photo_meta['hash'][0:8], ext) photo_path = generate_storage_path(photo_date, photo_meta['hash'][0:8], ext)
# shouldn't we just trust the database?
if self.library.storage.exists(photo_path): if self.library.storage.exists(photo_path):
return abort_upload(f"file already in library: {photo_path}") pass
# we trust that the database has enforced uniqueness of this image
# return abort_upload(f"file already in library: {photo_path}")
# write file to the path (and copy sha while in flight) # write file to the path (and copy sha while in flight)
with closing(self.library.storage.open(photo_path, 'wb')) as f: with closing(self.library.storage.open(photo_path, 'wb')) as f:

View File

@ -4,7 +4,7 @@ import argparse
import requests import requests
from requests.exceptions import HTTPError from requests.exceptions import HTTPError
from photoapp.utils import get_extension from photoapp.utils import get_extension
from photoapp.types import known_extensions, PhotoStatus, Tag from photoapp.types import known_extensions, PhotoStatus
from photoapp.common import pwhash from photoapp.common import pwhash
from photoapp.ingest import get_photosets from photoapp.ingest import get_photosets
from urllib.parse import urlparse from urllib.parse import urlparse
@ -233,6 +233,10 @@ def main():
raise raise
elif args.action == "ingest": elif args.action == "ingest":
if args.copy_of and args.tag:
raise NotImplementedError("--copy-of not allowed with --tag")
# TODO refactor this to be less ugly
tag = None tag = None
if args.tag: if args.tag:
tag = args.tag.lower() tag = args.tag.lower()

View File

@ -9,7 +9,7 @@ from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
from photoapp.dbsession import DatabaseSession from photoapp.dbsession import DatabaseSession
from photoapp.common import pwhash from photoapp.common import pwhash
from photoapp.api import PhotosApi, LibraryManager from photoapp.api import PhotosApi, LibraryManager
from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine, driver_statement
from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify
from photoapp.storage import uri_to_storage from photoapp.storage import uri_to_storage
from jinja2 import Environment, FileSystemLoader, select_autoescape from jinja2 import Environment, FileSystemLoader, select_autoescape
@ -218,13 +218,16 @@ class DateView(object):
yield self.master.render("date.html", page=page, pgsize=pgsize, total_sets=total_sets, yield self.master.render("date.html", page=page, pgsize=pgsize, total_sets=total_sets,
images=[i for i in images], date=dt) images=[i for i in images], date=dt)
return return
images = photoset_auth_filter(db.query(PhotoSet, func.strftime('%Y-%m-%d',
PhotoSet.date).label('gdate'), date_format = driver_statement({"sqlite": lambda date_format, value: func.strftime(date_format, value),
func.count('photos.id'), "mysql": lambda date_format, value: func.date_format(value, date_format)})
func.strftime('%Y', PhotoSet.date).label('year'),
func.strftime('%m', PhotoSet.date).label('month'), images = photoset_auth_filter(db.query(
func.strftime('%d', PhotoSet.date).label('day'))). \ func.count(PhotoSet.id),
group_by('gdate').order_by(desc('year'), 'month', 'day').all() date_format('%Y', PhotoSet.date).label('year'),
date_format('%m', PhotoSet.date).label('month'),
date_format('%d', PhotoSet.date).label('day'))). \
group_by('year', 'month', 'day').order_by(desc('year'), 'month', 'day').all()
yield self.master.render("dates.html", images=images) yield self.master.render("dates.html", images=images)

View File

@ -35,6 +35,27 @@ def get_db_session(uri):
return session return session
def driver_statement(statements):
"""
Select a value from the passed dict based on the sql driver in use. Must be used in request context. For example:
Sqlite and mysql use different date functions. This function can be used to build queries supporting either:
date_format = driver_statement({"sqlite": lambda date_format, value: func.strftime(date_format, value),
"mysql": lambda date_format, value: func.date_format(value, date_format)})
rows = db.query(PhotoSet.id, date_format('%Y-%m=%d', PhotoSet.date).label('year')).all()
:param statements: dict of driver_type->value. since sqlalchemy drivers vary per language (e.g. pymysql, pysqlite),
it is checked if the driver_type is a substring of sqlalchemy's driver name.
:type statements: dict
"""
driver = cherrypy.request.db.connection().engine.driver
for key, lambda_ in statements.items():
if key in driver:
return lambda_
raise Exception(f"Statement not supported for driver {driver}")
class DbAlias(object): class DbAlias(object):
""" """
This provides a shorter alias for the cherrypy.request.db object, which is a database session created bound to the This provides a shorter alias for the cherrypy.request.db object, which is a database session created bound to the

View File

@ -14,9 +14,7 @@ def get_jpg_info(fpath):
date, gps, dimensions, orientation = get_exif_data(fpath) date, gps, dimensions, orientation = get_exif_data(fpath)
if date is None: if date is None:
import pdb raise Exception("No date found, panicing for unknown reasons!")
pdb.set_trace()
raise Exception("fuk")
# gps is set to 0,0 if unavailable # gps is set to 0,0 if unavailable
lat, lon = gps or [None, None] lat, lon = gps or [None, None]

View File

@ -13,16 +13,15 @@
<div class="date-feed"> <div class="date-feed">
{% set locals.year = "" %} {% set locals.year = "" %}
{% set locals.month = "" %} {% set locals.month = "" %}
{% for item, date, count, _, _, _ in images %} {% for count, year, month, day in images %}
{% if item.date.year != locals.year %} {% if year != locals.year %}
{% set locals.year = item.date.year %} {% set locals.year = year %}
<div class="feed-divider year"><h4>{{ item.date.year }}</h4></div> <div class="feed-divider year"><h4>{{ year }}</h4></div>
{% endif %} {% endif %}
{% if item.date.month != locals.month %} {% if month != locals.month %}
{% set locals.month = item.date.month %} {% set locals.month = month %}
<div class="feed-divider month"><h4>{{ item.date.strftime("%B") }}</h4></div>
{% endif %} {% endif %}
<a class="date-item{% if count > 50 %} many{% endif %}" href="/date/{{ date }}">{{ date }} ({{ count }})</a> <a class="date-item{% if count > 50 %} many{% endif %}" href="/date/{{ year }}-{{ month }}-{{ day }}">{{ month }}-{{ day }} ({{ count }})</a>
{% endfor %} {% endfor %}
</div> </div>