From 50a55f08dfa7f64ef5c006d9ee558c8f8c48e636 Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 13 Jul 2019 15:47:59 -0700 Subject: [PATCH] missing gps data fix --- README.md | 4 + photoapp/api.py | 9 +- photoapp/daemon.py | 2 +- photoapp/image.py | 13 ++- photoapp/migrate.py | 210 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 231 insertions(+), 7 deletions(-) create mode 100644 photoapp/migrate.py diff --git a/README.md b/README.md index c17619d..396488d 100644 --- a/README.md +++ b/README.md @@ -132,3 +132,7 @@ Roadmap - "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest. - Dynamic svg placeholder for images we can't open - Proactive thumbnail generation + - on photo thumbs on the feed, a little "2" indicating there are 2 images in the set (or w/e number for that item) + - dark theme + - more information from the images like on http://exif.regex.info/exif.cgi + diff --git a/photoapp/api.py b/photoapp/api.py index df40ce3..1f6ab4c 100644 --- a/photoapp/api.py +++ b/photoapp/api.py @@ -9,6 +9,7 @@ from photoapp.image import special_magic_fobj from photoapp.storage import StorageAdapter from photoapp.dbutils import db from contextlib import closing +from decimal import Decimal import traceback @@ -117,13 +118,17 @@ class PhotosApiV1(object): ps = db.query(PhotoSet).filter(PhotoSet.uuid == meta["uuid"]).first() if not ps: return abort_upload("parent uuid not found") - ps.date = photo_date - ps.date_real = photo_date ps.files.extend(photo_objs) + if not ps.lat and meta["lat"] and meta["lon"]: + ps.lat = Decimal(meta["lat"]) + ps.lon = Decimal(meta["lon"]) + else: ps = PhotoSet(uuid=genuuid(), date=photo_date, date_real=photo_date, # TODO support time offsets + lat=Decimal(meta["lat"]) if meta["lat"] else None, + lon=Decimal(meta["lon"]) if meta["lon"] else None, files=photo_objs) # TODO support title field et db.add(ps) diff --git a/photoapp/daemon.py b/photoapp/daemon.py index 22f24f0..aebbb29 100644 --- a/photoapp/daemon.py +++ b/photoapp/daemon.py @@ -289,7 +289,7 @@ class DownloadView(object): raise cherrypy.HTTPError(404) extra = {} if not preview: - extra.update(disposition="attachement", name=os.path.basename(item.path)) + extra.update(disposition="attachement", name=item.fname) return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'), content_type=item.format, **extra) diff --git a/photoapp/image.py b/photoapp/image.py index 8f07587..fe9d5e2 100644 --- a/photoapp/image.py +++ b/photoapp/image.py @@ -46,10 +46,18 @@ def get_hash(path): def get_exif_data(path): + with open(path, 'rb') as f: + dateinfo, gpsinfo, sizeinfo, orientationinfo = get_exif_data_fobj(f) + if dateinfo is None: + dateinfo = get_mtime(path) + return dateinfo, gpsinfo, sizeinfo, orientationinfo + + +def get_exif_data_fobj(fobj): """ Return a (datetime, (decimal, decimal), (width, height), rotation) tuple describing the photo's exif date and gps coordinates """ - img = Image.open(path) + img = Image.open(fobj) # TODO do i need to close this? datestr = None gpsinfo = None @@ -94,9 +102,6 @@ def get_exif_data(path): gps_x *= -1 gpsinfo = (gps_y, gps_x) - if dateinfo is None: - dateinfo = get_mtime(path) - return dateinfo, gpsinfo, sizeinfo, orientationinfo diff --git a/photoapp/migrate.py b/photoapp/migrate.py new file mode 100644 index 0000000..31c706a --- /dev/null +++ b/photoapp/migrate.py @@ -0,0 +1,210 @@ +import os +import json +import sqlalchemy +from photoapp.dbutils import get_db_engine +from photoapp.types import Photo, PhotoSet, generate_storage_path +from photoapp.storage import uri_to_storage +from photoapp.image import get_exif_data_fobj +import shutil +from contextlib import closing +from concurrent.futures import ThreadPoolExecutor, as_completed + + +STORAGE_URI = "file://./library" +DB_URI = "sqlite:///photos.db" + + +""" +Before this application had support for multiple file storage backends, it only supported the filesystem and it was done +in a way that wasn't abstracted. This script contains steps in migrating and old database and file tree to the +modern one. + +SQLite to MySQL Steps: +====================== + +Migrating from mysql to sqlite is NOT required. Follow this part if you're converting from sqlite to mysql. + +If you're not migrating from sqlite to mysql, you just need to do the equivalent of step 4 to your sqlite database. + +1) Export the old sqlite database's contents. We need the data only, not the schema. First dump everything: + +``` +$ sqlite3 photos_old.db +sqlite> .output old.sql +sqlite> .dump +sqlite> .exit +``` + + +Then using your favorite editor format all the INSERT statements like: + +``` +START TRANSACTION; +SET FOREIGN_KEY_CHECKS=0; + +(many insert statements here) + +SET FOREIGN_KEY_CHECKS=1; +COMMIT; +``` + +2) Populate a mysql database with the app's schema + +You can just start and stop the app with it pointed at mysql like a normal person would. + +3) Modify the mysql schema to play nice with our data: + +``` +mysql> alter table files drop fname; +alter table files modify column path varchar(1024); +``` + +3) Import the sqlite data dump into mysql + +4) Put the mysql schema back + +``` +mysql> alter table files add fname varchar(256); +```` + +Now, continue with the steps below. When finished, it's safest to do a data-only dump from mysql, delete and recreate +your schema, and import the mysql data dump. + + +Filesystem migration +==================== + +This part is required, the layout of your library directory must be migrated. If you're moving to s3, that is done +here too. + +1) Run this script pointed at your new database (edit the connection uri and main function to run part1 below) + +It will produce a renames.json in the current dir, which is needed in later steps. + +2) Run this script pointed at your new storage (edit the storage uri and main function to run part2 below) + +This copies your photos to the new library. +""" + + +def getstorage(): + return uri_to_storage(STORAGE_URI) + + +def getsession(): + engine = get_db_engine(DB_URI) + sessionmaker = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) + sessionmaker.configure(bind=engine) + return sessionmaker() + + +def part1(): + session = getsession() + renames = [] + + if os.path.exists("renames.json"): + raise Exception("dont run me twice!!") + + for p in session.query(Photo).all(): + fname = os.path.basename(p.path) + p.fname = fname + ext = fname.split(".")[-1] + newpath = generate_storage_path(p.set.date_real, p.hash, ext) + assert p.path != newpath + renames.append((p.path, newpath)) + p.path = newpath + + with open("renames.json", "w") as f: + json.dump(renames, f) + + session.commit() + # session.rollback() + + +def part2(): + with open("path/to/renames.json", "r") as f: + renames = json.load(f) + + library_storage = getstorage() + + numdone = 0 + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {pool.submit(rename_set, library_storage, set_[0], set_[1]): set_ for set_ in renames} + print("Working...") + for future in as_completed(futures.keys()): + set_ = futures[future] + e = future.exception() + if e: + print("Screwed up on:", set_) + raise e + numdone += 1 + print("Done:", numdone) + + print("Done!") + + +def rename_set(storage, src_path, dest_path): + with closing(storage.open(dest_path, 'wb')) as df: + with open(src_path, 'rb') as sf: + shutil.copyfileobj(sf, df) + + +""" +At one point, the cli contained errors causing it to not submit photo gps data. Running migrate_gpsfix rescans files for +said gps data and updates the database. +""" + + +def migrate_gpsfix(): + session = getsession() + storage = getstorage() + + done = 0 + # iterate all images + for p in session.query(PhotoSet).filter(sqlalchemy.or_(PhotoSet.lat == 0, + PhotoSet.lat == None # NOQA: E711 + )).all(): + done += 1 + print(done) + + if p.lat and p.lon: + continue + + # just bail if there's a CR2, the paired jpg is known not to have gps data in my dataset :) + if any(["image/x-canon-cr2" == i.format for i in p.files]): + continue + + # pick the jpg out of the set + jpegs = [] + for pfile in p.files: + if pfile.format == "image/jpeg": + jpegs.append(pfile) + + if not jpegs: # no files with gps data found + continue + + gpsdata = None + for img in jpegs: + # scan it for gps data + # print(p.uuid, img.fname) + with closing(storage.open(img.path, 'rb')) as fsrc: + _, gpsdata, _, _ = get_exif_data_fobj(fsrc) # (datetime, (decimal, decimal), (width, height), rotation) + # print(gpsdata) + + if gpsdata and gpsdata[0]: + break + + if not gpsdata: + continue + + print(p.uuid, "->", gpsdata) + p.lat, p.lon = gpsdata[0], gpsdata[1] + + # update the db + session.commit() + + +# __name__ == '__main__' and part1() +# __name__ == '__main__' and part2() +# __name__ == '__main__' and migrate_gpsfix()