missing gps data fix

README.md

@@ -132,3 +132,7 @@ Roadmap
     - "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest.
     - Dynamic svg placeholder for images we can't open
     - Proactive thumbnail generation
+    - on photo thumbs on the feed, a little "2" indicating there are 2 images in the set (or w/e number for that item)
+    - dark theme
+    - more information from the images like on http://exif.regex.info/exif.cgi
+

photoapp/api.py

@@ -9,6 +9,7 @@ from photoapp.image import special_magic_fobj
 from photoapp.storage import StorageAdapter
 from photoapp.dbutils import db
 from contextlib import closing
+from decimal import Decimal
 import traceback
 
 
@@ -117,13 +118,17 @@ class PhotosApiV1(object):
             ps = db.query(PhotoSet).filter(PhotoSet.uuid == meta["uuid"]).first()
             if not ps:
                 return abort_upload("parent uuid not found")
-            ps.date = photo_date
-            ps.date_real = photo_date
             ps.files.extend(photo_objs)
+            if not ps.lat and meta["lat"] and meta["lon"]:
+                ps.lat = Decimal(meta["lat"])
+                ps.lon = Decimal(meta["lon"])
+
         else:
             ps = PhotoSet(uuid=genuuid(),
                           date=photo_date,
                           date_real=photo_date,  # TODO support time offsets
+                          lat=Decimal(meta["lat"]) if meta["lat"] else None,
+                          lon=Decimal(meta["lon"]) if meta["lon"] else None,
                           files=photo_objs)  # TODO support title field et
             db.add(ps)
 

photoapp/daemon.py

@@ -289,7 +289,7 @@ class DownloadView(object):
             raise cherrypy.HTTPError(404)
         extra = {}
         if not preview:
-            extra.update(disposition="attachement", name=os.path.basename(item.path))
+            extra.update(disposition="attachement", name=item.fname)
         return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'),
                                                  content_type=item.format, **extra)
 

photoapp/image.py

@@ -46,10 +46,18 @@ def get_hash(path):
 
 
 def get_exif_data(path):
+    with open(path, 'rb') as f:
+        dateinfo, gpsinfo, sizeinfo, orientationinfo = get_exif_data_fobj(f)
+        if dateinfo is None:
+            dateinfo = get_mtime(path)
+        return dateinfo, gpsinfo, sizeinfo, orientationinfo
+
+
+def get_exif_data_fobj(fobj):
     """
     Return a (datetime, (decimal, decimal), (width, height), rotation) tuple describing the photo's exif date and gps coordinates
     """
-    img = Image.open(path)
+    img = Image.open(fobj)  # TODO do i need to close this?
 
     datestr = None
     gpsinfo = None
@@ -94,9 +102,6 @@ def get_exif_data(path):
                         gps_x *= -1
                     gpsinfo = (gps_y, gps_x)
 
-    if dateinfo is None:
-        dateinfo = get_mtime(path)
-
     return dateinfo, gpsinfo, sizeinfo, orientationinfo
 
 

photoapp/migrate.py

@@ -0,0 +1,210 @@
+import os
+import json
+import sqlalchemy
+from photoapp.dbutils import get_db_engine
+from photoapp.types import Photo, PhotoSet, generate_storage_path
+from photoapp.storage import uri_to_storage
+from photoapp.image import get_exif_data_fobj
+import shutil
+from contextlib import closing
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+
+STORAGE_URI = "file://./library"
+DB_URI = "sqlite:///photos.db"
+
+
+"""
+Before this application had support for multiple file storage backends, it only supported the filesystem and it was done
+in a way that wasn't abstracted. This script contains steps in migrating and old database and file tree to the
+modern one.
+
+SQLite to MySQL Steps:
+======================
+
+Migrating from mysql to sqlite is NOT required. Follow this part if you're converting from sqlite to mysql.
+
+If you're not migrating from sqlite to mysql, you just need to do the equivalent of step 4 to your sqlite database.
+
+1) Export the old sqlite database's contents. We need the data only, not the schema. First dump everything:
+
+```
+$ sqlite3 photos_old.db
+sqlite> .output old.sql
+sqlite> .dump
+sqlite> .exit
+```
+
+
+Then using your favorite editor format all the INSERT statements like:
+
+```
+START TRANSACTION;
+SET FOREIGN_KEY_CHECKS=0;
+
+(many insert statements  here)
+
+SET FOREIGN_KEY_CHECKS=1;
+COMMIT;
+```
+
+2) Populate a mysql database with the app's schema
+
+You can just start and stop the app with it pointed at mysql like a normal person would.
+
+3) Modify the mysql schema to play nice with our data:
+
+```
+mysql> alter table files drop fname;
+alter table files modify column path varchar(1024);
+```
+
+3) Import the sqlite data dump into mysql
+
+4) Put the mysql schema back
+
+```
+mysql> alter table files add fname varchar(256);
+````
+
+Now, continue with the steps below. When finished, it's safest to do a data-only dump from mysql, delete and recreate
+your schema, and import the mysql data dump.
+
+
+Filesystem migration
+====================
+
+This part is required, the layout of your library directory must be migrated. If you're moving to s3, that is done
+here too.
+
+1) Run this script pointed at your new database (edit the connection uri and main function to run part1 below)
+
+It will produce a renames.json in the current dir, which is needed in later steps.
+
+2) Run this script pointed at your new storage (edit the storage uri and main function to run part2 below)
+
+This copies your photos to the new library.
+"""
+
+
+def getstorage():
+    return uri_to_storage(STORAGE_URI)
+
+
+def getsession():
+    engine = get_db_engine(DB_URI)
+    sessionmaker = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
+    sessionmaker.configure(bind=engine)
+    return sessionmaker()
+
+
+def part1():
+    session = getsession()
+    renames = []
+
+    if os.path.exists("renames.json"):
+        raise Exception("dont run me twice!!")
+
+    for p in session.query(Photo).all():
+        fname = os.path.basename(p.path)
+        p.fname = fname
+        ext = fname.split(".")[-1]
+        newpath = generate_storage_path(p.set.date_real, p.hash, ext)
+        assert p.path != newpath
+        renames.append((p.path, newpath))
+        p.path = newpath
+
+    with open("renames.json", "w") as f:
+        json.dump(renames, f)
+
+    session.commit()
+    # session.rollback()
+
+
+def part2():
+    with open("path/to/renames.json", "r") as f:
+        renames = json.load(f)
+
+    library_storage = getstorage()
+
+    numdone = 0
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        futures = {pool.submit(rename_set, library_storage, set_[0], set_[1]): set_ for set_ in renames}
+        print("Working...")
+        for future in as_completed(futures.keys()):
+            set_ = futures[future]
+            e = future.exception()
+            if e:
+                print("Screwed up on:", set_)
+                raise e
+            numdone += 1
+            print("Done:", numdone)
+
+        print("Done!")
+
+
+def rename_set(storage, src_path, dest_path):
+    with closing(storage.open(dest_path, 'wb')) as df:
+        with open(src_path, 'rb') as sf:
+                shutil.copyfileobj(sf, df)
+
+
+"""
+At one point, the cli contained errors causing it to not submit photo gps data. Running migrate_gpsfix rescans files for
+said gps data and updates the database.
+"""
+
+
+def migrate_gpsfix():
+    session = getsession()
+    storage = getstorage()
+
+    done = 0
+    # iterate all images
+    for p in session.query(PhotoSet).filter(sqlalchemy.or_(PhotoSet.lat == 0,
+                                                           PhotoSet.lat == None  # NOQA: E711
+                                                           )).all():
+        done += 1
+        print(done)
+
+        if p.lat and p.lon:
+            continue
+
+        # just bail if there's a CR2, the paired jpg is known not to have gps data in my dataset :)
+        if any(["image/x-canon-cr2" == i.format for i in p.files]):
+            continue
+
+        # pick the jpg out of the set
+        jpegs = []
+        for pfile in p.files:
+            if pfile.format == "image/jpeg":
+                jpegs.append(pfile)
+
+        if not jpegs:  # no files with gps data found
+            continue
+
+        gpsdata = None
+        for img in jpegs:
+            # scan it for gps data
+            # print(p.uuid, img.fname)
+            with closing(storage.open(img.path, 'rb')) as fsrc:
+                _, gpsdata, _, _ = get_exif_data_fobj(fsrc)  # (datetime, (decimal, decimal), (width, height), rotation)
+                # print(gpsdata)
+
+            if gpsdata and gpsdata[0]:
+                break
+
+        if not gpsdata:
+            continue
+
+        print(p.uuid, "->", gpsdata)
+        p.lat, p.lon = gpsdata[0], gpsdata[1]
+
+        # update the db
+        session.commit()
+
+
+# __name__ == '__main__' and part1()
+# __name__ == '__main__' and part2()
+# __name__ == '__main__' and migrate_gpsfix()