missing gps data fix
This commit is contained in:
parent
8877fb263a
commit
50a55f08df
|
@ -132,3 +132,7 @@ Roadmap
|
||||||
- "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest.
|
- "fast ingest" method that touches the db/storage directly. This would scale better than the API ingest.
|
||||||
- Dynamic svg placeholder for images we can't open
|
- Dynamic svg placeholder for images we can't open
|
||||||
- Proactive thumbnail generation
|
- Proactive thumbnail generation
|
||||||
|
- on photo thumbs on the feed, a little "2" indicating there are 2 images in the set (or w/e number for that item)
|
||||||
|
- dark theme
|
||||||
|
- more information from the images like on http://exif.regex.info/exif.cgi
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ from photoapp.image import special_magic_fobj
|
||||||
from photoapp.storage import StorageAdapter
|
from photoapp.storage import StorageAdapter
|
||||||
from photoapp.dbutils import db
|
from photoapp.dbutils import db
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
|
from decimal import Decimal
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
|
|
||||||
|
@ -117,13 +118,17 @@ class PhotosApiV1(object):
|
||||||
ps = db.query(PhotoSet).filter(PhotoSet.uuid == meta["uuid"]).first()
|
ps = db.query(PhotoSet).filter(PhotoSet.uuid == meta["uuid"]).first()
|
||||||
if not ps:
|
if not ps:
|
||||||
return abort_upload("parent uuid not found")
|
return abort_upload("parent uuid not found")
|
||||||
ps.date = photo_date
|
|
||||||
ps.date_real = photo_date
|
|
||||||
ps.files.extend(photo_objs)
|
ps.files.extend(photo_objs)
|
||||||
|
if not ps.lat and meta["lat"] and meta["lon"]:
|
||||||
|
ps.lat = Decimal(meta["lat"])
|
||||||
|
ps.lon = Decimal(meta["lon"])
|
||||||
|
|
||||||
else:
|
else:
|
||||||
ps = PhotoSet(uuid=genuuid(),
|
ps = PhotoSet(uuid=genuuid(),
|
||||||
date=photo_date,
|
date=photo_date,
|
||||||
date_real=photo_date, # TODO support time offsets
|
date_real=photo_date, # TODO support time offsets
|
||||||
|
lat=Decimal(meta["lat"]) if meta["lat"] else None,
|
||||||
|
lon=Decimal(meta["lon"]) if meta["lon"] else None,
|
||||||
files=photo_objs) # TODO support title field et
|
files=photo_objs) # TODO support title field et
|
||||||
db.add(ps)
|
db.add(ps)
|
||||||
|
|
||||||
|
|
|
@ -289,7 +289,7 @@ class DownloadView(object):
|
||||||
raise cherrypy.HTTPError(404)
|
raise cherrypy.HTTPError(404)
|
||||||
extra = {}
|
extra = {}
|
||||||
if not preview:
|
if not preview:
|
||||||
extra.update(disposition="attachement", name=os.path.basename(item.path))
|
extra.update(disposition="attachement", name=item.fname)
|
||||||
return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'),
|
return cherrypy.lib.static.serve_fileobj(self.master.library.storage.open(item.path, 'rb'),
|
||||||
content_type=item.format, **extra)
|
content_type=item.format, **extra)
|
||||||
|
|
||||||
|
|
|
@ -46,10 +46,18 @@ def get_hash(path):
|
||||||
|
|
||||||
|
|
||||||
def get_exif_data(path):
|
def get_exif_data(path):
|
||||||
|
with open(path, 'rb') as f:
|
||||||
|
dateinfo, gpsinfo, sizeinfo, orientationinfo = get_exif_data_fobj(f)
|
||||||
|
if dateinfo is None:
|
||||||
|
dateinfo = get_mtime(path)
|
||||||
|
return dateinfo, gpsinfo, sizeinfo, orientationinfo
|
||||||
|
|
||||||
|
|
||||||
|
def get_exif_data_fobj(fobj):
|
||||||
"""
|
"""
|
||||||
Return a (datetime, (decimal, decimal), (width, height), rotation) tuple describing the photo's exif date and gps coordinates
|
Return a (datetime, (decimal, decimal), (width, height), rotation) tuple describing the photo's exif date and gps coordinates
|
||||||
"""
|
"""
|
||||||
img = Image.open(path)
|
img = Image.open(fobj) # TODO do i need to close this?
|
||||||
|
|
||||||
datestr = None
|
datestr = None
|
||||||
gpsinfo = None
|
gpsinfo = None
|
||||||
|
@ -94,9 +102,6 @@ def get_exif_data(path):
|
||||||
gps_x *= -1
|
gps_x *= -1
|
||||||
gpsinfo = (gps_y, gps_x)
|
gpsinfo = (gps_y, gps_x)
|
||||||
|
|
||||||
if dateinfo is None:
|
|
||||||
dateinfo = get_mtime(path)
|
|
||||||
|
|
||||||
return dateinfo, gpsinfo, sizeinfo, orientationinfo
|
return dateinfo, gpsinfo, sizeinfo, orientationinfo
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,210 @@
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sqlalchemy
|
||||||
|
from photoapp.dbutils import get_db_engine
|
||||||
|
from photoapp.types import Photo, PhotoSet, generate_storage_path
|
||||||
|
from photoapp.storage import uri_to_storage
|
||||||
|
from photoapp.image import get_exif_data_fobj
|
||||||
|
import shutil
|
||||||
|
from contextlib import closing
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
|
||||||
|
|
||||||
|
STORAGE_URI = "file://./library"
|
||||||
|
DB_URI = "sqlite:///photos.db"
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
Before this application had support for multiple file storage backends, it only supported the filesystem and it was done
|
||||||
|
in a way that wasn't abstracted. This script contains steps in migrating and old database and file tree to the
|
||||||
|
modern one.
|
||||||
|
|
||||||
|
SQLite to MySQL Steps:
|
||||||
|
======================
|
||||||
|
|
||||||
|
Migrating from mysql to sqlite is NOT required. Follow this part if you're converting from sqlite to mysql.
|
||||||
|
|
||||||
|
If you're not migrating from sqlite to mysql, you just need to do the equivalent of step 4 to your sqlite database.
|
||||||
|
|
||||||
|
1) Export the old sqlite database's contents. We need the data only, not the schema. First dump everything:
|
||||||
|
|
||||||
|
```
|
||||||
|
$ sqlite3 photos_old.db
|
||||||
|
sqlite> .output old.sql
|
||||||
|
sqlite> .dump
|
||||||
|
sqlite> .exit
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
Then using your favorite editor format all the INSERT statements like:
|
||||||
|
|
||||||
|
```
|
||||||
|
START TRANSACTION;
|
||||||
|
SET FOREIGN_KEY_CHECKS=0;
|
||||||
|
|
||||||
|
(many insert statements here)
|
||||||
|
|
||||||
|
SET FOREIGN_KEY_CHECKS=1;
|
||||||
|
COMMIT;
|
||||||
|
```
|
||||||
|
|
||||||
|
2) Populate a mysql database with the app's schema
|
||||||
|
|
||||||
|
You can just start and stop the app with it pointed at mysql like a normal person would.
|
||||||
|
|
||||||
|
3) Modify the mysql schema to play nice with our data:
|
||||||
|
|
||||||
|
```
|
||||||
|
mysql> alter table files drop fname;
|
||||||
|
alter table files modify column path varchar(1024);
|
||||||
|
```
|
||||||
|
|
||||||
|
3) Import the sqlite data dump into mysql
|
||||||
|
|
||||||
|
4) Put the mysql schema back
|
||||||
|
|
||||||
|
```
|
||||||
|
mysql> alter table files add fname varchar(256);
|
||||||
|
````
|
||||||
|
|
||||||
|
Now, continue with the steps below. When finished, it's safest to do a data-only dump from mysql, delete and recreate
|
||||||
|
your schema, and import the mysql data dump.
|
||||||
|
|
||||||
|
|
||||||
|
Filesystem migration
|
||||||
|
====================
|
||||||
|
|
||||||
|
This part is required, the layout of your library directory must be migrated. If you're moving to s3, that is done
|
||||||
|
here too.
|
||||||
|
|
||||||
|
1) Run this script pointed at your new database (edit the connection uri and main function to run part1 below)
|
||||||
|
|
||||||
|
It will produce a renames.json in the current dir, which is needed in later steps.
|
||||||
|
|
||||||
|
2) Run this script pointed at your new storage (edit the storage uri and main function to run part2 below)
|
||||||
|
|
||||||
|
This copies your photos to the new library.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def getstorage():
|
||||||
|
return uri_to_storage(STORAGE_URI)
|
||||||
|
|
||||||
|
|
||||||
|
def getsession():
|
||||||
|
engine = get_db_engine(DB_URI)
|
||||||
|
sessionmaker = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
|
||||||
|
sessionmaker.configure(bind=engine)
|
||||||
|
return sessionmaker()
|
||||||
|
|
||||||
|
|
||||||
|
def part1():
|
||||||
|
session = getsession()
|
||||||
|
renames = []
|
||||||
|
|
||||||
|
if os.path.exists("renames.json"):
|
||||||
|
raise Exception("dont run me twice!!")
|
||||||
|
|
||||||
|
for p in session.query(Photo).all():
|
||||||
|
fname = os.path.basename(p.path)
|
||||||
|
p.fname = fname
|
||||||
|
ext = fname.split(".")[-1]
|
||||||
|
newpath = generate_storage_path(p.set.date_real, p.hash, ext)
|
||||||
|
assert p.path != newpath
|
||||||
|
renames.append((p.path, newpath))
|
||||||
|
p.path = newpath
|
||||||
|
|
||||||
|
with open("renames.json", "w") as f:
|
||||||
|
json.dump(renames, f)
|
||||||
|
|
||||||
|
session.commit()
|
||||||
|
# session.rollback()
|
||||||
|
|
||||||
|
|
||||||
|
def part2():
|
||||||
|
with open("path/to/renames.json", "r") as f:
|
||||||
|
renames = json.load(f)
|
||||||
|
|
||||||
|
library_storage = getstorage()
|
||||||
|
|
||||||
|
numdone = 0
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||||
|
futures = {pool.submit(rename_set, library_storage, set_[0], set_[1]): set_ for set_ in renames}
|
||||||
|
print("Working...")
|
||||||
|
for future in as_completed(futures.keys()):
|
||||||
|
set_ = futures[future]
|
||||||
|
e = future.exception()
|
||||||
|
if e:
|
||||||
|
print("Screwed up on:", set_)
|
||||||
|
raise e
|
||||||
|
numdone += 1
|
||||||
|
print("Done:", numdone)
|
||||||
|
|
||||||
|
print("Done!")
|
||||||
|
|
||||||
|
|
||||||
|
def rename_set(storage, src_path, dest_path):
|
||||||
|
with closing(storage.open(dest_path, 'wb')) as df:
|
||||||
|
with open(src_path, 'rb') as sf:
|
||||||
|
shutil.copyfileobj(sf, df)
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
At one point, the cli contained errors causing it to not submit photo gps data. Running migrate_gpsfix rescans files for
|
||||||
|
said gps data and updates the database.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_gpsfix():
|
||||||
|
session = getsession()
|
||||||
|
storage = getstorage()
|
||||||
|
|
||||||
|
done = 0
|
||||||
|
# iterate all images
|
||||||
|
for p in session.query(PhotoSet).filter(sqlalchemy.or_(PhotoSet.lat == 0,
|
||||||
|
PhotoSet.lat == None # NOQA: E711
|
||||||
|
)).all():
|
||||||
|
done += 1
|
||||||
|
print(done)
|
||||||
|
|
||||||
|
if p.lat and p.lon:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# just bail if there's a CR2, the paired jpg is known not to have gps data in my dataset :)
|
||||||
|
if any(["image/x-canon-cr2" == i.format for i in p.files]):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# pick the jpg out of the set
|
||||||
|
jpegs = []
|
||||||
|
for pfile in p.files:
|
||||||
|
if pfile.format == "image/jpeg":
|
||||||
|
jpegs.append(pfile)
|
||||||
|
|
||||||
|
if not jpegs: # no files with gps data found
|
||||||
|
continue
|
||||||
|
|
||||||
|
gpsdata = None
|
||||||
|
for img in jpegs:
|
||||||
|
# scan it for gps data
|
||||||
|
# print(p.uuid, img.fname)
|
||||||
|
with closing(storage.open(img.path, 'rb')) as fsrc:
|
||||||
|
_, gpsdata, _, _ = get_exif_data_fobj(fsrc) # (datetime, (decimal, decimal), (width, height), rotation)
|
||||||
|
# print(gpsdata)
|
||||||
|
|
||||||
|
if gpsdata and gpsdata[0]:
|
||||||
|
break
|
||||||
|
|
||||||
|
if not gpsdata:
|
||||||
|
continue
|
||||||
|
|
||||||
|
print(p.uuid, "->", gpsdata)
|
||||||
|
p.lat, p.lon = gpsdata[0], gpsdata[1]
|
||||||
|
|
||||||
|
# update the db
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
# __name__ == '__main__' and part1()
|
||||||
|
# __name__ == '__main__' and part2()
|
||||||
|
# __name__ == '__main__' and migrate_gpsfix()
|
Loading…
Reference in New Issue