most of api upload

photoapp/api.py

@@ -1,6 +1,7 @@
 import os
 import cherrypy
 import logging
+import json
 from datetime import datetime, timedelta
 from photoapp.library import PhotoLibrary
 from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
@@ -29,7 +30,35 @@ class PhotosApiV1(object):
 
     @cherrypy.expose
     def upload(self, files, meta):
-        pass
+        """
+        upload accepts one photoset (multiple images)
+        metadata format
+        """
+        meta = json.loads(meta)
+
+        if type(files) != list:
+            files = [files]
+
+        for file in files:
+            print("File name:", file.filename)
+            import hashlib
+            sha = hashlib.sha256()
+
+            total = 0
+            while True:
+                b = file.file.read(1024)
+                if not b:
+                    break
+                sha.update(b)
+                total += len(b)
+            print("Read length:", total)
+            print("Read sha256:", sha.hexdigest())
+
+            if str(file.filename) not in meta["files"].keys():
+                raise cherrypy.HTTPError(400, f"no mdatadata provided for filename '{file.filename}'")
+            print("we have metadata for this file:", meta["files"][file.filename])
+
+        print("____")
 
     @cherrypy.expose
     @cherrypy.tools.json_out()

photoapp/cli.py

@@ -1,14 +1,19 @@
+import os
+import json
 import argparse
 import requests
 from requests.exceptions import HTTPError
 from photoapp.utils import get_extension
 from photoapp.types import known_extensions
 from photoapp.common import pwhash
+from photoapp.ingest import get_photosets
 
 
 class PhotoApiClient(object):
-    def __init__(self, base_url):
+    def __init__(self, base_url, passwd=None):
         self.session = requests.Session()
+        if passwd:
+            self.session.auth = passwd  # user, pass tuple
         self.base_url = base_url
 
     def byhash(self, sha):
@@ -23,8 +28,8 @@ class PhotoApiClient(object):
     def delete(self, url, **params):
         return self.do("delete", url, **params)
 
-    def do(self, method, url, **params):
+    def do(self, method, url, **kwargs):
-        resp = getattr(self.session, method)(self.base_url + "/api/v1/" + url, **params)
+        resp = getattr(self.session, method)(self.base_url + "/api/v1/" + url, **kwargs)
         resp.raise_for_status()
         return resp
 
@@ -38,10 +43,17 @@ class PhotoApiClient(object):
     def delete_user(self, username):
         return self.delete("user", params={"username": username})
 
+    def upload(self, files, metadata):
+        # print(">>>>>>", metadata)
+        return self.post("upload", files=files, data={"meta": json.dumps(metadata)})
+
 
 def get_args():
     parser = argparse.ArgumentParser(description="photo library cli")
-    parser.add_argument("-s", "--host", required=True, help="photo library server address")
+    # TODO nicer uri parser
+    parser.add_argument("--host", required=True, help="photo library server address")
+    parser.add_argument("--user", required=True)
+    parser.add_argument("--password", required=True)
 
     sp_action = parser.add_subparsers(dest="action", help="action to take")
 
@@ -50,8 +62,8 @@ def get_args():
     p_dupes.add_argument("files", nargs="+", help="files to check")
 
     p_ingest = sp_action.add_parser("ingest", help="import images into the library")
-    p_ingest.add_argument("files", nargs="+", help="files to import")
     p_ingest.add_argument("-c", "--copy-of", help="existing uuid the imported images will be placed under")
+    p_ingest.add_argument("files", nargs="+", help="files to import")
 
     # User section
     p_adduser = sp_action.add_parser("user", help="user manipulation functions")
@@ -73,7 +85,7 @@ def main():
     args = get_args()
     print(args)
 
-    client = PhotoApiClient(args.host)
+    client = PhotoApiClient(args.host, (args.user, args.password, ))
 
     if args.action == "checkdupes":
         hashes = {}
@@ -107,7 +119,25 @@ def main():
                     raise
 
     elif args.action == "ingest":
-        pass
+        if args.copy_of:
+            raise NotImplementedError("--copy-of isn't implemented")
+
+        sets, skipped = get_photosets(args.files)
+
+        #TODO y/n confirmation and auto flag
+        #TODO optional progress printing
+        print("skipping:", skipped)
+        print("sets:", [[f.path for f in s.files] for s in sets])
+
+        for set_ in sets:
+            payload = set_.to_json()
+            payload["files"] = {os.path.basename(photo.path): photo.to_json() for photo in set_.files}
+
+            files = []
+            for file in set_.files:
+                files.append(("files", (os.path.basename(file.path), open(file.path, 'rb'), file.format), ))
+
+            client.upload(files, payload)
 
     elif args.action == "user":
         if args.action_user == "create":

photoapp/image.py

@@ -19,8 +19,8 @@ def get_jpg_info(fpath):
         raise Exception("fuk")
 
     # gps is set to 0,0 if unavailable
-    lat, lon = gps or [0, 0]
+    lat, lon = gps or [None, None]
-    dimensions = dimensions or (0, 0)
+    dimensions = dimensions or (None, None)
     mime = magic.from_file(fpath, mime=True)
     size = os.path.getsize(fpath)
 

photoapp/ingest.py

@@ -4,7 +4,7 @@ import traceback
 from photoapp.library import PhotoLibrary
 from photoapp.image import get_jpg_info, get_hash, get_mtime
 from itertools import chain
-from photoapp.types import Photo, PhotoSet, known_extensions, regular_images, files_raw, files_video
+from photoapp.types import Photo, PhotoSet, known_extensions, regular_images, files_raw, files_video, map_extension
 import os
 
 
@@ -26,76 +26,74 @@ def pprogress(done, total=None):
     print("  complete: {}{}\r".format(done, " / {} ".format(total) if total else ''), end='')
 
 
-def batch_ingest(library, files):
+def group_by_extension(files):
-    # group by extension
     byext = {k: [] for k in known_extensions}
+    excluded = []
 
-    total = len(files)
-    print("processing {} items".format(total))
-    print("Pre-sorting files")
     for item in files:
-        if not os.path.isfile(item):
+        if not os.path.isfile(item):  # Not a file
-            print("Skipping due to not a file: {}".format(item))
             continue
         extension = item.split(".")
-        if len(extension) < 2:
+        if len(extension) < 2:  # no extension
-            print("Skipping due to no extension: {}".format(item))
+            excluded.append(item)
             continue
-        extension = extension[-1].lower()
+        extension = map_extension(extension[-1].lower())
-        if extension == "jpeg":
+        if extension not in known_extensions:  # an extension we don't support
-            extension = "jpg"
+            excluded.append(item)
-        if extension not in known_extensions:
-            print("Skipping due to unknown extension: {}".format(item))
             continue
         byext[extension.lower()].append(item)
 
-    print("Scanning images")
+    return (byext, excluded)
-    photos = []
+
+
+def get_photosets(files):
+    byext, skipped = group_by_extension(files)
+
+    photosets = []
+
     # process regular images first.
     for item in chain(*[byext[ext] for ext in regular_images]):
-        photos.append(get_jpg_info(item))
+        photosets.append(get_jpg_info(item))
-        pprogress(len(photos), total)
 
-    print("\nScanning RAWs")
     # process raws
-    done = len(photos)
     for item in chain(*[byext[ext] for ext in files_raw]):
         itemmeta = Photo(hash=get_hash(item), path=item, size=os.path.getsize(item),
                          format=special_magic(item))
         fprefix = os.path.basename(item)[::-1].split(".", 1)[-1][::-1]
-        fmatch = "{}.jpg".format(fprefix.lower())
+        fmatch = "{}.jpg".format(fprefix.lower())  # if we're inspecting "foobar.raw", match it with "foobar.jpg"
+                                                   # TODO does this account for extension mappinh like jpeg->jpg?
         foundmatch = False
-        for photo in photos:
+        for photo in photosets:
             for fmt in photo.files[:]:
                 if os.path.basename(fmt.path).lower() == fmatch:
                     foundmatch = True
                     photo.files.append(itemmeta)
-                    done += 1
-                    pprogress(done, total)
                     break
             if foundmatch:
                 break
         if not foundmatch:
             mtime = get_mtime(item)
-            photos.append(PhotoSet(date=mtime, date_real=mtime, lat=0, lon=0, files=[itemmeta]))
+            print("no match found for", itemmeta.path, "but importing anyway")
-            done += 1
+            photosets.append(PhotoSet(date=mtime, date_real=mtime, files=[itemmeta]))
-            pprogress(done, total)
+        # TODO handle any xmp without an associated regular image or cr2
-        # TODO prune any xmp without an associated regular image or cr2
 
-    print("\nScanning other files")
+    # process other known formats
-    # process all other formats
     for item in chain(*[byext[ext] for ext in files_video]):
         itemmeta = Photo(hash=get_hash(item), path=item, size=os.path.getsize(item),
                          format=special_magic(item))
         mtime = get_mtime(item)
-        photos.append(PhotoSet(date=mtime, date_real=mtime, lat=0, lon=0, files=[itemmeta]))
+        photosets.append(PhotoSet(date=mtime, date_real=mtime, files=[itemmeta]))
-        done += 1
+
-        pprogress(done, total)
+    return photosets, skipped
+
+
+def batch_ingest(library, files):
+    sets, skipped = get_photosets(files)
 
     print("\nUpdating database")
     done = 0
-    total = len(photos)
+    total = len(sets)
-    for photoset in photos:
+    for photoset in sets:
         try:
             library.add_photoset(photoset)
             pprogress(done, total)

photoapp/types.py

@@ -11,6 +11,14 @@ known_extensions = ["jpg", "png", "cr2", "xmp", "mp4", "mov"]
 regular_images = ["jpg", "png"]
 files_raw = ["cr2", "xmp"]
 files_video = ["mp4", "mov"]
+mapped_extensions = {"jpg": {"jpeg", }}   # target: aliases
+
+
+def map_extension(ext):
+    for target, aliases in mapped_extensions.items():
+        if ext in aliases:
+            return target
+    return ext
 
 
 class PhotoStatus(enum.Enum):
@@ -41,8 +49,8 @@ class PhotoSet(Base):
 
     def to_json(self):
         s = {attr: getattr(self, attr) for attr in {"uuid", "title", "description"}}
-        s["lat"] = str(self.lat)
+        s["lat"] = str(self.lat) if self.lat else None
-        s["lon"] = str(self.lon)
+        s["lon"] = str(self.lon) if self.lon else None
         s["date"] = self.date.isoformat()
         s["files"] = {i.uuid: i.to_json() for i in self.files}
         s["tags"] = [t.name for t in self.tags]