better uploading ux

photoapp/api.py

@@ -199,8 +199,9 @@ class PhotosApiV1(object):
 
         try:
             db.commit()
-        except IntegrityError as ie:
-            return abort_upload(str(ie))
+        except Exception as e:
+            traceback.print_exc()
+            return abort_upload(str(e))
 
         return ps_json
 

photoapp/cli.py

@@ -3,10 +3,12 @@ import json
 import argparse
 import requests
 from requests.exceptions import HTTPError
-from photoapp.utils import get_extension, shasum
-from photoapp.types import known_extensions
+from photoapp.utils import get_extension
+from photoapp.types import known_extensions, PhotoStatus
 from photoapp.common import pwhash
 from photoapp.ingest import get_photosets
+from tabulate import tabulate
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 
 class PhotoApiClient(object):
@@ -59,12 +61,35 @@ def maybetruncate(s, length):
     return s
 
 
+def confirm(message, options=None, accept=None, confirm_msg="Are you sure?"):
+    if options is None:
+        options = {"Y": True, "n": False}
+
+    if accept is None:
+        accept = [True]
+
+    def show_prompt(message):
+        result = input("{} [{}]: ".format(message, '/'.join(options.keys())))
+        try:
+            result_value = options[result]
+            if result_value not in accept:
+                raise Exception("Aborted per request")
+            return result_value
+        except KeyError:
+            raise Exception("Invalid choice")
+
+    result = show_prompt(message)
+
+    return result
+
+
 def get_args():
     parser = argparse.ArgumentParser(description="photo library cli")
     # TODO nicer uri parser
     parser.add_argument("--host", required=True, help="photo library server address")
     parser.add_argument("--user", required=True)
     parser.add_argument("--password", required=True)
+    parser.add_argument("-y", "--yes", action="store_true", help="assume yes for all prompts")
 
     sp_action = parser.add_subparsers(dest="action", help="action to take")
 
@@ -74,6 +99,7 @@ def get_args():
 
     p_ingest = sp_action.add_parser("ingest", help="import images into the library")
     p_ingest.add_argument("-c", "--copy-of", help="existing uuid the imported images will be placed under")
+    p_ingest.add_argument("-w", "--workers", default=1, type=int, help="number of parallel uploads")
     p_ingest.add_argument("files", nargs="+", help="files to import")
 
     sp_action.add_parser("stats", help="show library statistics")
@@ -137,12 +163,31 @@ def main():
 
     elif args.action == "ingest":
         sets, skipped = get_photosets(args.files)
-        #TODO y/n confirmation and auto flag
-        #TODO optional progress printing
-        print("skipping:", skipped)
-        print("sets:", [[f.path for f in s.files] for s in sets])
 
-        for num, set_ in enumerate(sets):
+        rows = []
+        for set_ in sets:
+            rows.append([" ".join([f.path for f in set_.files]), "upload"])
+        for fname in skipped:
+            rows.append([fname, "skip"])
+
+        print(tabulate(rows, headers=["files", "action"]))  # TODO also dupe check here
+
+        if not sets:
+            return
+
+        print()
+        if not args.yes:
+            if not confirm("Continue?"):
+                return
+            print()
+
+        results = []
+        numerrors = 0
+
+        def printprogress():
+            print(f"\rProgress: total: {len(sets)}  completed: {len(results)}   errors: {numerrors} ", end="")
+
+        def upload_set(set_):
             payload = set_.to_json()
             payload["files"] = {os.path.basename(photo.path): photo.to_json() for photo in set_.files}
 
@@ -153,15 +198,35 @@ def main():
             if args.copy_of:
                 payload["uuid"] = args.copy_of
 
-            print("Uploading: ", [os.path.basename(file.path) for file in set_.files])
             try:
-                result = client.upload(files, payload)
-                print("Uploaded: ", result.json()["uuid"])
+                result = client.upload(files, payload).json()
+                return ["success", result["uuid"]]
             except HTTPError as he:
-                print(he.response.json())
-                # TODO collect errors and print later
-                # return
-            print(f"{num} / {len(sets)}")
+                return ["error", he.response.json()["error"]]
+
+        printprogress()
+
+        with ThreadPoolExecutor(max_workers=args.workers) as executor:
+            futures = {executor.submit(upload_set, set_): set_ for set_ in sets}
+            for future in as_completed(futures.keys()):
+                set_ = futures[future]
+                set_fnames = [os.path.basename(file.path) for file in set_.files]
+                e = future.exception()
+                if e:
+                    results.append([set_fnames, "exception", repr(e)])
+                else:
+                    result = future.result()
+                    if result[0] != "success":
+                        numerrors += 1
+                    results.append([set_fnames] + result)
+                printprogress()
+
+        print()
+        print()
+        print(tabulate([[" ".join(row[0]), row[1], row[2]] for row in results],
+                       headers=["files", "status", "uuid"]))
+
+        print("\nErrors:", numerrors)
         # TODO be nice and close the files
 
     elif args.action == "list":

photoapp/ingest.py

@@ -1,4 +1,3 @@
-import magic
 import argparse
 import traceback
 from photoapp.library import PhotoLibrary
@@ -32,6 +31,7 @@ def group_by_extension(files):
 
     for item in files:
         if not os.path.isfile(item):  # Not a file
+            excluded.append(item)
             continue
         extension = item.split(".")
         if len(extension) < 2:  # no extension
@@ -98,7 +98,7 @@ def batch_ingest(library, files):
             library.add_photoset(photoset)
             pprogress(done, total)
             done += 1
-        except:
+        except Exception:
             traceback.print_exc()
             pass
     print("\nUpdate complete")

requirements.txt

@@ -16,6 +16,7 @@ pytz==2018.5
 requests==2.22.0
 six==1.11.0
 SQLAlchemy==1.3.5
+tabulate==0.8.3
 tempora==1.13
 urllib3==1.25.3
 zc.lockfile==1.3.0