import os import json import argparse import requests from requests.exceptions import HTTPError from photoapp.utils import get_extension from photoapp.types import known_extensions, PhotoStatus from photoapp.common import pwhash from photoapp.ingest import get_photosets from urllib.parse import urlparse from tabulate import tabulate from concurrent.futures import ThreadPoolExecutor, as_completed import appdirs APPNAME = "photocli" DEFAULT_CONFIG = {"context": "default", "contexts": {"default": {"uri": None, "upload_workers": 2}}} class CliConfig(dict): def __init__(self, appname): self.confdir = appdirs.user_config_dir(appname) self.confpath = os.path.join(self.confdir, "cli.json") # ~/Library/Application Support/photocli/cli.json super().__init__(**DEFAULT_CONFIG) self.init_config() def init_config(self): os.makedirs(self.confdir, exist_ok=True) if os.path.exists(self.confpath): self.load() else: self.write() def load(self): with open(self.confpath) as f: self.update(**json.load(f)) def write(self): with open(self.confpath, "w") as f: json.dump(self, f, indent=4) def __setitem__(self, key, value): """ prevents setting of invalid config keys as they are not defined """ if key not in self: raise KeyError(key) super().__setitem__(key, value) self.write() @property def context(self): return self["context"] def set_server(self, context_name, server_uri): self["contexts"][context_name]["uri"] = server_uri self.write() def select_context(self, context_name): if context_name not in self["contexts"]: raise KeyError("Context '{}' is not defined".format(context_name)) return self["contexts"][context_name] def set_context(self, context_name): if context_name not in self["contexts"]: raise KeyError("Context '{}' is not defined".format(context_name)) self["context"] = context_name self.write() def edit_context(self, context_name, context): self["contexts"][context_name] = context self.write() def delete_context(self, context_name): del self["context_name"] self.write() class PhotoApiClient(object): def __init__(self, base_url, passwd=None): self.session = requests.Session() if passwd: self.session.auth = passwd # user, pass tuple self.base_url = base_url def byhash(self, sha): return self.get("byhash", params={"sha": sha}).json() def get(self, url, **params): return self.do("get", url, **params) def post(self, url, **params): return self.do("post", url, **params) def delete(self, url, **params): return self.do("delete", url, **params) def do(self, method, url, **kwargs): resp = getattr(self.session, method)(self.base_url + "/api/v1/" + url, **kwargs) resp.raise_for_status() return resp def create_user(self, username, password): return self.post("user", data={"username": username, "password_hash": pwhash(password)}) def list_users(self): return self.get("user") def delete_user(self, username): return self.delete("user", params={"username": username}) def upload(self, files, metadata): return self.post("upload", files=files, data={"meta": json.dumps(metadata)}) def stats(self): return self.get("stats").json() def list_photos(self, page=0, pagesize=25): return self.get("photos", params={"page": page, "pagesize": pagesize}).json() def create_tag(self, name, title, description): return self.post("tags", json={"name": name, "title": title, "description": description}) def list_tags(self, name=None): params = {} if name: params["name"] = name return self.get("tags", params=params) # def delete_user(self, username): # return self.delete("user", params={"username": username}) def maybetruncate(s, length): if s and len(s) > length: s = s[0:length - 3].strip() + "..." return s def confirm(message, options=None, accept=None, confirm_msg="Are you sure?"): if options is None: options = {"Y": True, "n": False} if accept is None: accept = [True] def show_prompt(message): result = input("{} [{}]: ".format(message, '/'.join(options.keys()))) try: result_value = options[result] if result_value not in accept: raise Exception("Aborted per request") return result_value except KeyError: raise Exception("Invalid choice") result = show_prompt(message) return result def get_args(): parser = argparse.ArgumentParser(description="photo library cli") # TODO nicer uri parser parser.add_argument("--host", default=os.environ.get("PHOTOLIB_URI", None), help="photo api server address") parser.add_argument("--context", help="api server context defined in config file", default=os.environ.get("PHOTOLIB_CONTEXT", None)) parser.add_argument("-y", "--yes", action="store_true", help="assume yes for all prompts") sp_action = parser.add_subparsers(dest="action", help="action to take") p_dupes = sp_action.add_parser("checkdupes", help="check if files/hash lists are already in the library") p_dupes.add_argument("--sha-files", action="store_true", help="read hashes from a file instead of hashing images") # p_dupes.add_argument("--print-uuids", action="store_true", help="lookup and print uuids of duplicate files instead") p_dupes.add_argument("files", nargs="+", help="files to check") p_ingest = sp_action.add_parser("ingest", help="import images into the library") p_ingest.add_argument("-c", "--copy-of", help="existing uuid the imported images will be placed under") p_ingest.add_argument("-w", "--workers", default=1, type=int, help="number of parallel uploads") p_ingest.add_argument("-t", "--tag", help="tag name to apply to the import") p_ingest.add_argument("files", nargs="+", help="files to import") # p_merge = sp_action.add_parser("merge", help="merge photoset copies into a master") # p_merge.add_argument("-m", "--master", required=True, help="master photoset uuid") # p_merge.add_argument("-c", "--copies", required=True, nargs="+", help="other photoset uuids") sp_action.add_parser("stats", help="show library statistics") p_list = sp_action.add_parser("list", help="list images in library") p_list.add_argument("-p", "--page", default=0, help="page number") p_list.add_argument("-z", "--page-size", default=25, help="page size") p_list.add_argument("-l", "--show-link", action="store_true", help="print urls to each photo") # Tag section p_tag = sp_action.add_parser("tag", help="tag manipulation functions") p_tagaction = p_tag.add_subparsers(dest="action_tag", help="action to take") p_create = p_tagaction.add_parser('create', help='create tag') p_create.add_argument("-n", "--name", required=True) p_create.add_argument("-t", "--title") p_create.add_argument("-d", "--description") p_tag_list = p_tagaction.add_parser('list', help='list tags') p_tag_list.add_argument("-n", "--name", help="tag name to lookup") p_delete = p_tagaction.add_parser('delete', help='delete tags') p_delete.add_argument("-n", "--name", required=True) # User section p_adduser = sp_action.add_parser("user", help="user manipulation functions") p_useraction = p_adduser.add_subparsers(dest="action_user", help="action to take") p_create = p_useraction.add_parser('create', help='create user') p_create.add_argument("-u", "--username", help="username", required=True) p_create.add_argument("-p", "--password", help="password", required=True) p_useraction.add_parser('list', help='list users') p_delete = p_useraction.add_parser('delete', help='delete users') p_delete.add_argument("-u", "--username", help="username", required=True) p_ctx = sp_action.add_parser("ctx", help="change server contexts") p_ctx.add_argument("new_context", help="context name") p_editctx = sp_action.add_parser("update-context", help="create/edit server contexts") p_editctx.add_argument("context_name", help="context name to modify") p_editctx.add_argument("--set-host", help="photo api server address") p_editctx.add_argument("--delete", action="store_true", help="delete the context") sp_action.add_parser("list-contexts", help="list available contexts") return parser.parse_args(), parser def main(): args, parser = get_args() config = CliConfig(APPNAME) # figure out the selected context - first the --context flag, then conf file, then "default" context = args.context or config.context # default context is "sticky" and remembers the last --host used if args.host and context == "default": # TODO move this after parsing new_default = urlparse(args.host) if not new_default.netloc: parser.error("--host uri is invalid") config.set_server("default", new_default.geturl()) if args.host and context != "default": parser.error("--host not allowed for non-default contexts. Use `ctx` to switch to the default context or use " "`update-context --host HOST` to create a new context.") # these commands manipulate the config file so we don't check it yet # change or edit contexts if args.action == "ctx": try: config.set_context(args.new_context) except KeyError as ke: parser.error("{}. Use `update-context` to define it.".format(ke.args[0])) return elif args.action == "update-context": try: ctx = config.select_context(args.context_name) except KeyError: ctx = DEFAULT_CONFIG["contexts"]["default"] if args.set_host: new_uri = urlparse(args.set_host) if not new_uri.netloc: parser.error("--set-host uri is invalid") ctx["uri"] = new_uri.geturl() if args.delete: try: config.delete_context(args.context_name) print("Context deleted") except KeyError: pass return if not ctx["uri"]: parser.error("New context must include a host, use `--set-host`.") config.edit_context(args.context_name, ctx) print("Context updated") return elif args.action == "list-contexts": for entry in sorted(config["contexts"].keys()): print("{}{}".format(entry, " (active)" if entry == context else "")) return # commands below here need a valid config file and api client # validate the config try: server_conf = config.select_context(context) except KeyError as ke: parser.error("{}. Use `update-context` to define it.".format(ke.args[0])) uri = urlparse(server_conf["uri"]) if not uri.netloc: parser.error("need --host, $PHOTOLIB_URI, or config file") # create the client port = f":{uri.port}" if uri.port else "" client = PhotoApiClient(f"{uri.scheme}://{uri.hostname}{port}", (uri.username, uri.password, )) if args.action == "checkdupes": hashes = {} if args.sha_files: # sha file format should look like the output of `find ./source -type f -exec shasum -a 256 {} \;`: # e931d286a8377ea8f49ee928e793d9e1fd18a25402cac43afdee3eec3b3b18a5 source/2018-12-26 20.54.40.jpg # 24c1cd20c961839f14d608f5719c4f380af902c4774ddb7cb9ff747f652ca302 source/2018-12-30 22.39.27.jpg for fname in args.files: with open(fname) as f: for line in f.readlines(): if not line: continue sha, path = line.split(maxsplit=1) path = path.rstrip("\n") if get_extension(path) not in known_extensions: continue hashes[sha] = path else: raise NotImplementedError("must pass --sha-files for now") for sha, path in hashes.items(): # http://localhost:8080/api/v1/byhash?sha=afe49172f709725a4503c9219fb4c6a9db8ad0354fc493f2f500269ac6faeaf6 try: client.byhash(sha) # if the file is a dupe, do nothing except HTTPError as he: # if the file is original, print its path if he.response.status_code == 404: print(path) else: raise elif args.action == "ingest": if args.copy_of and args.tag: raise NotImplementedError("--copy-of not allowed with --tag") # TODO refactor this to be less ugly tag = None if args.tag: tag = args.tag.lower() try: client.list_tags(name=tag).json() except requests.exceptions.HTTPError as he: if he.response.status_code != 404: raise client.create_tag(tag, tag, "") sets, skipped = get_photosets(args.files) rows = [] for set_ in sets: rows.append([" ".join([f.path for f in set_.files]), "upload"]) for fname in skipped: rows.append([fname, "skip"]) print(tabulate(rows, headers=["files", "action"])) # TODO also dupe check here if not sets: return print() if not args.yes: if not confirm("Continue?"): return print() results = [] numerrors = 0 def printprogress(): print(f"\rProgress: total: {len(sets)} completed: {len(results)} errors: {numerrors} ", end="") def upload_set(set_): payload = set_.to_json() # add tags to set if args.tag: # it's kinda cheap to do this in json instead of building out the PhotoSet data structure # but trust me it's worth it payload["tags"] = [tag] payload["files"] = {os.path.basename(photo.path): photo.to_json() for photo in set_.files} files = [] for file in set_.files: files.append(("files", (os.path.basename(file.path), open(file.path, 'rb'), file.format), )) if args.copy_of: payload["uuid"] = args.copy_of try: result = client.upload(files, payload).json() return ["success", result["uuid"]] except HTTPError as he: return ["error", he.response.json()["error"]] printprogress() with ThreadPoolExecutor(max_workers=args.workers) as executor: futures = {executor.submit(upload_set, set_): set_ for set_ in sets} for future in as_completed(futures.keys()): set_ = futures[future] set_fnames = [os.path.basename(file.path) for file in set_.files] e = future.exception() if e: results.append([set_fnames, "exception", repr(e)]) numerrors += 1 else: result = future.result() if result[0] != "success": numerrors += 1 results.append([set_fnames] + result) printprogress() print() print() print(tabulate([[" ".join(row[0]), row[1], row[2]] for row in results], headers=["files", "status", "uuid"])) print("\nErrors:", numerrors) # TODO be nice and close the files elif args.action == "list": headers = ["link" if args.show_link else "uuid", "status", "size", "formats", "date", "title"]# TODO tags rows = [] for set_ in client.list_photos(args.page, args.page_size): row = [set_["uuid"] if not args.show_link else args.host.rstrip("/") + "/photo/" + set_["uuid"], PhotoStatus[set_["status"]].name, sum([p["size"] for p in set_["files"].values()]), " ".join(set([p["format"] for p in set_["files"].values()])), set_["date"], maybetruncate(set_["title"], 24), ] rows.append(row) print(tabulate(rows, headers=headers)) elif args.action == "stats": print(tabulate(sorted([[k, v] for k, v in client.stats().items()], key=lambda row: row[0]), headers=["item", "count"])) elif args.action == "user": if args.action_user == "create": print(client.create_user(args.username, args.password).json()) elif args.action_user == "list": for u in client.list_users().json(): print(f"{u['name']}: {u['status']}") elif args.action_user == "delete": print(client.delete_user(args.username).json()) elif args.action == "tag": if args.action_tag == "create": client.create_tag(args.name, args.title, args.description) elif args.action_tag == "list": rows = [] for tag in client.list_tags(name=args.name).json(): rows.append([tag["name"], tag["title"], maybetruncate(tag["description"], 24), tag["uuid"]]) print(tabulate(rows, headers=["name", "title", "description", "uuid"])) if __name__ == "__main__": main()