photolib/photoapp/cli.py

import os
import json
import argparse
import requests
from requests.exceptions import HTTPError
from photoapp.utils import get_extension
from photoapp.types import known_extensions, PhotoStatus
from photoapp.common import pwhash
from photoapp.ingest import get_photosets
from urllib.parse import urlparse
from tabulate import tabulate
from concurrent.futures import ThreadPoolExecutor, as_completed
import appdirs


APPNAME = "photocli"
DEFAULT_CONFIG = {"context": "default",
                  "contexts": {"default": {"uri": None,
                                           "upload_workers": 2}}}


class CliConfig(dict):
    def __init__(self, appname):
        self.confdir = appdirs.user_config_dir(appname)
        self.confpath = os.path.join(self.confdir, "cli.json")  # ~/Library/Application Support/photocli/cli.json
        super().__init__(**DEFAULT_CONFIG)
        self.init_config()

    def init_config(self):
        os.makedirs(self.confdir, exist_ok=True)
        if os.path.exists(self.confpath):
            self.load()
        else:
            self.write()

    def load(self):
        with open(self.confpath) as f:
            self.update(**json.load(f))

    def write(self):
        with open(self.confpath, "w") as f:
            json.dump(self, f, indent=4)

    def __setitem__(self, key, value):
        """
        prevents setting of invalid config keys as they are not defined
        """
        if key not in self:
            raise KeyError(key)
        super().__setitem__(key, value)
        self.write()

    @property
    def context(self):
        return self["context"]

    def set_server(self, context_name, server_uri):
        self["contexts"][context_name]["uri"] = server_uri
        self.write()

    def select_context(self, context_name):
        if context_name not in self["contexts"]:
            raise KeyError("Context '{}' is not defined".format(context_name))
        return self["contexts"][context_name]

    def set_context(self, context_name):
        if context_name not in self["contexts"]:
            raise KeyError("Context '{}' is not defined".format(context_name))
        self["context"] = context_name
        self.write()

    def edit_context(self, context_name, context):
        self["contexts"][context_name] = context
        self.write()

    def delete_context(self, context_name):
        del self["context_name"]
        self.write()


class PhotoApiClient(object):
    def __init__(self, base_url, passwd=None):
        self.session = requests.Session()
        if passwd:
            self.session.auth = passwd  # user, pass tuple
        self.base_url = base_url

    def byhash(self, sha):
        return self.get("byhash", params={"sha": sha}).json()

    def get(self, url, **params):
        return self.do("get", url, **params)

    def post(self, url, **params):
        return self.do("post", url, **params)

    def delete(self, url, **params):
        return self.do("delete", url, **params)

    def do(self, method, url, **kwargs):
        resp = getattr(self.session, method)(self.base_url + "/api/v1/" + url, **kwargs)
        resp.raise_for_status()
        return resp

    def create_user(self, username, password):
        return self.post("user", data={"username": username,
                                       "password_hash": pwhash(password)})

    def list_users(self):
        return self.get("user")

    def delete_user(self, username):
        return self.delete("user", params={"username": username})

    def upload(self, files, metadata):
        return self.post("upload", files=files, data={"meta": json.dumps(metadata)})

    def stats(self):
        return self.get("stats").json()

    def list_photos(self, page=0, pagesize=25):
        return self.get("photos", params={"page": page, "pagesize": pagesize}).json()

    def create_tag(self, name, title, description):
        return self.post("tags", json={"name": name, "title": title, "description": description})

    def list_tags(self, name=None):
        params = {}
        if name:
            params["name"] = name
        return self.get("tags", params=params)

    # def delete_user(self, username):
    #     return self.delete("user", params={"username": username})


def maybetruncate(s, length):
    if s and len(s) > length:
        s = s[0:length - 3].strip() + "..."
    return s


def confirm(message, options=None, accept=None, confirm_msg="Are you sure?"):
    if options is None:
        options = {"Y": True, "n": False}

    if accept is None:
        accept = [True]

    def show_prompt(message):
        result = input("{} [{}]: ".format(message, '/'.join(options.keys())))
        try:
            result_value = options[result]
            if result_value not in accept:
                raise Exception("Aborted per request")
            return result_value
        except KeyError:
            raise Exception("Invalid choice")

    result = show_prompt(message)

    return result


def get_args():
    parser = argparse.ArgumentParser(description="photo library cli")
    # TODO nicer uri parser
    parser.add_argument("--host", default=os.environ.get("PHOTOLIB_URI", None), help="photo api server address")
    parser.add_argument("--context", help="api server context defined in config file",
                        default=os.environ.get("PHOTOLIB_CONTEXT", None))
    parser.add_argument("-y", "--yes", action="store_true", help="assume yes for all prompts")

    sp_action = parser.add_subparsers(dest="action", help="action to take")

    p_dupes = sp_action.add_parser("checkdupes", help="check if files/hash lists are already in the library")
    p_dupes.add_argument("--sha-files", action="store_true", help="read hashes from a file instead of hashing images")
    # p_dupes.add_argument("--print-uuids", action="store_true", help="lookup and print uuids of duplicate files instead")
    p_dupes.add_argument("files", nargs="+", help="files to check")

    p_ingest = sp_action.add_parser("ingest", help="import images into the library")
    p_ingest.add_argument("-c", "--copy-of", help="existing uuid the imported images will be placed under")
    p_ingest.add_argument("-w", "--workers", default=1, type=int, help="number of parallel uploads")
    p_ingest.add_argument("-t", "--tag", help="tag name to apply to the import")
    p_ingest.add_argument("files", nargs="+", help="files to import")

    # p_merge = sp_action.add_parser("merge", help="merge photoset copies into a master")
    # p_merge.add_argument("-m", "--master", required=True, help="master photoset uuid")
    # p_merge.add_argument("-c", "--copies", required=True, nargs="+", help="other photoset uuids")

    sp_action.add_parser("stats", help="show library statistics")

    p_list = sp_action.add_parser("list", help="list images in library")
    p_list.add_argument("-p", "--page", default=0, help="page number")
    p_list.add_argument("-z", "--page-size", default=25, help="page size")
    p_list.add_argument("-l", "--show-link", action="store_true", help="print urls to each photo")

    # Tag section
    p_tag = sp_action.add_parser("tag", help="tag manipulation functions")
    p_tagaction = p_tag.add_subparsers(dest="action_tag", help="action to take")

    p_create = p_tagaction.add_parser('create', help='create tag')
    p_create.add_argument("-n", "--name", required=True)
    p_create.add_argument("-t", "--title")
    p_create.add_argument("-d", "--description")

    p_tag_list = p_tagaction.add_parser('list', help='list tags')
    p_tag_list.add_argument("-n", "--name", help="tag name to lookup")

    p_delete = p_tagaction.add_parser('delete', help='delete tags')
    p_delete.add_argument("-n", "--name", required=True)

    # User section
    p_adduser = sp_action.add_parser("user", help="user manipulation functions")
    p_useraction = p_adduser.add_subparsers(dest="action_user", help="action to take")

    p_create = p_useraction.add_parser('create', help='create user')
    p_create.add_argument("-u", "--username", help="username", required=True)
    p_create.add_argument("-p", "--password", help="password", required=True)

    p_useraction.add_parser('list', help='list users')

    p_delete = p_useraction.add_parser('delete', help='delete users')
    p_delete.add_argument("-u", "--username", help="username", required=True)

    p_ctx = sp_action.add_parser("ctx", help="change server contexts")
    p_ctx.add_argument("new_context", help="context name")

    p_editctx = sp_action.add_parser("update-context", help="create/edit server contexts")
    p_editctx.add_argument("context_name", help="context name to modify")
    p_editctx.add_argument("--set-host", help="photo api server address")
    p_editctx.add_argument("--delete", action="store_true", help="delete the context")

    sp_action.add_parser("list-contexts", help="list available contexts")

    return parser.parse_args(), parser


def main():
    args, parser = get_args()
    config = CliConfig(APPNAME)

    # figure out the selected context - first the --context flag, then conf file, then "default"
    context = args.context or config.context

    # default context is "sticky" and remembers the last --host used
    if args.host and context == "default":  # TODO move this after parsing
        new_default = urlparse(args.host)
        if not new_default.netloc:
            parser.error("--host uri is invalid")
        config.set_server("default", new_default.geturl())

    if args.host and context != "default":
        parser.error("--host not allowed for non-default contexts. Use `ctx` to switch to the default context or use "
                     "`update-context --host HOST` to create a new context.")

    # these commands manipulate the config file so we don't check it yet
    # change or edit contexts
    if args.action == "ctx":
        try:
            config.set_context(args.new_context)
        except KeyError as ke:
            parser.error("{}. Use `update-context` to define it.".format(ke.args[0]))
        return

    elif args.action == "update-context":
        try:
            ctx = config.select_context(args.context_name)
        except KeyError:
            ctx = DEFAULT_CONFIG["contexts"]["default"]

        if args.set_host:
            new_uri = urlparse(args.set_host)
            if not new_uri.netloc:
                parser.error("--set-host uri is invalid")
            ctx["uri"] = new_uri.geturl()

        if args.delete:
            try:
                config.delete_context(args.context_name)
                print("Context deleted")
            except KeyError:
                pass
            return

        if not ctx["uri"]:
            parser.error("New context must include a host, use `--set-host`.")

        config.edit_context(args.context_name, ctx)
        print("Context updated")
        return

    elif args.action == "list-contexts":
        for entry in sorted(config["contexts"].keys()):
            print("{}{}".format(entry, "    (active)" if entry == context else ""))
        return

    # commands below here need a valid config file and api client
    # validate the config
    try:
        server_conf = config.select_context(context)
    except KeyError as ke:
        parser.error("{}. Use `update-context` to define it.".format(ke.args[0]))

    uri = urlparse(server_conf["uri"])

    if not uri.netloc:
        parser.error("need --host, $PHOTOLIB_URI, or config file")

    # create the client
    port = f":{uri.port}" if uri.port else ""
    client = PhotoApiClient(f"{uri.scheme}://{uri.hostname}{port}", (uri.username, uri.password, ))

    if args.action == "checkdupes":
        hashes = {}
        if args.sha_files:
            # sha file format should look like the output of `find ./source -type f -exec shasum -a 256 {} \;`:
            # e931d286a8377ea8f49ee928e793d9e1fd18a25402cac43afdee3eec3b3b18a5  source/2018-12-26 20.54.40.jpg
            # 24c1cd20c961839f14d608f5719c4f380af902c4774ddb7cb9ff747f652ca302  source/2018-12-30 22.39.27.jpg
            for fname in args.files:
                with open(fname) as f:
                    for line in f.readlines():
                        if not line:
                            continue
                        sha, path = line.split(maxsplit=1)
                        path = path.rstrip("\n")
                        if get_extension(path) not in known_extensions:
                            continue
                        hashes[sha] = path
        else:
            raise NotImplementedError("must pass --sha-files for now")

        for sha, path in hashes.items():
            # http://localhost:8080/api/v1/byhash?sha=afe49172f709725a4503c9219fb4c6a9db8ad0354fc493f2f500269ac6faeaf6
            try:
                client.byhash(sha)
                # if the file is a dupe, do nothing
            except HTTPError as he:
                # if the file is original, print its path
                if he.response.status_code == 404:
                    print(path)
                else:
                    raise

    elif args.action == "ingest":
        if args.copy_of and args.tag:
            raise NotImplementedError("--copy-of not allowed with --tag")

        # TODO refactor this to be less ugly
        tag = None
        if args.tag:
            tag = args.tag.lower()
            try:
                client.list_tags(name=tag).json()
            except requests.exceptions.HTTPError as he:
                if he.response.status_code != 404:
                    raise
                client.create_tag(tag, tag, "")

        sets, skipped = get_photosets(args.files)

        rows = []
        for set_ in sets:
            rows.append([" ".join([f.path for f in set_.files]), "upload"])
        for fname in skipped:
            rows.append([fname, "skip"])

        print(tabulate(rows, headers=["files", "action"]))  # TODO also dupe check here

        if not sets:
            return

        print()
        if not args.yes:
            if not confirm("Continue?"):
                return
            print()

        results = []
        numerrors = 0

        def printprogress():
            print(f"\rProgress: total: {len(sets)}  completed: {len(results)}   errors: {numerrors} ", end="")

        def upload_set(set_):
            payload = set_.to_json()

            # add tags to set
            if args.tag:
                # it's kinda cheap to do this in json instead of building out the PhotoSet data structure
                # but trust me it's worth it
                payload["tags"] = [tag]

            payload["files"] = {os.path.basename(photo.path): photo.to_json() for photo in set_.files}

            files = []
            for file in set_.files:
                files.append(("files", (os.path.basename(file.path), open(file.path, 'rb'), file.format), ))

            if args.copy_of:
                payload["uuid"] = args.copy_of

            try:
                result = client.upload(files, payload).json()
                return ["success", result["uuid"]]
            except HTTPError as he:
                return ["error", he.response.json()["error"]]

        printprogress()

        with ThreadPoolExecutor(max_workers=args.workers) as executor:
            futures = {executor.submit(upload_set, set_): set_ for set_ in sets}
            for future in as_completed(futures.keys()):
                set_ = futures[future]
                set_fnames = [os.path.basename(file.path) for file in set_.files]
                e = future.exception()
                if e:
                    results.append([set_fnames, "exception", repr(e)])
                    numerrors += 1
                else:
                    result = future.result()
                    if result[0] != "success":
                        numerrors += 1
                    results.append([set_fnames] + result)
                printprogress()

        print()
        print()
        print(tabulate([[" ".join(row[0]), row[1], row[2]] for row in results],
                       headers=["files", "status", "uuid"]))

        print("\nErrors:", numerrors)
        # TODO be nice and close the files

    elif args.action == "list":
        headers = ["link" if args.show_link else "uuid",
                   "status",
                   "size",
                   "formats",
                   "date",
                   "title"]# TODO tags
        rows = []
        for set_ in client.list_photos(args.page, args.page_size):
            row = [set_["uuid"] if not args.show_link else args.host.rstrip("/") + "/photo/" + set_["uuid"],
                   PhotoStatus[set_["status"]].name,
                   sum([p["size"] for p in set_["files"].values()]),
                   " ".join(set([p["format"] for p in set_["files"].values()])),
                   set_["date"],
                   maybetruncate(set_["title"], 24),
                   ]
            rows.append(row)
        print(tabulate(rows, headers=headers))

    elif args.action == "stats":
        print(tabulate(sorted([[k, v] for k, v in client.stats().items()],
                              key=lambda row: row[0]),
                       headers=["item", "count"]))

    elif args.action == "user":
        if args.action_user == "create":
            print(client.create_user(args.username, args.password).json())
        elif args.action_user == "list":
            for u in client.list_users().json():
                print(f"{u['name']}: {u['status']}")
        elif args.action_user == "delete":
            print(client.delete_user(args.username).json())

    elif args.action == "tag":
        if args.action_tag == "create":
            client.create_tag(args.name, args.title, args.description)
        elif args.action_tag == "list":
            rows = []
            for tag in client.list_tags(name=args.name).json():
                rows.append([tag["name"], tag["title"], maybetruncate(tag["description"], 24), tag["uuid"]])
            print(tabulate(rows, headers=["name", "title", "description", "uuid"]))


if __name__ == "__main__":
    main()