From bbce07f974745b8126f03291cd37be00ce48e205 Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 17 Aug 2019 17:57:02 -0700 Subject: [PATCH] support multiple deluge instances --- README.md | 85 ++++++++++++----- mediaweb/__init__.py | 198 +++++++++++++++++++++++++--------------- mediaweb/shows.py | 6 +- templates/index.html | 6 +- templates/moveform.html | 2 +- templates/page.html | 3 + templates/sortform.html | 3 +- 7 files changed, 198 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index 8f5f392..dca32d8 100644 --- a/README.md +++ b/README.md @@ -7,46 +7,89 @@ Webapp for quick or automatic media sorting and integration with the Deluge torr configuration ------------- -Basic flags +Using a configuration file is recommended due to the complexity of the information needed. An example config is shown +and explained below: -|flag|meaning|example| +``` +{ + "port": 8081, + "debug": true, + "library_path": "/media/raid/Media/TV/HD/", + "label": "sickrage", + "label_done": "sorted", + "stashprefix": "Sorted", + "deluges": [ + { + "name": "T2", + "uri": "deluge://user:pass@localhost:58846", + "pathmap": ["/media/storage", "/media/raid/Torrents/Torrent1"] + }, + { + "name": "T5", + "uri": "deluge://user:pass@otherhost.com:58846", + "pathmap": ["/media/storage", "/media/raid/Torrents/Torrent2"] + } + ], + "trackermap": { + "empirehost.me": "ipt" + }, + "movedests": [ + "Complete", + "SBComplete", + "Sorted/ipt" + ] +} +``` + +Basic options + +|key|meaning|example| |---|---|---| -|--server|deluge rpc uri|deluge://username:password@host:port| -|--port|8081|http port to listen on| +|port|8081|http port to listen on| +|debug|true|run the server in debug mode| +|library_path|/data/library|file path to tv media library| +|label|"sickrage"|deluge label to look for sortable torrents under| +|label_done|"sorted"|deluge label to mark sortable torrents as| +|stashprefix|"Sorted"|when stashing torrents, prefix for the stash path| -Mediasort has several options to tune how it sorts your media. +Beyond these options, mediasort has several more options to tune how it sorts your media that require more explanation. -First, `--library` should be set to your media library's path. The media library must contain top level directories for -each show, appropriately name. The name of the directory will be used to determine what show to put in it. In each show -directory, there should be season dirs (such as "Season 6" or "2019") within which the actual media files are placed. +Details about client instances are list under the `deluges` key. Each client requires a deluge connection string (a uri +starting with deluge://) and a pathmap. The pathmap is explained later. -If needed, `--pathmap` can be set to translate paths when sorting files. This would be needed if your torrent client has -a different view of the filesystem than `mediasort` does, e.g. if they're running in docker containers. Consider these -two paths: +Next, `library_path` should be set to your media library's path. The media library must contain top level directories +for each show, matchingly named. The name of the directory will be used to determine what show to put in it. In each +show directory, there should be one directory per season (such as "Season 6" or "2019") within which the actual media +files are placed. + +Per client, `pathmap` must be set to translate paths when sorting files. This would be needed if your torrent client +has a different view of the filesystem than `mediasort` does, e.g. if they're running in docker containers. Consider +these two paths: * `/media/storage/mylibrary/myshow/Season 5/episode.mkv` * `/data/torrents/Complete/myshow.mkv` The first is the destination path in your media library as seen by mediasort. The second is the path Deluge sees. -Setting `--pathmap` to `/data/torrents/:/media/storage/torrents/` gives mediasort the info it needs to resolve these -paths - it translates the Deluge path by simply replacing the prefix obtained from the left half of the `--pathmap` with +Setting `pathmap` to `/data/torrents/:/media/storage/torrents/` gives mediasort the info it needs to resolve these +paths - it translates the Deluge path by simply replacing the prefix obtained from the left half of the `pathmap` with the right. -Finally, `--ui-movedests` provides a list of pre-filled destinations that Deluge may move files to after they're sorted. -These paths are passed directly to deluge and should be pathed from that perspective. +Finally, `movedests` provides a list of pre-filled destinations that Deluge may move files to after they're sorted. +These paths are appended to the left side of the pathmap. -After a torrent is sorted into your library, the torrent will be moved to the "stash" dir, simply to the completed -downloads directory sparse. The `--stashprefix` flag controls what directory the files are moved into. Within the stash -directory the torrents are placed in a subdirectory matching the hostname of the torrent's tracker, optionally -translated to another name with the values specified in `--trackermap`. Passing "foo.com:foo" would cause the "foo.com" -tracker directory to be named "foo" instead. +After a torrent is sorted into your library, the torrent will "stashed", that is, moved to a directory other than your +default download location as well as labeled as such. Within the stash directory, the torrents are placed in another +subdirectory matching the hostname of the torrent's tracker. Optionally, these names can be translated to another name +with the values specified in `trackermap`. For example, having the `trackermap` entry `"foo.com": "foo",` would cause +the "foo.com" tracker directory to be named "foo" instead. todo ---- +* support full configuration through cli flags again +* support env vars in addition to command line flags for full configuration * support re-labeling sorted torrents * more options than just hard linking (soft link, copy) * support sorting season torrents -* support multiple deluge instances - need a pathmap per instance * make UI pretty * support post-sort webhooks e.g. to tell plex to rescan when we add something diff --git a/mediaweb/__init__.py b/mediaweb/__init__.py index 2584463..6760b39 100644 --- a/mediaweb/__init__.py +++ b/mediaweb/__init__.py @@ -1,4 +1,5 @@ import os +import json import logging import cherrypy from time import sleep @@ -21,22 +22,36 @@ class Cache: moves: dict = field(default_factory=dict) +@dataclass +class Client: + rpc: DelugeRPCClient + pathmap: (str, str) + + class ClientCache(object): - def __init__(self, client, libpath): - self.client = client + def __init__(self, options, libpath): + self.options = options + self.clients = {} self.data = Cache() self.q = Queue() self.inflight = False self.libpath = libpath self.background_t = Thread(target=self.background, daemon=True) - self.background_t.start() - self.timer_t = Thread(target=self.timer, daemon=True) + + def start(self): + self.background_t.start() self.timer_t.start() + def add_client(self, rpc, pathmap): # not thread safe + cnum = len(self.clients) + self.data.torrents[cnum] = {} + self.data.moves[cnum] = {} + self.clients[cnum] = Client(rpc, pathmap) + def refresh(self): - if not self.inflight and self.q.qsize() == 0: # best effort duplicate work reduction + if self.q.qsize() <= 1: # guarantees a refresh after your call and also deduplicates effort self.q.put(None) def background(self): @@ -61,9 +76,14 @@ class ClientCache(object): def build_torrentindex(self): logging.info("refreshing torrents") - self.data.torrents = self.client.core.get_torrents_status({"label": "sickrage"}, # TODO parameterize - ['name', 'label', 'save_path', 'is_seed', - 'is_finished', 'progress', 'files']) + for cid, client in self.clients.items(): # TODO parallelize + self.data.torrents[cid] = client.rpc.core.get_torrents_status({"label": self.options["label"]}, + ['name', 'label', 'save_path', 'is_seed', + 'is_finished', 'progress', 'files']) + newkeys = self.data.torrents[cid].keys() + for key in list(self.data.moves[cid].keys()): + if key not in newkeys: + del self.data.moves[cid][key] # delete precomputed sort operations for torrents that went away def build_showindex(self): logging.info("updating show index") @@ -72,28 +92,54 @@ class ClientCache(object): def queue_sorts(self): logging.info("precomputing sorts") - for thash, torrent in self.data.torrents.items(): - if thash not in self.data.moves: - matches = shows.match_episode(get_fname(torrent), self.data.shows) - if matches: - self.data.moves[thash] = matches[0] + for cid, torrents in self.data.torrents.items(): + for thash, torrent in torrents.items(): + if thash not in self.data.moves[cid]: + matches = shows.match_episode(get_fname(torrent), self.data.shows) + if matches: + self.data.moves[cid][thash] = matches[0] + + @property + def torrents(self): + return {self.format_tkey(cid, thash): torrent + for cid, torrents in self.data.torrents.items() + for thash, torrent in torrents.items()} + + @property + def moves(self): + return {self.format_tkey(cid, thash): move + for cid, moves in self.data.moves.items() + for thash, move in moves.items()} + + def client(self, tkey): + cid, thash = self.extract_key(tkey) + return thash, self.clients[cid] + + @staticmethod + def extract_key(tkey): + cid, thash = tkey.split(":") + return (int(cid), thash) + + @staticmethod + def format_tkey(cid, thash): + return f"{cid}:{thash}" class MediaWeb(object): - def __init__(self, rpc, templater, options): + def __init__(self, cache, templater, options): self.tpl = templater - self.rpc = rpc + self.cache = cache self.options = options def render(self, template, **kwargs): """ Render a template """ - return self.tpl.get_template(template).render(**kwargs, - options=self.options, - torrents=self.rpc.data.torrents, - shows=self.rpc.data.shows, - moves=self.rpc.data.moves, + return self.tpl.get_template(template).render(options=self.options, + torrents=self.cache.torrents, + moves=self.cache.moves, + shows=self.cache.data.shows, + **kwargs, **self.get_default_vars()) def get_default_vars(self): @@ -103,31 +149,35 @@ class MediaWeb(object): def index(self, action=None): if action: if action == "update": - self.rpc.refresh() + self.cache.refresh() raise cherrypy.HTTPRedirect("/") - return self.render("index.html") + return self.render("index.html", inflight=self.cache.inflight) @cherrypy.expose - def move(self, thash, dest=None, otherdest=None): - torrent = self.rpc.client.core.get_torrent_status(thash, []) # TODO reduce to needed fields + def move(self, tkey, dest=None, otherdest=None): + thash, client = self.cache.client(tkey) + torrent = client.rpc.core.get_torrent_status(thash, []) - if cherrypy.request.method == "POST" and (dest or otherdest): - target = otherdest or dest - self.rpc.client.core.move_storage([thash], target) - self.rpc.refresh() + if cherrypy.request.method == "POST" and (dest or otherdest): # TODO maybe support otherdest list per client + target = os.path.join(client.pathmap[0], + otherdest or dest) + client.rpc.core.move_storage([thash], target) + self.cache.refresh() raise cherrypy.HTTPRedirect("/") - return self.render("moveform.html", torrent=torrent) + return self.render("moveform.html", torrent=torrent, tkey=tkey) @cherrypy.expose - def sort(self, thash, dest=None): - torrent = self.rpc.client.core.get_torrent_status(thash, []) # TODO reduce to needed fields + def sort(self, tkey, thresh=65, dest=None): + thash, client = self.cache.client(tkey) + torrent = client.rpc.core.get_torrent_status(thash, []) # find the actual file among the torrent's files - really we just pick the biggest one fname = get_fname(torrent) # find candidate dest locations - matches = shows.match_episode(fname, self.rpc.data.shows) + thresh = int(thresh) + matches = shows.match_episode(fname, self.cache.data.shows, thresh=thresh) if cherrypy.request.method == "POST" and dest: # pick the candidate dest the user specified @@ -137,20 +187,22 @@ class MediaWeb(object): thematch = m break - self.execute_move(torrent, thematch) - self.rpc.refresh() + self.execute_move(tkey, torrent, thematch) + self.cache.refresh() # TODO summary display return "OK" - return self.render("sortform.html", torrent=torrent, matches=matches) + return self.render("sortform.html", torrent=torrent, matches=matches, tkey=tkey, thresh=thresh) - def execute_move(self, torrent, match): + def execute_move(self, tkey, torrent, match): # resolve the pathmap - pmap = self.options['pathmap'] + thash, client = self.cache.client(tkey) + + pmap = client.pathmap fname = get_fname(torrent) - in_library_path = os.path.join(match.dest.dir, match.subdest, match.ep.file) # path relative from the library's root + in_library_path = os.path.join(match.dest.dir, match.subdest, fname) # path relative from the library's root client_full_path = os.path.join(torrent["save_path"], fname) # absolute storage path in deluge assert client_full_path[0:len(pmap[0])] == pmap[0] # sanity check: deluge's path must starts with our pathmap @@ -160,30 +212,32 @@ class MediaWeb(object): # hard link into library os.link(local_torrent_path, local_library_path) + print(f"os.link('{local_torrent_path}', '{local_library_path}')") client_stashdir = os.path.join(pmap[0], self.options["stashprefix"], get_mapped_stashdir(self.options["trackermap"], torrent["trackers"])) # move deluge path to stash dir - self.rpc.client.core.move_storage([torrent["hash"]], client_stashdir) + client.rpc.core.move_storage([torrent["hash"]], client_stashdir) # label torrent as sorted - self.rpc.client.label.set_torrent(torrent["hash"], self.options["label"]) + client.rpc.label.set_torrent(torrent["hash"], self.options["label_done"]) @cherrypy.expose - def autosort(self, torrents): - if not isinstance(torrents, list): - torrents = [torrents] + def autosort(self, tkeys): + if not isinstance(tkeys, list): + tkeys = [tkeys] - for thash in torrents: - torrent = self.rpc.client.core.get_torrent_status(thash, []) # TODO reduce to needed fields - self.execute_move(torrent, self.rpc.data.moves[thash]) + for tkey in tkeys: + thash, client = self.cache.client(tkey) + torrent = client.rpc.core.get_torrent_status(thash, []) # TODO reduce to needed fields + self.execute_move(tkey, torrent, self.cache.moves[tkey]) self.rpc.refresh() # TODO summary display of results - return f"autosorted: {repr(torrents)}" + return f"autosorted: {repr(tkeys)}" def get_fname(torrent): @@ -213,33 +267,22 @@ def main(): import signal parser = argparse.ArgumentParser(description="mediaweb server") - - parser.add_argument('-p', '--port', help="tcp port to listen on", - default=int(os.environ.get("MEDIAWEB_PORT", 8080)), type=int) - parser.add_argument('-o', '--library', default=os.environ.get("STORAGE_URL"), help="media library path") - parser.add_argument('-a', '--pathmap', help="torrent to library path mapping") + parser.add_argument("-c", "--config", required=True, help="config file path") parser.add_argument('--debug', action="store_true", help="enable development options") - parser.add_argument('-s', '--server', help="deluge uris", action="append", required=True) - parser.add_argument('--ui-movedests', help="move destination options in the UI", nargs="+", required=True) - - parser.add_argument('--stashprefix', help="dir to move finished files in deluge", default="Sorted") - parser.add_argument('--trackermap', help="map tracker names to shash dir names", nargs="+") - parser.add_argument('--label', help="label to apply to sorted torrents", default="sorted") - args = parser.parse_args() - options = { - "movedests": args.ui_movedests, - "pathmap": args.pathmap.split(":"), - "library_path": args.library, - "stashprefix": args.stashprefix, - "trackermap": {i.split(":")[0]: i.split(":")[1] for i in args.trackermap}, - "label": args.label - } + with open(args.config) as f: + cfg = json.load(f) - # TODO smarter argparser that understands env vars - if not args.library: - parser.error("--library or MEDIAWEB_DLDIR is required") + options = { # :| + "movedests": cfg["movedests"], + # "pathmap": args.pathmap.split(":"), + "library_path": cfg["library_path"], + "stashprefix": cfg["stashprefix"], + "trackermap": cfg["trackermap"], + "label": cfg["label"], + "label_done": cfg["label_done"] + } logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") @@ -252,12 +295,14 @@ def main(): def validate_password(realm, user, passw): return user == passw # lol - # assume 1 deluge server for now - uri = urlparse(args.server[0]) - assert uri.scheme == "deluge" - rpc = DelugeRPCClient(uri.hostname, uri.port if uri.port else 58846, uri.username, uri.password, decode_utf8=True) + rpc_cache = ClientCache(options, cfg["library_path"]) - rpc_cache = ClientCache(rpc, args.library) + for client in cfg["deluges"]: + uri = urlparse(client["uri"]) + assert uri.scheme == "deluge" + port = uri.port if uri.port else 58846 + rpc_cache.add_client(DelugeRPCClient(uri.hostname, port, uri.username, uri.password, decode_utf8=True), + tuple(client["pathmap"])) web = MediaWeb(rpc_cache, tpl, options) cherrypy.tree.mount(web, '/', {'/': {'tools.auth_basic.on': True, @@ -269,7 +314,7 @@ def main(): 'tools.sessions.on': False, 'request.show_tracebacks': True, 'server.show_tracebacks': True, - 'server.socket_port': args.port, + 'server.socket_port': cfg["port"], 'server.thread_pool': 1 if args.debug else 5, 'server.socket_host': '0.0.0.0', 'log.screen': False, @@ -285,6 +330,7 @@ def main(): signal.signal(signal.SIGTERM, signal_handler) try: + rpc_cache.start() cherrypy.engine.start() cherrypy.engine.block() finally: diff --git a/mediaweb/shows.py b/mediaweb/shows.py index 848ef64..73a17e6 100644 --- a/mediaweb/shows.py +++ b/mediaweb/shows.py @@ -21,7 +21,7 @@ COMMON_CRAP = [re.compile(i, flags=re.I) for i in r'web(\-?(dl|rip))?', r'[\.\-\s](amzn|amazon)[\.\-\s]', r'dd.5.\d', - r'AAC2.\d']] + r'aac2.\d']] class EpisodeParseException(Exception): @@ -132,7 +132,7 @@ def parse_episode(fname): """ # Remove file extensions - # item = fname.rstrip(".mkv").lower() #TODO make this better + fname = fname.lower() item = '.'.join(fname.split(".")[0:-1]) # Extract season information @@ -214,7 +214,7 @@ def match_episode(fname, shows, thresh=65): # Find a show from the library best matching this episode for show in shows: value = fuzz.token_set_ratio(show.name.lower(), item.lower()) #TODO add algorithm swap arg for snakeoil - if value > thresh: + if value >= thresh: matches.append( MatchedEpisode(fname, epinfo, show, sub_bucket_name(show, epinfo.major, epinfo.minor, epinfo.extra), diff --git a/templates/index.html b/templates/index.html index beee073..b195590 100644 --- a/templates/index.html +++ b/templates/index.html @@ -2,7 +2,7 @@ {% block toolbar %}
- + {% if inflight %}update in progress...{% else %}{% endif %}
{% endblock %} {% block body %} @@ -35,8 +35,8 @@ ? {% endif %} - - + + {% endif %}{% endfor %} diff --git a/templates/moveform.html b/templates/moveform.html index 379bc6c..7822ce7 100644 --- a/templates/moveform.html +++ b/templates/moveform.html @@ -17,7 +17,7 @@
- +
new path +
destination @@ -44,6 +44,7 @@

+ show more