diff --git a/pysonic/api.py b/pysonic/api.py index e55c7a2..eecb26d 100644 --- a/pysonic/api.py +++ b/pysonic/api.py @@ -188,6 +188,10 @@ class PysonicApi(object): # bitRate="320" # path="Cosmic Gate/Sign Of The Times/03 Flatline (featuring Kyler England).mp3" type="music") + if item["size"] != -1: + child.attrs["size"] = item["size"] + if "media_length" in item_meta: + child.attrs["duration"] = item_meta["media_length"] if "albumId" in directory: child.attrs["albumId"] = directory["id"] if "artistId" in directory: @@ -212,8 +216,13 @@ class PysonicApi(object): assert maxBitRate >= 32 and maxBitRate <= 320 fpath = self.library.get_filepath(id) meta = self.library.get_file_metadata(id) + to_bitrate = min(maxBitRate, self.options.max_bitrate, meta.get("media_kbitrate", 320)) cherrypy.response.headers['Content-Type'] = 'audio/mpeg' - if self.options.skip_transcode and meta["type"] == "audio/mpeg": + if "media_length" in meta: + cherrypy.response.headers['X-Content-Duration'] = str(int(meta['media_length'])) + cherrypy.response.headers['X-Content-Kbitrate'] = str(to_bitrate) + if (self.options.skip_transcode or meta.get("media_kbitrate", -1) == to_bitrate) \ + and meta["type"] == "audio/mpeg": def content(): with open(fpath, "rb") as f: while True: @@ -221,26 +230,37 @@ class PysonicApi(object): if not data: break yield data + return content() else: + transcode_meta = "transcoded_{}_size".format(to_bitrate) + if transcode_meta in meta: + cherrypy.response.headers['Content-Length'] = str(int(meta[transcode_meta])) + transcode_args = ["ffmpeg", "-i", fpath, "-map", "0:0", "-b:a", - "{}k".format(min(maxBitRate, self.options.max_bitrate)), + "{}k".format(to_bitrate), "-v", "0", "-f", "mp3", "-"] logging.info(' '.join(transcode_args)) proc = subprocess.Popen(transcode_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) def content(proc): + length = 0 + completed = False start = time() try: while True: data = proc.stdout.read(16 * 1024) if not data: + completed = True break yield data + length += len(data) finally: proc.poll() - if proc.returncode is None: + if proc.returncode is None or proc.returncode == 0: logging.warning("transcoded {} in {}s".format(id, int(time() - start))) + if completed: + self.library.report_transcode(id, to_bitrate, length) else: logging.error("transcode of {} exited with code {} after {}s".format(id, proc.returncode, int(time() - start))) @@ -255,7 +275,7 @@ class PysonicApi(object): Thread(target=stopit, args=(proc, )).start() - return content(proc) + return content(proc) stream_view._cp_config = {'response.stream': True} @cherrypy.expose diff --git a/pysonic/daemon.py b/pysonic/daemon.py index ddbeba1..567d5f3 100644 --- a/pysonic/daemon.py +++ b/pysonic/daemon.py @@ -22,7 +22,11 @@ def main(): group = parser.add_argument_group("app options") group.add_argument("--skip-transcode", action="store_true", help="instead of trancoding mp3s, send as-is") + group.add_argument("--no-rescan", action="store_true", help="don't perform simple scan on startup") + group.add_argument("--deep-rescap", action="store_true", help="perform deep scan (read id3 etc)") + group.add_argument("--enable-prune", action="store_true", help="enable removal of media not found on disk") group.add_argument("--max-bitrate", type=int, default=320, help="maximum send bitrate") + group.add_argument("--enable-cors", action="store_true", help="add response headers to allow cors") args = parser.parse_args() @@ -53,6 +57,12 @@ def main(): api_config.update({'tools.auth_basic.on': True, 'tools.auth_basic.realm': 'pysonic', 'tools.auth_basic.checkpassword': db.validate_password}) + if args.enable_cors: + def cors(): + cherrypy.response.headers["Access-Control-Allow-Origin"] = "*" + cherrypy.tools.cors = cherrypy.Tool('before_handler', cors) + api_config.update({'tools.cors.on': True}) + cherrypy.tree.mount(api, '/rest/', {'/': api_config}) cherrypy.config.update({ diff --git a/pysonic/database.py b/pysonic/database.py index b2f41a9..97e01b0 100644 --- a/pysonic/database.py +++ b/pysonic/database.py @@ -3,11 +3,11 @@ import json import sqlite3 import logging from hashlib import sha512 -from itertools import chain from contextlib import closing logging = logging.getLogger("database") +keys_in_table = ["title", "album", "artist", "type", "size"] def dict_factory(cursor, row): @@ -39,18 +39,29 @@ class PysonicDatabase(object): queries = ["""CREATE TABLE 'meta' ( 'key' TEXT PRIMARY KEY NOT NULL, 'value' TEXT);""", - """INSERT INTO meta VALUES ('db_version', '0');""", + """INSERT INTO meta VALUES ('db_version', '3');""", """CREATE TABLE 'nodes' ( 'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 'parent' INTEGER NOT NULL, 'isdir' BOOLEAN NOT NULL, + 'size' INTEGER NOT NULL DEFAULT -1, 'name' TEXT NOT NULL, 'type' TEXT, 'title' TEXT, 'album' TEXT, 'artist' TEXT, 'metadata' TEXT - )"""] + )""", + """CREATE TABLE 'users' ( + 'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + 'username' TEXT UNIQUE NOT NULL, + 'password' TEXT NOT NULL, + 'admin' BOOLEAN DEFAULT 0, + 'email' TEXT)""", + """CREATE TABLE 'stars' ( + 'userid' INTEGER, + 'nodeid' INTEGER, + primary key ('userid', 'nodeid'))"""] with closing(self.db.cursor()) as cursor: cursor.execute("SELECT * FROM sqlite_master WHERE type='table' AND name='meta';") @@ -81,6 +92,11 @@ class PysonicDatabase(object): primary key ('userid', 'nodeid'))""" cursor.execute(stars_table) version = 2 + if version < 3: + logging.warning("migrating database to v3 from %s", version) + size_col = """ALTER TABLE nodes ADD 'size' INTEGER NOT NULL DEFAULT -1;""" + cursor.execute(size_col) + version = 3 cursor.execute("""UPDATE meta SET value=? WHERE key="db_version";""", (str(version), )) logging.warning("db schema is version {}".format(version)) @@ -125,7 +141,7 @@ class PysonicDatabase(object): if types: add_filter("type", types) - query = query.rstrip(" AND") + query = query.rstrip(" AND").rstrip("WHERE ") if order: query += "ORDER BY " @@ -138,15 +154,15 @@ class PysonicDatabase(object): with closing(self.db.cursor()) as cursor: return list(map(self._populate_meta, cursor.execute(query, qargs).fetchall())) - def addnode(self, parent_id, fspath, name): + def addnode(self, parent_id, fspath, name, size=-1): fullpath = os.path.join(fspath, name) is_dir = os.path.isdir(fullpath) - return self._addnode(parent_id, name, is_dir) + return self._addnode(parent_id, name, is_dir, size=size) - def _addnode(self, parent_id, name, is_dir=True): + def _addnode(self, parent_id, name, is_dir=True, size=-1): with closing(self.db.cursor()) as cursor: - cursor.execute("INSERT INTO nodes (parent, isdir, name) VALUES (?, ?, ?);", - (parent_id, 1 if is_dir else 0, name)) + cursor.execute("INSERT INTO nodes (parent, isdir, name, size) VALUES (?, ?, ?, ?);", + (parent_id, 1 if is_dir else 0, name, size)) return self.getnode(cursor.lastrowid) def delnode(self, node_id): @@ -159,7 +175,6 @@ class PysonicDatabase(object): def update_metadata(self, node_id, mergedict=None, **kwargs): mergedict = mergedict if mergedict else {} - keys_in_table = ["title", "album", "artist", "type"] mergedict.update(kwargs) with closing(self.db.cursor()) as cursor: for table_key in keys_in_table: @@ -173,7 +188,6 @@ class PysonicDatabase(object): cursor.execute("UPDATE nodes SET metadata=? WHERE id=?;", (json.dumps(metadata), node_id, )) def get_metadata(self, node_id): - keys_in_table = ["title", "album", "artist", "type"] node = self.getnode(node_id) meta = node["metadata"] meta.update({item: node[item] for item in keys_in_table}) diff --git a/pysonic/library.py b/pysonic/library.py index 28045dd..614e319 100644 --- a/pysonic/library.py +++ b/pysonic/library.py @@ -108,3 +108,14 @@ class PysonicLibrary(object): def get_songs(self, limit=50, shuffle=True): return self.db.getnodes(types=MUSIC_TYPES, limit=limit, order="rand") + + def get_song(self, id=None): + if id: + return self.db.getnode(id) + else: + return self.db.getnodes(types=MUSIC_TYPES, limit=1, order="rand") + + def report_transcode(self, item_id, bitrate, num_bytes): + assert type(bitrate) is int and bitrate > 0 and bitrate <= 320 + logging.info("Got transcode report of {} for item {} @ {}".format(num_bytes, item_id, bitrate)) + self.db.update_metadata(item_id, {"transcoded_{}_size".format(bitrate):int(num_bytes)}) diff --git a/pysonic/scanner.py b/pysonic/scanner.py index c18c7b6..151df54 100644 --- a/pysonic/scanner.py +++ b/pysonic/scanner.py @@ -4,10 +4,12 @@ import logging import mimetypes from time import time from threading import Thread -from pysonic.types import KNOWN_MIMES, MUSIC_TYPES +from pysonic.types import KNOWN_MIMES, MUSIC_TYPES, MPX_TYPES, FLAC_TYPES, WAV_TYPES from mutagen.id3 import ID3 from mutagen import MutagenError from mutagen.id3._util import ID3NoHeaderError +from mutagen.flac import FLAC +from mutagen.mp3 import MP3 logging = logging.getLogger("scanner") @@ -31,7 +33,7 @@ class PysonicFilesystemScanner(object): logging.info("Scanning {}".format(meta["fspath"])) def recurse_dir(path, parent): - logging.info("Scanning {} with parent {}".format(path, parent)) + logging.info("Scanning {}".format(path)) # create or update the database of nodes by comparing sets of names fs_entries = set(os.listdir(path)) db_entires = self.library.db.getnodes(parent["id"]) @@ -39,9 +41,17 @@ class PysonicFilesystemScanner(object): to_delete = db_entires_names - fs_entries to_create = fs_entries - db_entires_names + # If any size have changed, mark the file to be rescanned + for entry in db_entires: + finfo = os.stat(os.path.join(path, entry["name"])) + if finfo.st_size != entry["size"]: + logging.info("{} has changed in size, marking for meta rescan".format(entry["id"])) + self.library.db.update_metadata(entry['id'], id3_done=False, size=finfo.st_size) + # Create any nodes not found in the db for create in to_create: - new_node = self.library.db.addnode(parent["id"], path, create) + new_finfo = os.stat(os.path.join(path, create)) + new_node = self.library.db.addnode(parent["id"], path, create, size=new_finfo.st_size) logging.info("Added {}".format(os.path.join(path, create))) db_entires.append(new_node) @@ -56,9 +66,9 @@ class PysonicFilesystemScanner(object): for entry in db_entires: if entry["name"] in to_delete: continue - if int(entry['isdir']): # 1 means dir recurse_dir(os.path.join(path, entry["name"]), entry) + # Populate all files for this top-level root recurse_dir(meta["fspath"], parent) # @@ -100,7 +110,7 @@ class PysonicFilesystemScanner(object): # # # - # Add advanced id3 metadata + # Add advanced id3 / media info metadata for artist_dir in self.library.db.getnodes(parent["id"]): artist = artist_dir["name"] for album_dir in self.library.db.getnodes(artist_dir["id"]): @@ -110,37 +120,56 @@ class PysonicFilesystemScanner(object): track_meta = track_file['metadata'] title = track_file["name"] fpath = self.library.get_filepath(track_file["id"]) - if track_meta.get('id3_done', False) or track_file.get("type", "x") not in MUSIC_TYPES: + if track_meta.get('id3_done', False) or track_file.get("type", None) not in MUSIC_TYPES: continue - print("Mutagening", fpath) tags = {'id3_done': True} try: - id3 = ID3(fpath) - # print(id3.pprint()) + audio = None + if track_file.get("type", None) in MPX_TYPES: + audio = MP3(fpath) + if audio.info.sketchy: + logging.warning("media reported as sketchy: %s", fpath) + elif track_file.get("type", None) in FLAC_TYPES: + audio = FLAC(fpath) + else: + audio = ID3(fpath) + # print(audio.pprint()) try: - tags["track"] = int(RE_NUMBERS.findall(''.join(id3['TRCK'].text))[0]) + tags["media_length"] = int(audio.info.length) + except (ValueError, AttributeError): + pass + try: + bitrate = int(audio.info.bitrate) + tags["media_bitrate"] = bitrate + tags["media_kbitrate"] = int(bitrate / 1024) + except (ValueError, AttributeError): + pass + try: + tags["track"] = int(RE_NUMBERS.findall(''.join(audio['TRCK'].text))[0]) except (KeyError, IndexError): pass try: - tags["id3_artist"] = ''.join(id3['TPE1'].text) + tags["id3_artist"] = ''.join(audio['TPE1'].text) except KeyError: pass try: - tags["id3_album"] = ''.join(id3['TALB'].text) + tags["id3_album"] = ''.join(audio['TALB'].text) except KeyError: pass try: - tags["id3_title"] = ''.join(id3['TIT2'].text) + tags["id3_title"] = ''.join(audio['TIT2'].text) except KeyError: pass try: - tags["id3_year"] = id3['TDRC'].text[0].year + tags["id3_year"] = audio['TDRC'].text[0].year except (KeyError, IndexError): pass + logging.info("got all media info from %s", fpath) except ID3NoHeaderError: pass except MutagenError as m: - logging.error(m) + logging.error("failed to read audio information: %s", m) + continue self.library.db.update_metadata(track_file["id"], **tags) - logging.warning("Library scan complete in {}s".format(int(time() - start))) + logging.warning("Library scan complete in {}s".format(round(time() - start, 2))) diff --git a/pysonic/types.py b/pysonic/types.py index ab6912a..28f99fe 100644 --- a/pysonic/types.py +++ b/pysonic/types.py @@ -1,4 +1,7 @@ KNOWN_MIMES = ["audio/mpeg", "audio/flac", "audio/x-wav", "image/jpeg", "image/png"] MUSIC_TYPES = ["audio/mpeg", "audio/flac", "audio/x-wav"] +MPX_TYPES = ["audio/mpeg"] +FLAC_TYPES = ["audio/flac"] +WAV_TYPES = ["audio/x-wav"] IMAGE_TYPES = ["image/jpeg", "image/png"]