File size/duration awarenness

pysonic/api.py

@@ -188,6 +188,10 @@ class PysonicApi(object):
                             # bitRate="320"
                             # path="Cosmic Gate/Sign Of The Times/03 Flatline (featuring Kyler England).mp3"
                             type="music")
+        if item["size"] != -1:
+            child.attrs["size"] = item["size"]
+        if "media_length" in item_meta:
+            child.attrs["duration"] = item_meta["media_length"]
         if "albumId" in directory:
             child.attrs["albumId"] = directory["id"]
         if "artistId" in directory:
@@ -212,8 +216,13 @@ class PysonicApi(object):
         assert maxBitRate >= 32 and maxBitRate <= 320
         fpath = self.library.get_filepath(id)
         meta = self.library.get_file_metadata(id)
+        to_bitrate = min(maxBitRate, self.options.max_bitrate, meta.get("media_kbitrate", 320))
         cherrypy.response.headers['Content-Type'] = 'audio/mpeg'
-        if self.options.skip_transcode and meta["type"] == "audio/mpeg":
+        if "media_length" in meta:
+            cherrypy.response.headers['X-Content-Duration'] = str(int(meta['media_length']))
+        cherrypy.response.headers['X-Content-Kbitrate'] = str(to_bitrate)
+        if (self.options.skip_transcode or meta.get("media_kbitrate", -1) == to_bitrate) \
+           and meta["type"] == "audio/mpeg":
             def content():
                 with open(fpath, "rb") as f:
                     while True:
@@ -221,26 +230,37 @@ class PysonicApi(object):
                         if not data:
                             break
                         yield data
+            return content()
         else:
+            transcode_meta = "transcoded_{}_size".format(to_bitrate)
+            if transcode_meta in meta:
+                cherrypy.response.headers['Content-Length'] = str(int(meta[transcode_meta]))
+
             transcode_args = ["ffmpeg", "-i", fpath, "-map", "0:0", "-b:a",
-                              "{}k".format(min(maxBitRate, self.options.max_bitrate)),
+                              "{}k".format(to_bitrate),
                               "-v", "0", "-f", "mp3", "-"]
             logging.info(' '.join(transcode_args))
             proc = subprocess.Popen(transcode_args, stdin=subprocess.PIPE,
                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
             def content(proc):
+                length = 0
+                completed = False
                 start = time()
                 try:
                     while True:
                         data = proc.stdout.read(16 * 1024)
                         if not data:
+                            completed = True
                             break
                         yield data
+                        length += len(data)
                 finally:
                     proc.poll()
-                    if proc.returncode is None:
+                    if proc.returncode is None or proc.returncode == 0:
                         logging.warning("transcoded {} in {}s".format(id, int(time() - start)))
+                        if completed:
+                            self.library.report_transcode(id, to_bitrate, length)
                     else:
                         logging.error("transcode of {} exited with code {} after {}s".format(id, proc.returncode,
                                                                                              int(time() - start)))
@@ -255,7 +275,7 @@ class PysonicApi(object):
 
             Thread(target=stopit, args=(proc, )).start()
 
-        return content(proc)
+            return content(proc)
     stream_view._cp_config = {'response.stream': True}
 
     @cherrypy.expose

pysonic/daemon.py

@@ -22,7 +22,11 @@ def main():
 
     group = parser.add_argument_group("app options")
     group.add_argument("--skip-transcode", action="store_true", help="instead of trancoding mp3s, send as-is")
+    group.add_argument("--no-rescan", action="store_true", help="don't perform simple scan on startup")
+    group.add_argument("--deep-rescap", action="store_true", help="perform deep scan (read id3 etc)")
+    group.add_argument("--enable-prune", action="store_true", help="enable removal of media not found on disk")
     group.add_argument("--max-bitrate", type=int, default=320, help="maximum send bitrate")
+    group.add_argument("--enable-cors", action="store_true", help="add response headers to allow cors")
 
     args = parser.parse_args()
 
@@ -53,6 +57,12 @@ def main():
         api_config.update({'tools.auth_basic.on': True,
                            'tools.auth_basic.realm': 'pysonic',
                            'tools.auth_basic.checkpassword': db.validate_password})
+    if args.enable_cors:
+        def cors():
+            cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
+        cherrypy.tools.cors = cherrypy.Tool('before_handler', cors)
+        api_config.update({'tools.cors.on': True})
+
     cherrypy.tree.mount(api, '/rest/', {'/': api_config})
 
     cherrypy.config.update({

pysonic/database.py

@@ -3,11 +3,11 @@ import json
 import sqlite3
 import logging
 from hashlib import sha512
-from itertools import chain
 from contextlib import closing
 
 
 logging = logging.getLogger("database")
+keys_in_table = ["title", "album", "artist", "type", "size"]
 
 
 def dict_factory(cursor, row):
@@ -39,18 +39,29 @@ class PysonicDatabase(object):
         queries = ["""CREATE TABLE 'meta' (
                         'key' TEXT PRIMARY KEY NOT NULL,
                         'value' TEXT);""",
-                   """INSERT INTO meta VALUES ('db_version', '0');""",
+                   """INSERT INTO meta VALUES ('db_version', '3');""",
                    """CREATE TABLE 'nodes' (
                         'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
                         'parent' INTEGER NOT NULL,
                         'isdir' BOOLEAN NOT NULL,
+                        'size' INTEGER NOT NULL DEFAULT -1,
                         'name' TEXT NOT NULL,
                         'type' TEXT,
                         'title' TEXT,
                         'album' TEXT,
                         'artist' TEXT,
                         'metadata' TEXT
-                        )"""]
+                        )""",
+                   """CREATE TABLE 'users' (
+                        'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+                        'username' TEXT UNIQUE NOT NULL,
+                        'password' TEXT NOT NULL,
+                        'admin' BOOLEAN DEFAULT 0,
+                        'email' TEXT)""",
+                   """CREATE TABLE 'stars' (
+                        'userid' INTEGER,
+                        'nodeid' INTEGER,
+                        primary key ('userid', 'nodeid'))"""]
 
         with closing(self.db.cursor()) as cursor:
             cursor.execute("SELECT * FROM sqlite_master WHERE type='table' AND name='meta';")
@@ -81,6 +92,11 @@ class PysonicDatabase(object):
                                         primary key ('userid', 'nodeid'))"""
                     cursor.execute(stars_table)
                     version = 2
+                if version < 3:
+                    logging.warning("migrating database to v3 from %s", version)
+                    size_col = """ALTER TABLE nodes ADD 'size' INTEGER NOT NULL DEFAULT -1;"""
+                    cursor.execute(size_col)
+                    version = 3
 
                 cursor.execute("""UPDATE meta SET value=? WHERE key="db_version";""", (str(version), ))
                 logging.warning("db schema is version {}".format(version))
@@ -125,7 +141,7 @@ class PysonicDatabase(object):
         if types:
             add_filter("type", types)
 
-        query = query.rstrip(" AND")
+        query = query.rstrip(" AND").rstrip("WHERE ")
 
         if order:
             query += "ORDER BY "
@@ -138,15 +154,15 @@ class PysonicDatabase(object):
         with closing(self.db.cursor()) as cursor:
             return list(map(self._populate_meta, cursor.execute(query, qargs).fetchall()))
 
-    def addnode(self, parent_id, fspath, name):
+    def addnode(self, parent_id, fspath, name, size=-1):
         fullpath = os.path.join(fspath, name)
         is_dir = os.path.isdir(fullpath)
-        return self._addnode(parent_id, name, is_dir)
+        return self._addnode(parent_id, name, is_dir, size=size)
 
-    def _addnode(self, parent_id, name, is_dir=True):
+    def _addnode(self, parent_id, name, is_dir=True, size=-1):
         with closing(self.db.cursor()) as cursor:
-            cursor.execute("INSERT INTO nodes (parent, isdir, name) VALUES (?, ?, ?);",
-                           (parent_id, 1 if is_dir else 0, name))
+            cursor.execute("INSERT INTO nodes (parent, isdir, name, size) VALUES (?, ?, ?, ?);",
+                           (parent_id, 1 if is_dir else 0, name, size))
             return self.getnode(cursor.lastrowid)
 
     def delnode(self, node_id):
@@ -159,7 +175,6 @@ class PysonicDatabase(object):
 
     def update_metadata(self, node_id, mergedict=None, **kwargs):
         mergedict = mergedict if mergedict else {}
-        keys_in_table = ["title", "album", "artist", "type"]
         mergedict.update(kwargs)
         with closing(self.db.cursor()) as cursor:
             for table_key in keys_in_table:
@@ -173,7 +188,6 @@ class PysonicDatabase(object):
                 cursor.execute("UPDATE nodes SET metadata=? WHERE id=?;", (json.dumps(metadata), node_id, ))
 
     def get_metadata(self, node_id):
-        keys_in_table = ["title", "album", "artist", "type"]
         node = self.getnode(node_id)
         meta = node["metadata"]
         meta.update({item: node[item] for item in keys_in_table})

pysonic/library.py

@@ -108,3 +108,14 @@ class PysonicLibrary(object):
 
     def get_songs(self, limit=50, shuffle=True):
         return self.db.getnodes(types=MUSIC_TYPES, limit=limit, order="rand")
+
+    def get_song(self, id=None):
+        if id:
+            return self.db.getnode(id)
+        else:
+            return self.db.getnodes(types=MUSIC_TYPES, limit=1, order="rand")
+
+    def report_transcode(self, item_id, bitrate, num_bytes):
+        assert type(bitrate) is int and bitrate > 0 and bitrate <= 320
+        logging.info("Got transcode report of {} for item {} @ {}".format(num_bytes, item_id, bitrate))
+        self.db.update_metadata(item_id, {"transcoded_{}_size".format(bitrate):int(num_bytes)})

pysonic/scanner.py

@@ -4,10 +4,12 @@ import logging
 import mimetypes
 from time import time
 from threading import Thread
-from pysonic.types import KNOWN_MIMES, MUSIC_TYPES
+from pysonic.types import KNOWN_MIMES, MUSIC_TYPES, MPX_TYPES, FLAC_TYPES, WAV_TYPES
 from mutagen.id3 import ID3
 from mutagen import MutagenError
 from mutagen.id3._util import ID3NoHeaderError
+from mutagen.flac import FLAC
+from mutagen.mp3 import MP3
 
 
 logging = logging.getLogger("scanner")
@@ -31,7 +33,7 @@ class PysonicFilesystemScanner(object):
             logging.info("Scanning {}".format(meta["fspath"]))
 
             def recurse_dir(path, parent):
-                logging.info("Scanning {} with parent {}".format(path, parent))
+                logging.info("Scanning {}".format(path))
                 # create or update the database of nodes by comparing sets of names
                 fs_entries = set(os.listdir(path))
                 db_entires = self.library.db.getnodes(parent["id"])
@@ -39,9 +41,17 @@ class PysonicFilesystemScanner(object):
                 to_delete = db_entires_names - fs_entries
                 to_create = fs_entries - db_entires_names
 
+                # If any size have changed, mark the file to be rescanned
+                for entry in db_entires:
+                    finfo = os.stat(os.path.join(path, entry["name"]))
+                    if finfo.st_size != entry["size"]:
+                        logging.info("{} has changed in size, marking for meta rescan".format(entry["id"]))
+                        self.library.db.update_metadata(entry['id'], id3_done=False, size=finfo.st_size)
+
                 # Create any nodes not found in the db
                 for create in to_create:
-                    new_node = self.library.db.addnode(parent["id"], path, create)
+                    new_finfo = os.stat(os.path.join(path, create))
+                    new_node = self.library.db.addnode(parent["id"], path, create, size=new_finfo.st_size)
                     logging.info("Added {}".format(os.path.join(path, create)))
                     db_entires.append(new_node)
 
@@ -56,9 +66,9 @@ class PysonicFilesystemScanner(object):
                 for entry in db_entires:
                     if entry["name"] in to_delete:
                         continue
-
                     if int(entry['isdir']):  # 1 means dir
                         recurse_dir(os.path.join(path, entry["name"]), entry)
+
             # Populate all files for this top-level root
             recurse_dir(meta["fspath"], parent)
             #
@@ -100,7 +110,7 @@ class PysonicFilesystemScanner(object):
             #
             #
             #
-            # Add advanced id3 metadata
+            # Add advanced id3 / media info metadata
             for artist_dir in self.library.db.getnodes(parent["id"]):
                 artist = artist_dir["name"]
                 for album_dir in self.library.db.getnodes(artist_dir["id"]):
@@ -110,37 +120,56 @@ class PysonicFilesystemScanner(object):
                         track_meta = track_file['metadata']
                         title = track_file["name"]
                         fpath = self.library.get_filepath(track_file["id"])
-                        if track_meta.get('id3_done', False) or track_file.get("type", "x") not in MUSIC_TYPES:
+                        if track_meta.get('id3_done', False) or track_file.get("type", None) not in MUSIC_TYPES:
                             continue
-                        print("Mutagening", fpath)
                         tags = {'id3_done': True}
                         try:
-                            id3 = ID3(fpath)
-                            # print(id3.pprint())
+                            audio = None
+                            if track_file.get("type", None) in MPX_TYPES:
+                                audio = MP3(fpath)
+                                if audio.info.sketchy:
+                                    logging.warning("media reported as sketchy: %s", fpath)
+                            elif track_file.get("type", None) in FLAC_TYPES:
+                                audio = FLAC(fpath)
+                            else:
+                                audio = ID3(fpath)
+                            # print(audio.pprint())
                             try:
-                                tags["track"] = int(RE_NUMBERS.findall(''.join(id3['TRCK'].text))[0])
+                                tags["media_length"] = int(audio.info.length)
+                            except (ValueError, AttributeError):
+                                pass
+                            try:
+                                bitrate = int(audio.info.bitrate)
+                                tags["media_bitrate"] = bitrate
+                                tags["media_kbitrate"] = int(bitrate / 1024)
+                            except (ValueError, AttributeError):
+                                pass
+                            try:
+                                tags["track"] = int(RE_NUMBERS.findall(''.join(audio['TRCK'].text))[0])
                             except (KeyError, IndexError):
                                 pass
                             try:
-                                tags["id3_artist"] = ''.join(id3['TPE1'].text)
+                                tags["id3_artist"] = ''.join(audio['TPE1'].text)
                             except KeyError:
                                 pass
                             try:
-                                tags["id3_album"] = ''.join(id3['TALB'].text)
+                                tags["id3_album"] = ''.join(audio['TALB'].text)
                             except KeyError:
                                 pass
                             try:
-                                tags["id3_title"] = ''.join(id3['TIT2'].text)
+                                tags["id3_title"] = ''.join(audio['TIT2'].text)
                             except KeyError:
                                 pass
                             try:
-                                tags["id3_year"] = id3['TDRC'].text[0].year
+                                tags["id3_year"] = audio['TDRC'].text[0].year
                             except (KeyError, IndexError):
                                 pass
+                            logging.info("got all media info from %s", fpath)
                         except ID3NoHeaderError:
                             pass
                         except MutagenError as m:
-                            logging.error(m)
+                            logging.error("failed to read audio information: %s", m)
+                            continue
                         self.library.db.update_metadata(track_file["id"], **tags)
 
-            logging.warning("Library scan complete in {}s".format(int(time() - start)))
+            logging.warning("Library scan complete in {}s".format(round(time() - start, 2)))

pysonic/types.py

@@ -1,4 +1,7 @@
 
 KNOWN_MIMES = ["audio/mpeg", "audio/flac", "audio/x-wav", "image/jpeg", "image/png"]
 MUSIC_TYPES = ["audio/mpeg", "audio/flac", "audio/x-wav"]
+MPX_TYPES = ["audio/mpeg"]
+FLAC_TYPES = ["audio/flac"]
+WAV_TYPES = ["audio/x-wav"]
 IMAGE_TYPES = ["image/jpeg", "image/png"]