File size/duration awarenness

This commit is contained in:
dave 2017-08-19 22:03:09 -07:00
parent 3fff05bc28
commit 7c9b1d7869
6 changed files with 118 additions and 31 deletions

View File

@ -188,6 +188,10 @@ class PysonicApi(object):
# bitRate="320" # bitRate="320"
# path="Cosmic Gate/Sign Of The Times/03 Flatline (featuring Kyler England).mp3" # path="Cosmic Gate/Sign Of The Times/03 Flatline (featuring Kyler England).mp3"
type="music") type="music")
if item["size"] != -1:
child.attrs["size"] = item["size"]
if "media_length" in item_meta:
child.attrs["duration"] = item_meta["media_length"]
if "albumId" in directory: if "albumId" in directory:
child.attrs["albumId"] = directory["id"] child.attrs["albumId"] = directory["id"]
if "artistId" in directory: if "artistId" in directory:
@ -212,8 +216,13 @@ class PysonicApi(object):
assert maxBitRate >= 32 and maxBitRate <= 320 assert maxBitRate >= 32 and maxBitRate <= 320
fpath = self.library.get_filepath(id) fpath = self.library.get_filepath(id)
meta = self.library.get_file_metadata(id) meta = self.library.get_file_metadata(id)
to_bitrate = min(maxBitRate, self.options.max_bitrate, meta.get("media_kbitrate", 320))
cherrypy.response.headers['Content-Type'] = 'audio/mpeg' cherrypy.response.headers['Content-Type'] = 'audio/mpeg'
if self.options.skip_transcode and meta["type"] == "audio/mpeg": if "media_length" in meta:
cherrypy.response.headers['X-Content-Duration'] = str(int(meta['media_length']))
cherrypy.response.headers['X-Content-Kbitrate'] = str(to_bitrate)
if (self.options.skip_transcode or meta.get("media_kbitrate", -1) == to_bitrate) \
and meta["type"] == "audio/mpeg":
def content(): def content():
with open(fpath, "rb") as f: with open(fpath, "rb") as f:
while True: while True:
@ -221,26 +230,37 @@ class PysonicApi(object):
if not data: if not data:
break break
yield data yield data
return content()
else: else:
transcode_meta = "transcoded_{}_size".format(to_bitrate)
if transcode_meta in meta:
cherrypy.response.headers['Content-Length'] = str(int(meta[transcode_meta]))
transcode_args = ["ffmpeg", "-i", fpath, "-map", "0:0", "-b:a", transcode_args = ["ffmpeg", "-i", fpath, "-map", "0:0", "-b:a",
"{}k".format(min(maxBitRate, self.options.max_bitrate)), "{}k".format(to_bitrate),
"-v", "0", "-f", "mp3", "-"] "-v", "0", "-f", "mp3", "-"]
logging.info(' '.join(transcode_args)) logging.info(' '.join(transcode_args))
proc = subprocess.Popen(transcode_args, stdin=subprocess.PIPE, proc = subprocess.Popen(transcode_args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def content(proc): def content(proc):
length = 0
completed = False
start = time() start = time()
try: try:
while True: while True:
data = proc.stdout.read(16 * 1024) data = proc.stdout.read(16 * 1024)
if not data: if not data:
completed = True
break break
yield data yield data
length += len(data)
finally: finally:
proc.poll() proc.poll()
if proc.returncode is None: if proc.returncode is None or proc.returncode == 0:
logging.warning("transcoded {} in {}s".format(id, int(time() - start))) logging.warning("transcoded {} in {}s".format(id, int(time() - start)))
if completed:
self.library.report_transcode(id, to_bitrate, length)
else: else:
logging.error("transcode of {} exited with code {} after {}s".format(id, proc.returncode, logging.error("transcode of {} exited with code {} after {}s".format(id, proc.returncode,
int(time() - start))) int(time() - start)))
@ -255,7 +275,7 @@ class PysonicApi(object):
Thread(target=stopit, args=(proc, )).start() Thread(target=stopit, args=(proc, )).start()
return content(proc) return content(proc)
stream_view._cp_config = {'response.stream': True} stream_view._cp_config = {'response.stream': True}
@cherrypy.expose @cherrypy.expose

View File

@ -22,7 +22,11 @@ def main():
group = parser.add_argument_group("app options") group = parser.add_argument_group("app options")
group.add_argument("--skip-transcode", action="store_true", help="instead of trancoding mp3s, send as-is") group.add_argument("--skip-transcode", action="store_true", help="instead of trancoding mp3s, send as-is")
group.add_argument("--no-rescan", action="store_true", help="don't perform simple scan on startup")
group.add_argument("--deep-rescap", action="store_true", help="perform deep scan (read id3 etc)")
group.add_argument("--enable-prune", action="store_true", help="enable removal of media not found on disk")
group.add_argument("--max-bitrate", type=int, default=320, help="maximum send bitrate") group.add_argument("--max-bitrate", type=int, default=320, help="maximum send bitrate")
group.add_argument("--enable-cors", action="store_true", help="add response headers to allow cors")
args = parser.parse_args() args = parser.parse_args()
@ -53,6 +57,12 @@ def main():
api_config.update({'tools.auth_basic.on': True, api_config.update({'tools.auth_basic.on': True,
'tools.auth_basic.realm': 'pysonic', 'tools.auth_basic.realm': 'pysonic',
'tools.auth_basic.checkpassword': db.validate_password}) 'tools.auth_basic.checkpassword': db.validate_password})
if args.enable_cors:
def cors():
cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
cherrypy.tools.cors = cherrypy.Tool('before_handler', cors)
api_config.update({'tools.cors.on': True})
cherrypy.tree.mount(api, '/rest/', {'/': api_config}) cherrypy.tree.mount(api, '/rest/', {'/': api_config})
cherrypy.config.update({ cherrypy.config.update({

View File

@ -3,11 +3,11 @@ import json
import sqlite3 import sqlite3
import logging import logging
from hashlib import sha512 from hashlib import sha512
from itertools import chain
from contextlib import closing from contextlib import closing
logging = logging.getLogger("database") logging = logging.getLogger("database")
keys_in_table = ["title", "album", "artist", "type", "size"]
def dict_factory(cursor, row): def dict_factory(cursor, row):
@ -39,18 +39,29 @@ class PysonicDatabase(object):
queries = ["""CREATE TABLE 'meta' ( queries = ["""CREATE TABLE 'meta' (
'key' TEXT PRIMARY KEY NOT NULL, 'key' TEXT PRIMARY KEY NOT NULL,
'value' TEXT);""", 'value' TEXT);""",
"""INSERT INTO meta VALUES ('db_version', '0');""", """INSERT INTO meta VALUES ('db_version', '3');""",
"""CREATE TABLE 'nodes' ( """CREATE TABLE 'nodes' (
'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
'parent' INTEGER NOT NULL, 'parent' INTEGER NOT NULL,
'isdir' BOOLEAN NOT NULL, 'isdir' BOOLEAN NOT NULL,
'size' INTEGER NOT NULL DEFAULT -1,
'name' TEXT NOT NULL, 'name' TEXT NOT NULL,
'type' TEXT, 'type' TEXT,
'title' TEXT, 'title' TEXT,
'album' TEXT, 'album' TEXT,
'artist' TEXT, 'artist' TEXT,
'metadata' TEXT 'metadata' TEXT
)"""] )""",
"""CREATE TABLE 'users' (
'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
'username' TEXT UNIQUE NOT NULL,
'password' TEXT NOT NULL,
'admin' BOOLEAN DEFAULT 0,
'email' TEXT)""",
"""CREATE TABLE 'stars' (
'userid' INTEGER,
'nodeid' INTEGER,
primary key ('userid', 'nodeid'))"""]
with closing(self.db.cursor()) as cursor: with closing(self.db.cursor()) as cursor:
cursor.execute("SELECT * FROM sqlite_master WHERE type='table' AND name='meta';") cursor.execute("SELECT * FROM sqlite_master WHERE type='table' AND name='meta';")
@ -81,6 +92,11 @@ class PysonicDatabase(object):
primary key ('userid', 'nodeid'))""" primary key ('userid', 'nodeid'))"""
cursor.execute(stars_table) cursor.execute(stars_table)
version = 2 version = 2
if version < 3:
logging.warning("migrating database to v3 from %s", version)
size_col = """ALTER TABLE nodes ADD 'size' INTEGER NOT NULL DEFAULT -1;"""
cursor.execute(size_col)
version = 3
cursor.execute("""UPDATE meta SET value=? WHERE key="db_version";""", (str(version), )) cursor.execute("""UPDATE meta SET value=? WHERE key="db_version";""", (str(version), ))
logging.warning("db schema is version {}".format(version)) logging.warning("db schema is version {}".format(version))
@ -125,7 +141,7 @@ class PysonicDatabase(object):
if types: if types:
add_filter("type", types) add_filter("type", types)
query = query.rstrip(" AND") query = query.rstrip(" AND").rstrip("WHERE ")
if order: if order:
query += "ORDER BY " query += "ORDER BY "
@ -138,15 +154,15 @@ class PysonicDatabase(object):
with closing(self.db.cursor()) as cursor: with closing(self.db.cursor()) as cursor:
return list(map(self._populate_meta, cursor.execute(query, qargs).fetchall())) return list(map(self._populate_meta, cursor.execute(query, qargs).fetchall()))
def addnode(self, parent_id, fspath, name): def addnode(self, parent_id, fspath, name, size=-1):
fullpath = os.path.join(fspath, name) fullpath = os.path.join(fspath, name)
is_dir = os.path.isdir(fullpath) is_dir = os.path.isdir(fullpath)
return self._addnode(parent_id, name, is_dir) return self._addnode(parent_id, name, is_dir, size=size)
def _addnode(self, parent_id, name, is_dir=True): def _addnode(self, parent_id, name, is_dir=True, size=-1):
with closing(self.db.cursor()) as cursor: with closing(self.db.cursor()) as cursor:
cursor.execute("INSERT INTO nodes (parent, isdir, name) VALUES (?, ?, ?);", cursor.execute("INSERT INTO nodes (parent, isdir, name, size) VALUES (?, ?, ?, ?);",
(parent_id, 1 if is_dir else 0, name)) (parent_id, 1 if is_dir else 0, name, size))
return self.getnode(cursor.lastrowid) return self.getnode(cursor.lastrowid)
def delnode(self, node_id): def delnode(self, node_id):
@ -159,7 +175,6 @@ class PysonicDatabase(object):
def update_metadata(self, node_id, mergedict=None, **kwargs): def update_metadata(self, node_id, mergedict=None, **kwargs):
mergedict = mergedict if mergedict else {} mergedict = mergedict if mergedict else {}
keys_in_table = ["title", "album", "artist", "type"]
mergedict.update(kwargs) mergedict.update(kwargs)
with closing(self.db.cursor()) as cursor: with closing(self.db.cursor()) as cursor:
for table_key in keys_in_table: for table_key in keys_in_table:
@ -173,7 +188,6 @@ class PysonicDatabase(object):
cursor.execute("UPDATE nodes SET metadata=? WHERE id=?;", (json.dumps(metadata), node_id, )) cursor.execute("UPDATE nodes SET metadata=? WHERE id=?;", (json.dumps(metadata), node_id, ))
def get_metadata(self, node_id): def get_metadata(self, node_id):
keys_in_table = ["title", "album", "artist", "type"]
node = self.getnode(node_id) node = self.getnode(node_id)
meta = node["metadata"] meta = node["metadata"]
meta.update({item: node[item] for item in keys_in_table}) meta.update({item: node[item] for item in keys_in_table})

View File

@ -108,3 +108,14 @@ class PysonicLibrary(object):
def get_songs(self, limit=50, shuffle=True): def get_songs(self, limit=50, shuffle=True):
return self.db.getnodes(types=MUSIC_TYPES, limit=limit, order="rand") return self.db.getnodes(types=MUSIC_TYPES, limit=limit, order="rand")
def get_song(self, id=None):
if id:
return self.db.getnode(id)
else:
return self.db.getnodes(types=MUSIC_TYPES, limit=1, order="rand")
def report_transcode(self, item_id, bitrate, num_bytes):
assert type(bitrate) is int and bitrate > 0 and bitrate <= 320
logging.info("Got transcode report of {} for item {} @ {}".format(num_bytes, item_id, bitrate))
self.db.update_metadata(item_id, {"transcoded_{}_size".format(bitrate):int(num_bytes)})

View File

@ -4,10 +4,12 @@ import logging
import mimetypes import mimetypes
from time import time from time import time
from threading import Thread from threading import Thread
from pysonic.types import KNOWN_MIMES, MUSIC_TYPES from pysonic.types import KNOWN_MIMES, MUSIC_TYPES, MPX_TYPES, FLAC_TYPES, WAV_TYPES
from mutagen.id3 import ID3 from mutagen.id3 import ID3
from mutagen import MutagenError from mutagen import MutagenError
from mutagen.id3._util import ID3NoHeaderError from mutagen.id3._util import ID3NoHeaderError
from mutagen.flac import FLAC
from mutagen.mp3 import MP3
logging = logging.getLogger("scanner") logging = logging.getLogger("scanner")
@ -31,7 +33,7 @@ class PysonicFilesystemScanner(object):
logging.info("Scanning {}".format(meta["fspath"])) logging.info("Scanning {}".format(meta["fspath"]))
def recurse_dir(path, parent): def recurse_dir(path, parent):
logging.info("Scanning {} with parent {}".format(path, parent)) logging.info("Scanning {}".format(path))
# create or update the database of nodes by comparing sets of names # create or update the database of nodes by comparing sets of names
fs_entries = set(os.listdir(path)) fs_entries = set(os.listdir(path))
db_entires = self.library.db.getnodes(parent["id"]) db_entires = self.library.db.getnodes(parent["id"])
@ -39,9 +41,17 @@ class PysonicFilesystemScanner(object):
to_delete = db_entires_names - fs_entries to_delete = db_entires_names - fs_entries
to_create = fs_entries - db_entires_names to_create = fs_entries - db_entires_names
# If any size have changed, mark the file to be rescanned
for entry in db_entires:
finfo = os.stat(os.path.join(path, entry["name"]))
if finfo.st_size != entry["size"]:
logging.info("{} has changed in size, marking for meta rescan".format(entry["id"]))
self.library.db.update_metadata(entry['id'], id3_done=False, size=finfo.st_size)
# Create any nodes not found in the db # Create any nodes not found in the db
for create in to_create: for create in to_create:
new_node = self.library.db.addnode(parent["id"], path, create) new_finfo = os.stat(os.path.join(path, create))
new_node = self.library.db.addnode(parent["id"], path, create, size=new_finfo.st_size)
logging.info("Added {}".format(os.path.join(path, create))) logging.info("Added {}".format(os.path.join(path, create)))
db_entires.append(new_node) db_entires.append(new_node)
@ -56,9 +66,9 @@ class PysonicFilesystemScanner(object):
for entry in db_entires: for entry in db_entires:
if entry["name"] in to_delete: if entry["name"] in to_delete:
continue continue
if int(entry['isdir']): # 1 means dir if int(entry['isdir']): # 1 means dir
recurse_dir(os.path.join(path, entry["name"]), entry) recurse_dir(os.path.join(path, entry["name"]), entry)
# Populate all files for this top-level root # Populate all files for this top-level root
recurse_dir(meta["fspath"], parent) recurse_dir(meta["fspath"], parent)
# #
@ -100,7 +110,7 @@ class PysonicFilesystemScanner(object):
# #
# #
# #
# Add advanced id3 metadata # Add advanced id3 / media info metadata
for artist_dir in self.library.db.getnodes(parent["id"]): for artist_dir in self.library.db.getnodes(parent["id"]):
artist = artist_dir["name"] artist = artist_dir["name"]
for album_dir in self.library.db.getnodes(artist_dir["id"]): for album_dir in self.library.db.getnodes(artist_dir["id"]):
@ -110,37 +120,56 @@ class PysonicFilesystemScanner(object):
track_meta = track_file['metadata'] track_meta = track_file['metadata']
title = track_file["name"] title = track_file["name"]
fpath = self.library.get_filepath(track_file["id"]) fpath = self.library.get_filepath(track_file["id"])
if track_meta.get('id3_done', False) or track_file.get("type", "x") not in MUSIC_TYPES: if track_meta.get('id3_done', False) or track_file.get("type", None) not in MUSIC_TYPES:
continue continue
print("Mutagening", fpath)
tags = {'id3_done': True} tags = {'id3_done': True}
try: try:
id3 = ID3(fpath) audio = None
# print(id3.pprint()) if track_file.get("type", None) in MPX_TYPES:
audio = MP3(fpath)
if audio.info.sketchy:
logging.warning("media reported as sketchy: %s", fpath)
elif track_file.get("type", None) in FLAC_TYPES:
audio = FLAC(fpath)
else:
audio = ID3(fpath)
# print(audio.pprint())
try: try:
tags["track"] = int(RE_NUMBERS.findall(''.join(id3['TRCK'].text))[0]) tags["media_length"] = int(audio.info.length)
except (ValueError, AttributeError):
pass
try:
bitrate = int(audio.info.bitrate)
tags["media_bitrate"] = bitrate
tags["media_kbitrate"] = int(bitrate / 1024)
except (ValueError, AttributeError):
pass
try:
tags["track"] = int(RE_NUMBERS.findall(''.join(audio['TRCK'].text))[0])
except (KeyError, IndexError): except (KeyError, IndexError):
pass pass
try: try:
tags["id3_artist"] = ''.join(id3['TPE1'].text) tags["id3_artist"] = ''.join(audio['TPE1'].text)
except KeyError: except KeyError:
pass pass
try: try:
tags["id3_album"] = ''.join(id3['TALB'].text) tags["id3_album"] = ''.join(audio['TALB'].text)
except KeyError: except KeyError:
pass pass
try: try:
tags["id3_title"] = ''.join(id3['TIT2'].text) tags["id3_title"] = ''.join(audio['TIT2'].text)
except KeyError: except KeyError:
pass pass
try: try:
tags["id3_year"] = id3['TDRC'].text[0].year tags["id3_year"] = audio['TDRC'].text[0].year
except (KeyError, IndexError): except (KeyError, IndexError):
pass pass
logging.info("got all media info from %s", fpath)
except ID3NoHeaderError: except ID3NoHeaderError:
pass pass
except MutagenError as m: except MutagenError as m:
logging.error(m) logging.error("failed to read audio information: %s", m)
continue
self.library.db.update_metadata(track_file["id"], **tags) self.library.db.update_metadata(track_file["id"], **tags)
logging.warning("Library scan complete in {}s".format(int(time() - start))) logging.warning("Library scan complete in {}s".format(round(time() - start, 2)))

View File

@ -1,4 +1,7 @@
KNOWN_MIMES = ["audio/mpeg", "audio/flac", "audio/x-wav", "image/jpeg", "image/png"] KNOWN_MIMES = ["audio/mpeg", "audio/flac", "audio/x-wav", "image/jpeg", "image/png"]
MUSIC_TYPES = ["audio/mpeg", "audio/flac", "audio/x-wav"] MUSIC_TYPES = ["audio/mpeg", "audio/flac", "audio/x-wav"]
MPX_TYPES = ["audio/mpeg"]
FLAC_TYPES = ["audio/flac"]
WAV_TYPES = ["audio/x-wav"]
IMAGE_TYPES = ["image/jpeg", "image/png"] IMAGE_TYPES = ["image/jpeg", "image/png"]