dave
/
pysonic
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

File size/duration awarenness

This commit is contained in:
dave 2017-08-19 22:03:09 -07:00
parent 3fff05bc28
commit 7c9b1d7869
6 changed files with 118 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pysonic/api.py
View File

@ -188,6 +188,10 @@ class PysonicApi(object):
# bitRate="320"
# path="Cosmic Gate/Sign Of The Times/03 Flatline (featuring Kyler England).mp3"
type="music")
if item["size"] != -1:
child.attrs["size"] = item["size"]
if "media_length" in item_meta:
child.attrs["duration"] = item_meta["media_length"]
if "albumId" in directory:
child.attrs["albumId"] = directory["id"]
if "artistId" in directory:
@ -212,8 +216,13 @@ class PysonicApi(object):
assert maxBitRate >= 32 and maxBitRate <= 320
fpath = self.library.get_filepath(id)
meta = self.library.get_file_metadata(id)
to_bitrate = min(maxBitRate, self.options.max_bitrate, meta.get("media_kbitrate", 320))
cherrypy.response.headers['Content-Type'] = 'audio/mpeg'
if self.options.skip_transcode and meta["type"] == "audio/mpeg":
if "media_length" in meta:
cherrypy.response.headers['X-Content-Duration'] = str(int(meta['media_length']))
cherrypy.response.headers['X-Content-Kbitrate'] = str(to_bitrate)
if (self.options.skip_transcode or meta.get("media_kbitrate", -1) == to_bitrate) \
and meta["type"] == "audio/mpeg":
def content():
with open(fpath, "rb") as f:
while True:
@ -221,26 +230,37 @@ class PysonicApi(object):
if not data:
break
yield data
return content()
else:
transcode_meta = "transcoded_{}_size".format(to_bitrate)
if transcode_meta in meta:
cherrypy.response.headers['Content-Length'] = str(int(meta[transcode_meta]))
transcode_args = ["ffmpeg", "-i", fpath, "-map", "0:0", "-b:a",
"{}k".format(min(maxBitRate, self.options.max_bitrate)),
"{}k".format(to_bitrate),
"-v", "0", "-f", "mp3", "-"]
logging.info(' '.join(transcode_args))
proc = subprocess.Popen(transcode_args, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
def content(proc):
length = 0
completed = False
start = time()
try:
while True:
data = proc.stdout.read(16 * 1024)
if not data:
completed = True
break
yield data
length += len(data)
finally:
proc.poll()
if proc.returncode is None:
if proc.returncode is None or proc.returncode == 0:
logging.warning("transcoded {} in {}s".format(id, int(time() - start)))
if completed:
self.library.report_transcode(id, to_bitrate, length)
else:
logging.error("transcode of {} exited with code {} after {}s".format(id, proc.returncode,
int(time() - start)))
@ -255,7 +275,7 @@ class PysonicApi(object):
Thread(target=stopit, args=(proc, )).start()
return content(proc)
return content(proc)
stream_view._cp_config = {'response.stream': True}
@cherrypy.expose

10
pysonic/daemon.py
View File

@ -22,7 +22,11 @@ def main():
group = parser.add_argument_group("app options")
group.add_argument("--skip-transcode", action="store_true", help="instead of trancoding mp3s, send as-is")
group.add_argument("--no-rescan", action="store_true", help="don't perform simple scan on startup")
group.add_argument("--deep-rescap", action="store_true", help="perform deep scan (read id3 etc)")
group.add_argument("--enable-prune", action="store_true", help="enable removal of media not found on disk")
group.add_argument("--max-bitrate", type=int, default=320, help="maximum send bitrate")
group.add_argument("--enable-cors", action="store_true", help="add response headers to allow cors")
args = parser.parse_args()
@ -53,6 +57,12 @@ def main():
api_config.update({'tools.auth_basic.on': True,
'tools.auth_basic.realm': 'pysonic',
'tools.auth_basic.checkpassword': db.validate_password})
if args.enable_cors:
def cors():
cherrypy.response.headers["Access-Control-Allow-Origin"] = "*"
cherrypy.tools.cors = cherrypy.Tool('before_handler', cors)
api_config.update({'tools.cors.on': True})
cherrypy.tree.mount(api, '/rest/', {'/': api_config})
cherrypy.config.update({

36
pysonic/database.py
View File

@ -3,11 +3,11 @@ import json
import sqlite3
import logging
from hashlib import sha512
from itertools import chain
from contextlib import closing
logging = logging.getLogger("database")
keys_in_table = ["title", "album", "artist", "type", "size"]
def dict_factory(cursor, row):
@ -39,18 +39,29 @@ class PysonicDatabase(object):
queries = ["""CREATE TABLE 'meta' (
'key' TEXT PRIMARY KEY NOT NULL,
'value' TEXT);""",
"""INSERT INTO meta VALUES ('db_version', '0');""",
"""INSERT INTO meta VALUES ('db_version', '3');""",
"""CREATE TABLE 'nodes' (
'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
'parent' INTEGER NOT NULL,
'isdir' BOOLEAN NOT NULL,
'size' INTEGER NOT NULL DEFAULT -1,
'name' TEXT NOT NULL,
'type' TEXT,
'title' TEXT,
'album' TEXT,
'artist' TEXT,
'metadata' TEXT
)"""]
)""",
"""CREATE TABLE 'users' (
'id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
'username' TEXT UNIQUE NOT NULL,
'password' TEXT NOT NULL,
'admin' BOOLEAN DEFAULT 0,
'email' TEXT)""",
"""CREATE TABLE 'stars' (
'userid' INTEGER,
'nodeid' INTEGER,
primary key ('userid', 'nodeid'))"""]
with closing(self.db.cursor()) as cursor:
cursor.execute("SELECT * FROM sqlite_master WHERE type='table' AND name='meta';")
@ -81,6 +92,11 @@ class PysonicDatabase(object):
primary key ('userid', 'nodeid'))"""
cursor.execute(stars_table)
version = 2
if version < 3:
logging.warning("migrating database to v3 from %s", version)
size_col = """ALTER TABLE nodes ADD 'size' INTEGER NOT NULL DEFAULT -1;"""
cursor.execute(size_col)
version = 3
cursor.execute("""UPDATE meta SET value=? WHERE key="db_version";""", (str(version), ))
logging.warning("db schema is version {}".format(version))
@ -125,7 +141,7 @@ class PysonicDatabase(object):
if types:
add_filter("type", types)
query = query.rstrip(" AND")
query = query.rstrip(" AND").rstrip("WHERE ")
if order:
query += "ORDER BY "
@ -138,15 +154,15 @@ class PysonicDatabase(object):
with closing(self.db.cursor()) as cursor:
return list(map(self._populate_meta, cursor.execute(query, qargs).fetchall()))
def addnode(self, parent_id, fspath, name):
def addnode(self, parent_id, fspath, name, size=-1):
fullpath = os.path.join(fspath, name)
is_dir = os.path.isdir(fullpath)
return self._addnode(parent_id, name, is_dir)
return self._addnode(parent_id, name, is_dir, size=size)
def _addnode(self, parent_id, name, is_dir=True):
def _addnode(self, parent_id, name, is_dir=True, size=-1):
with closing(self.db.cursor()) as cursor:
cursor.execute("INSERT INTO nodes (parent, isdir, name) VALUES (?, ?, ?);",
(parent_id, 1 if is_dir else 0, name))
cursor.execute("INSERT INTO nodes (parent, isdir, name, size) VALUES (?, ?, ?, ?);",
(parent_id, 1 if is_dir else 0, name, size))
return self.getnode(cursor.lastrowid)
def delnode(self, node_id):
@ -159,7 +175,6 @@ class PysonicDatabase(object):
def update_metadata(self, node_id, mergedict=None, **kwargs):
mergedict = mergedict if mergedict else {}
keys_in_table = ["title", "album", "artist", "type"]
mergedict.update(kwargs)
with closing(self.db.cursor()) as cursor:
for table_key in keys_in_table:
@ -173,7 +188,6 @@ class PysonicDatabase(object):
cursor.execute("UPDATE nodes SET metadata=? WHERE id=?;", (json.dumps(metadata), node_id, ))
def get_metadata(self, node_id):
keys_in_table = ["title", "album", "artist", "type"]
node = self.getnode(node_id)
meta = node["metadata"]
meta.update({item: node[item] for item in keys_in_table})

11
pysonic/library.py
View File

@ -108,3 +108,14 @@ class PysonicLibrary(object):
def get_songs(self, limit=50, shuffle=True):
return self.db.getnodes(types=MUSIC_TYPES, limit=limit, order="rand")
def get_song(self, id=None):
if id:
return self.db.getnode(id)
else:
return self.db.getnodes(types=MUSIC_TYPES, limit=1, order="rand")
def report_transcode(self, item_id, bitrate, num_bytes):
assert type(bitrate) is int and bitrate > 0 and bitrate <= 320
logging.info("Got transcode report of {} for item {} @ {}".format(num_bytes, item_id, bitrate))
self.db.update_metadata(item_id, {"transcoded_{}_size".format(bitrate):int(num_bytes)})

61
pysonic/scanner.py
View File

@ -4,10 +4,12 @@ import logging
import mimetypes
from time import time
from threading import Thread
from pysonic.types import KNOWN_MIMES, MUSIC_TYPES
from pysonic.types import KNOWN_MIMES, MUSIC_TYPES, MPX_TYPES, FLAC_TYPES, WAV_TYPES
from mutagen.id3 import ID3
from mutagen import MutagenError
from mutagen.id3._util import ID3NoHeaderError
from mutagen.flac import FLAC
from mutagen.mp3 import MP3
logging = logging.getLogger("scanner")
@ -31,7 +33,7 @@ class PysonicFilesystemScanner(object):
logging.info("Scanning {}".format(meta["fspath"]))
def recurse_dir(path, parent):
logging.info("Scanning {} with parent {}".format(path, parent))
logging.info("Scanning {}".format(path))
# create or update the database of nodes by comparing sets of names
fs_entries = set(os.listdir(path))
db_entires = self.library.db.getnodes(parent["id"])
@ -39,9 +41,17 @@ class PysonicFilesystemScanner(object):
to_delete = db_entires_names - fs_entries
to_create = fs_entries - db_entires_names
# If any size have changed, mark the file to be rescanned
for entry in db_entires:
finfo = os.stat(os.path.join(path, entry["name"]))
if finfo.st_size != entry["size"]:
logging.info("{} has changed in size, marking for meta rescan".format(entry["id"]))
self.library.db.update_metadata(entry['id'], id3_done=False, size=finfo.st_size)
# Create any nodes not found in the db
for create in to_create:
new_node = self.library.db.addnode(parent["id"], path, create)
new_finfo = os.stat(os.path.join(path, create))
new_node = self.library.db.addnode(parent["id"], path, create, size=new_finfo.st_size)
logging.info("Added {}".format(os.path.join(path, create)))
db_entires.append(new_node)
@ -56,9 +66,9 @@ class PysonicFilesystemScanner(object):
for entry in db_entires:
if entry["name"] in to_delete:
continue
if int(entry['isdir']): # 1 means dir
recurse_dir(os.path.join(path, entry["name"]), entry)
# Populate all files for this top-level root
recurse_dir(meta["fspath"], parent)
#
@ -100,7 +110,7 @@ class PysonicFilesystemScanner(object):
#
#
#
# Add advanced id3 metadata
# Add advanced id3 / media info metadata
for artist_dir in self.library.db.getnodes(parent["id"]):
artist = artist_dir["name"]
for album_dir in self.library.db.getnodes(artist_dir["id"]):
@ -110,37 +120,56 @@ class PysonicFilesystemScanner(object):
track_meta = track_file['metadata']
title = track_file["name"]
fpath = self.library.get_filepath(track_file["id"])
if track_meta.get('id3_done', False) or track_file.get("type", "x") not in MUSIC_TYPES:
if track_meta.get('id3_done', False) or track_file.get("type", None) not in MUSIC_TYPES:
continue
print("Mutagening", fpath)
tags = {'id3_done': True}
try:
id3 = ID3(fpath)
# print(id3.pprint())
audio = None
if track_file.get("type", None) in MPX_TYPES:
audio = MP3(fpath)
if audio.info.sketchy:
logging.warning("media reported as sketchy: %s", fpath)
elif track_file.get("type", None) in FLAC_TYPES:
audio = FLAC(fpath)
else:
audio = ID3(fpath)
# print(audio.pprint())
try:
tags["track"] = int(RE_NUMBERS.findall(''.join(id3['TRCK'].text))[0])
tags["media_length"] = int(audio.info.length)
except (ValueError, AttributeError):
pass
try:
bitrate = int(audio.info.bitrate)
tags["media_bitrate"] = bitrate
tags["media_kbitrate"] = int(bitrate / 1024)
except (ValueError, AttributeError):
pass
try:
tags["track"] = int(RE_NUMBERS.findall(''.join(audio['TRCK'].text))[0])
except (KeyError, IndexError):
pass
try:
tags["id3_artist"] = ''.join(id3['TPE1'].text)
tags["id3_artist"] = ''.join(audio['TPE1'].text)
except KeyError:
pass
try:
tags["id3_album"] = ''.join(id3['TALB'].text)
tags["id3_album"] = ''.join(audio['TALB'].text)
except KeyError:
pass
try:
tags["id3_title"] = ''.join(id3['TIT2'].text)
tags["id3_title"] = ''.join(audio['TIT2'].text)
except KeyError:
pass
try:
tags["id3_year"] = id3['TDRC'].text[0].year
tags["id3_year"] = audio['TDRC'].text[0].year
except (KeyError, IndexError):
pass
logging.info("got all media info from %s", fpath)
except ID3NoHeaderError:
pass
except MutagenError as m:
logging.error(m)
logging.error("failed to read audio information: %s", m)
continue
self.library.db.update_metadata(track_file["id"], **tags)
logging.warning("Library scan complete in {}s".format(int(time() - start)))
logging.warning("Library scan complete in {}s".format(round(time() - start, 2)))

3
pysonic/types.py
View File

@ -1,4 +1,7 @@
KNOWN_MIMES = ["audio/mpeg", "audio/flac", "audio/x-wav", "image/jpeg", "image/png"]
MUSIC_TYPES = ["audio/mpeg", "audio/flac", "audio/x-wav"]
MPX_TYPES = ["audio/mpeg"]
FLAC_TYPES = ["audio/flac"]
WAV_TYPES = ["audio/x-wav"]
IMAGE_TYPES = ["image/jpeg", "image/png"]