diff --git a/app.py b/app.py index a0fe4f5..b30a46a 100644 --- a/app.py +++ b/app.py @@ -15,8 +15,9 @@ from libs import database from libs import recordTick from datetime import datetime + if __name__ == '__main__' or 'uwsgi' in __name__: - appdir = "/home/streamrecord/app" + appdir = os.path.abspath(os.path.normpath(os.path.dirname(__file__))) appconf = { '/': { #'tools.proxy.on':True, @@ -27,12 +28,12 @@ if __name__ == '__main__' or 'uwsgi' in __name__: 'tools.sessions.timeout':525600, 'request.show_tracebacks': True }, - '/media': { + '/static': { 'tools.staticdir.on': True, 'tools.staticdir.dir': appdir+"/static/" } } - + cherrypy.config.update({ 'server.socket_port':3000, 'server.thread_pool':1, @@ -40,32 +41,32 @@ if __name__ == '__main__' or 'uwsgi' in __name__: 'sessionFilter.on':True, 'server.show.tracebacks': True }) - + cherrypy.server.socket_timeout = 5 - + # env - jinja2 template renderer - env = Environment(loader=FileSystemLoader("/home/streamrecord/app/templates")) + env = Environment(loader=FileSystemLoader(os.path.join(appdir, "templates"))) # db - slightly custom sqlite3 object. rows = db.execute(query, args) db = database() # REC - recorder thread - see recordTick.py #REC = recordTick(db) - + def render(template, args): templatesCache = pysite.cacheTemplates() defaults = {"templates":templatesCache} for item in args: defaults[item] = args[item] return quickRender(template, defaults) - + def quickRender(template, args): template = env.get_template(template) return template.render(args) - + class siteRoot(object): def __init__(self): print("Siteroot init !") self.templateCache = self.cacheTemplates() - + def cacheTemplates(self): templateFiles = os.listdir("jstemplates/") templateList = [] @@ -75,66 +76,66 @@ if __name__ == '__main__' or 'uwsgi' in __name__: templateList.append({"name":name[0],"content":open("jstemplates/"+item,"r").read().replace("\t", "").replace("\n","")}) nameList.append(name[0]) return quickRender("templates.html", {"names":json.dumps(nameList), "templates":templateList}) - + @cherrypy.expose def index(self): return render("html.html", {}) - + @cherrypy.expose def htmltest(self): return render("html.tpl", {}) #index.exposed = True - + @cherrypy.expose def templates(self): return self.templateCache - + class api(object): def __init__(self): self.REC = recordTick(db) - + @cherrypy.expose def tick(self): self.REC.tick() return "OK" - + @cherrypy.expose def getStreams(self): streamList = db.execute('SELECT * FROM "streams" ORDER BY "name" ASC;') - + for stream in streamList: stream["time"] = db.execute('SELECT * FROM "times" WHERE streamid=?', [stream["id"]])[0] stream["files"] = self._getFiles(stream["id"]) stream["recorder_status"] = self.REC.streamStatus(stream["id"]) stream["is_running"] = stream["recorder_status"] not in [0, -1] # idle states - + return json.dumps(streamList) - + def _getStream(self,id): streamList = db.execute('SELECT * FROM "streams" WHERE "id"=?', [int(id)]) - + for stream in streamList: stream["time"] = db.execute('SELECT * FROM "times" WHERE streamid=?', [stream["id"]])[0] stream["files"]=self._getFiles(id) stream["recorder_status"] = self.REC.streamStatus(stream["id"]) stream["is_running"] = stream["recorder_status"] not in [0, -1] # idle states return streamList[0] - + @cherrypy.expose def getStream(self, id): return json.dumps(self._getStream(id)) - + @cherrypy.expose def changeStatus(self, streamid, status): streamid = int(streamid) db.execute('UPDATE "streams" SET "status"=? WHERE "id"=? ;', [status, streamid]) return json.dumps({"result":True}) - + @cherrypy.expose def changeTimeDay(self, streamid, day, value): streamid = int(streamid) value = value == "true" - + col = "" if day == "daysu": col="su" @@ -152,11 +153,11 @@ if __name__ == '__main__' or 'uwsgi' in __name__: col="sa" else: raise cherrypy.HTTPError(500, message="Day not found") - + db.execute('UPDATE "times" SET "'+col+'"=? WHERE "streamid"=? ;', [1 if value else 0,streamid]) - + return json.dumps({"result":True}) - + @cherrypy.expose def changeName(self, streamid, value): streamid = int(streamid) @@ -177,10 +178,10 @@ if __name__ == '__main__' or 'uwsgi' in __name__: assert endHour>=0 and endHour<=23 endMin=int(endMin) assert endMin>=0 and endMin<=59 - + db.execute('UPDATE "times" SET "starthour"=?, "startmin"=?, "endhour"=?, "endmin"=? WHERE "streamid"=? ;', [startHour, startMin, endHour, endMin, streamid]) return json.dumps({"result":True}) - + def _filterName(self, input): allowed="abcdefghijklmnopqrstuvwxyz123456789-" input = input.replace(" ", "-").lower() @@ -189,17 +190,17 @@ if __name__ == '__main__' or 'uwsgi' in __name__: if input[i:i+1] in allowed: output.append(input[i:i+1]) return ''.join(output) - + @cherrypy.expose def createStream(self, data): data = json.loads(data) - + assert not data["name"] == "" assert not data["url"] == "" assert data["time"]["su"] or data["time"]["m"] or data["time"]["t"] or data["time"]["w"] or data["time"]["r"] or data["time"]["f"] or data["time"]["sa"] - + dirName = self._filterName(data["name"]) - + rowid = db.execute('INSERT INTO "streams" ("user", "name", "url", "directory", "status", "message") VALUES (?, ?, ?, ?, ?, ?);', [0, data["name"], data["url"], dirName, data["status"], ""]) db.execute('INSERT INTO "times" ("streamid", "su", "m", "t", "w", "r", "f", "sa", "starthour", "startmin", "endhour", "endmin") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', [ rowid, @@ -215,9 +216,9 @@ if __name__ == '__main__' or 'uwsgi' in __name__: data["time"]["endHour"], data["time"]["endMin"] ]) - + return json.dumps({"result":rowid}) - + def _getFiles(self, id): stream = db.execute('SELECT * FROM "streams" WHERE "id"=?', [int(id)])[0] recordingsDir = "files/output/"+stream["directory"]+"/" @@ -241,24 +242,24 @@ if __name__ == '__main__' or 'uwsgi' in __name__: "date":os.path.getctime(recordingsDir+item) }) return allFiles - + @cherrypy.expose def getFiles(self, id): files = self._getFiles(id) return json.dumps({"data":files}) - + @cherrypy.expose def download(self, id, fn): files = self._getFiles(id) item = files[int(fn)] raise cherrypy.HTTPRedirect("/static/output/"+item["streamdir"]+"/"+item["filename"], 302) - + @cherrypy.expose def getUrl(self, id, fn): files = self._getFiles(id) item = files[int(fn)] return json.dumps({"result":"/static/output/"+item["streamdir"]+"/"+item["filename"]}) - + @cherrypy.expose @cherrypy.tools.response_headers(headers=[('Content-Type', 'application/rss+xml')]) def getPodcast(self, id): @@ -270,17 +271,17 @@ if __name__ == '__main__' or 'uwsgi' in __name__: "stream":stream, "builddate": datetime.now().strftime("%a, %d %b %Y %H:%M:%S +0800")#Thu, 31 Jul 2014 07:13:48 +0000 })) - - + + @cherrypy.expose def getRecStatus(self, id): return json.dumps({"data":self.REC.streamStatus(int(id))}) - + pysite = siteRoot() pysite.api = api() - + print( "Ready to start application" ) - + if(len(sys.argv)>1 and sys.argv[1]=="test"): print("test!") application = cherrypy.quickstart(pysite, '/', appconf) diff --git a/libs/database.py b/libs/database.py index fcc2568..ea179ff 100755 --- a/libs/database.py +++ b/libs/database.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 import sqlite3 import os.path +from contextlib import closing + class database: def __init__(self): self.createDatabase() self.db = self.openDB() - + def createDatabase(self): queries = [ "CREATE TABLE 'streams' ('id' INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 'user' INTEGER, 'name' TEXT, 'url' TEXT, 'directory' TEXT, 'status' INTEGER, 'message' TEXT);", @@ -21,27 +23,27 @@ class database: print("Executing: %s" % query) cursor.execute(query) db.close() - + def openDB(self): db = sqlite3.connect("db.sqlite", check_same_thread=False, cached_statements=0, isolation_level=None) db.row_factory = self.dict_factory return db - + def dict_factory(self, cursor, row): d = {} for idx, col in enumerate(cursor.description): d[col[0]] = row[idx] return d - + def execute(self, sql, params=None): db = self.db - cursor = db.cursor() - if params: - cursor.execute(sql, params) - else: - cursor.execute(sql) - data = cursor.fetchall() - if not cursor.lastrowid==None: - return cursor.lastrowid - cursor.close() - return data \ No newline at end of file + with closing(db.cursor()) as cursor: + if params: + cursor.execute(sql, params) + else: + cursor.execute(sql) + data = cursor.fetchall() + if sql.lower().startswith("insert "): + return cursor.lastrowid + cursor.close() + return data \ No newline at end of file diff --git a/libs/recordTick.py b/libs/recordTick.py index 6a987df..fe778c9 100644 --- a/libs/recordTick.py +++ b/libs/recordTick.py @@ -16,52 +16,52 @@ class recordTick: self.db = database # list of downloader threads self.threads = {} - + def tick(self): now=datetime.datetime.now() #print("Tick start: %s" % now) - + # Look for starting times set to now days = ["m", "t", "w", "r", "f", "sa", "su"] day = days[datetime.datetime.now().weekday()] - + startTimes = self.db.execute('SELECT * FROM "times" JOIN "streams" ON "streams"."id" = "times"."streamid" where "starthour"=? AND "startmin"=? AND "'+day+'"=1 AND "status"=0', (now.hour, now.minute)) for startTime in startTimes: # Start each downloader self.startStream(startTime["streamid"]) - + # Look for end times set to now endTimes = self.db.execute('SELECT * FROM "times" where "endhour"=? AND "endmin"=?', (now.hour, now.minute)) for endTime in endTimes: # terminate each downloader self.endStream(endTime["streamid"]) - + #print("Tick end: %s" % now) - + def startStream(self, id): # Find stream information stream = self.db.execute('SELECT * FROM "streams" WHERE "id"=? ;', (id,))[0] - + # if the downloader isnt running already: if not stream["id"] in self.threads: # Create the recording thread self.threads[stream["id"]] = recordThread(stream["url"], stream["directory"]) - + def endStream(self, id): if id in self.threads: # tell the downloader to finish self.threads[id].cancel() del self.threads[id] - + def streamStatus(self, id): if not id in self.threads: return -1 - + return self.threads[id].status - + def getSelf(self): return self - + def timeToNextMinute(self): # calculate time to the milliscond until the next minute rolls over # Find the next minute @@ -86,13 +86,14 @@ class recordThread(Thread): self.running = True # Start time of the recording self.startdate = None - + self.start() - + def run(self): print("%s starting downloader for %s" % (datetime.datetime.now(), self.url)) # Download the stream to temp file(s) self.status = 2 + self.clearTempDir() self.downloadStream() # Combine files into 1 audio file self.status = 3 @@ -105,33 +106,30 @@ class recordThread(Thread): self.cleanup() print("%s finished downloader for %s" % (datetime.datetime.now(), self.url)) self.status = 0 - + + def clearTempDir(self): + # Delete temp files + files = os.listdir("files/temp/%s"%self.directory) + for f in files: + os.unlink("files/temp/%s/%s"%(self.directory,f)) + def downloadStream(self): self.startdate = datetime.datetime.now() + recdate = str(int(time.time())) # As long as we're supposed to keep retrying while self.running: # Create the temp dir for this stream - if not os.path.exists("files/temp/"+self.directory): - os.mkdir("files/temp/"+self.directory) - + os.makedirs("files/temp/"+self.directory, exist_ok=True) + # If there are already files, we're resuming. take the next available number recNum = 0 - while os.path.exists("files/temp/%s/recdate%s.mp3" % (self.directory, ".%s"%recNum)): + while os.path.exists("files/temp/%s/%s_%s.mp3" % (self.directory, recdate, recNum)): recNum = recNum + 1 # Filename is something like files/temp/stream-name/rec-y-m-d_h-m-s.0.mp3 - fileName = "files/temp/%s/recdate%s.mp3" % (self.directory, "" if recNum == None else ".%s"%recNum) - - # Args if we download with curl (bad) - args_curl = [ - '/usr/bin/curl', - #'-s', - '-A', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36', - '--output', fileName, self.url] - - # args if we download/transcode with avconv (HERO!) + fileName = "files/temp/%s/%s_%s.mp3" % (self.directory, recdate, recNum) + args_libav = [ - '/usr/bin/avconv', + os.environ.get("CONVERT_PROGRAM") or "/usr/bin/ffmpeg", '-loglevel', 'error', '-i', @@ -140,12 +138,12 @@ class recordThread(Thread): '128k', fileName ] + self.proc = subprocess.Popen(args_libav, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = self.proc.communicate() print("LibAV output for %s:\n%s" % (self.url, output)) self.proc = None - - + def mergeStream(self): # Get an ordered list of the piece files files = os.listdir("files/temp/%s"%self.directory) @@ -154,37 +152,40 @@ class recordThread(Thread): command = ['/usr/bin/mkvmerge', '-o', "files/temp/%s/temp.mka"%self.directory, "files/temp/%s/%s"%(self.directory,files.pop(0))] for fname in files: command.append("+files/temp/%s/%s"%(self.directory,fname)) + self.mergeproc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Wait for the merge to finish output = self.mergeproc.communicate() - + def transcodeStream(self): # Delete the existing output file if os.path.exists("files/temp/%s/out.mp3"%self.directory): os.unlink("files/temp/%s/out.mp3"%self.directory) - + # Convert the matroska file to mp3 - command = ['/usr/bin/avconv', '-i', "files/temp/%s/temp.mka"%self.directory, '-q:a', '0', '-ab', '128k', "files/temp/%s/out.mp3"%self.directory] + command = [ + os.environ.get("CONVERT_PROGRAM") or '/usr/bin/ffmpeg', + '-i', "files/temp/%s/temp.mka"%self.directory, + '-q:a', '0', + '-ab', '128k', + "files/temp/%s/out.mp3"%self.directory + ] self.transcodeproc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # wait for the trancode to finish output = self.transcodeproc.communicate() - + def cleanup(self): # create a dated name for the file newname = self.startdate.strftime("%Y-%m-%d_%H-%M-%S")+".mp3" - + # make it's finished storage location - if not os.path.exists("files/output/"+self.directory): - os.mkdir("files/output/"+self.directory) - + os.makedirs("files/output/"+self.directory, exist_ok=True) + # copy final recording to output dir os.rename("files/temp/%s/out.mp3"%(self.directory), "files/output/%s/%s"%(self.directory,newname)) - - # Delete temp files - files = os.listdir("files/temp/%s"%self.directory) - for f in files: - os.unlink("files/temp/%s/%s"%(self.directory,f)) - + + self.clearTempDir() + def cancel(self): print("Closing %s" % self.url) # turn off keep-alive dow the downloader @@ -192,7 +193,7 @@ class recordThread(Thread): # Kill the download process self.proc.terminate() Thread(target=self.kill).start() - + def kill(self): print("Starting kill thread for %s" % self.url) time.sleep(3) @@ -206,4 +207,3 @@ class recordThread(Thread): if not self.proc == None: # Kill it self.proc.kill() - \ No newline at end of file diff --git a/static/js/ui.js b/static/js/ui.js index 3aa4a4d..09fcd4a 100644 --- a/static/js/ui.js +++ b/static/js/ui.js @@ -10,7 +10,7 @@ var ui = { modal.find(".modal-body").html(content) modal.modal() }, - + __streamstatus:function() { $(".btn-group-status button").click(function() { $(this).parent().find(".active").removeClass("active") @@ -124,8 +124,8 @@ var validators = { startMin = parseInt(startMin) endHour = parseInt(endHour) endMin = parseInt(endMin) - - + + if(startHour<0 || startHour>=24 || isNaN(startHour)) { messages.push("Start hour must be 0-23") } @@ -135,17 +135,17 @@ var validators = { if(endHour<0 || endHour>=24 || isNaN(endHour)) { messages.push("End hour must be 0-23") } - if(endMin<0 || endMin>50 || isNaN(endMin)) { + if(endMin<0 || endMin>59 || isNaN(endMin)) { messages.push("End minute must be 0-59") } - + } catch(err) { messages.push("Time values must be numeric") } - - - - + + + + if(messages.length==0) { return true }