From 636a386ec421b18320b7e4d6c3ebe7764bae23cb Mon Sep 17 00:00:00 2001 From: dave Date: Wed, 18 Aug 2021 22:23:43 -0700 Subject: [PATCH] thumb server base work --- photoapp/dbutils.py | 3 +- photoapp/thumbserver.py | 223 ++++++++++++++++++++++++++++++++++++++++ photoapp/users.py | 4 +- setup.py | 1 + 4 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 photoapp/thumbserver.py diff --git a/photoapp/dbutils.py b/photoapp/dbutils.py index cd8d497..c574864 100644 --- a/photoapp/dbutils.py +++ b/photoapp/dbutils.py @@ -28,8 +28,7 @@ def get_db_engine(uri, debug=False): return engine -def get_db_session(uri): - engine = get_db_engine(uri) +def create_db_sessionmaker(engine): session = sessionmaker() session.configure(bind=engine) return session diff --git a/photoapp/thumbserver.py b/photoapp/thumbserver.py new file mode 100644 index 0000000..8c2b0ca --- /dev/null +++ b/photoapp/thumbserver.py @@ -0,0 +1,223 @@ +import os +import logging +import cherrypy +import shutil +import tempfile +import traceback +from threading import Thread +from contextlib import closing +from queue import Queue, Empty +from subprocess import check_call +from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine, create_db_sessionmaker +from photoapp.storage import uri_to_storage +from photoapp.api import PhotosApi, LibraryManager +from photoapp.types import User, Photo +from photoapp.utils import require_auth +from photoapp.common import pwhash +from photoapp.dbsession import DatabaseSession +from photoapp.thumb import ThumbGenerator + + +def get_video_thumb(srcpath, outpath): + #TODO + # download the video file + # use `ffmpeg -i filename.flv` to inspect the video, parse output for duration + # take a thumb halfway through the video + + cmd = [ + "ffmpeg", + "-i", srcpath, + "-vframes", "1", # Output one frame + "-an", # Disable audio + # "-s", "400x222" # Output size + "-ss", "1", # grab the frame from 1 second into the video + outpath + ] + + check_call(cmd) + + +def setup_thumb_user(engine): + #TODO create the internal User used to talk to this service + # if user doesnt exist + # create + # log the password + pass + + +class ThumbWorker(Thread): + def __init__(self, engine, library, cache, thumbtool): + super().__init__() + self.daemon = True + + self.queue = Queue() + self.engine = engine + self.library = library + self.cache = cache + self.thumbtool = thumbtool + + def run(self): + while True: + try: + image_uuid, style_name = self.queue.get(block=True, timeout=5.0) + except Empty: + continue + + try: + with ( + closing(create_db_sessionmaker(self.engine)()) as s, + tempfile.TemporaryDirectory() as d, + ): + self.do_thumb(image_uuid, style_name, s, d) + except: + traceback.print_exc() #TODO something like _failed_thumbs_cache + + def do_thumb(self, image_uuid, style_name, session, tmpdir): + """ + Generate a thumbnail for the given image identified by uuid + """ + + # find the image + image = session.query(Photo).filter(Photo.uuid == image_uuid).first() + if not image: + logging.info("attempted invalid uuid: %s", image_uuid) + return + + # download the image + local_src_path = os.path.join(tmpdir, image.fname) # TODO fname isn't sanitized? + thumb_path = os.path.join(tmpdir, "thumb.jpg") + with ( + self.library.open(image.path, "rb") as src, + open(local_src_path, "wb") as dest, + ): + shutil.copyfileobj(src, dest) + + # generate a still from the image + get_video_thumb(local_src_path, thumb_path) + + # copy thumbnail to cache storage + + # TODO write thumb.jpg to thumb storage + os.system(f"cp {thumb_path} /Users/dave/Desktop/") + + +class ThumbServiceWeb(object): + def __init__(self, queue_thumbnail): + self.queue_thumbnail = queue_thumbnail + + @cherrypy.expose + def index(self): + yield "photoapp thumbnail service OK" + + @cherrypy.expose + # @require_auth + def thumb(self, uuid, style): + """ + Generate a thumbnail for the file identified. Calling this endpoint adds the image to the queue. Duplicate + requests are OK and are ignored later + """ + self.queue_thumbnail((uuid, style, )) + yield "ok" + + +# TODO dedupe me +def validate_password(realm, username, password): + if db.query(User).filter(User.name == username, User.password == pwhash(password)).first(): + return True + return False + + +def main(): + # this is a slimmed down version of daemon.py TODO dedupe me + import argparse + import signal + + parser = argparse.ArgumentParser(description="Photod photo server") + + parser.add_argument('-p', '--port', help="tcp port to listen on", + default=int(os.environ.get("THUMB_SERVICE_PORT", 8081)), type=int) + parser.add_argument('-l', '--library', default=os.environ.get("STORAGE_URL"), help="library path") + parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_URL"), help="cache url") + # https://docs.sqlalchemy.org/en/13/core/engines.html + parser.add_argument('-s', '--database', help="sqlalchemy database connection uri", + default=os.environ.get("DATABASE_URL")), + parser.add_argument('--debug', action="store_true", help="enable development options") + + args = parser.parse_args() + + if not args.database: + parser.error("--database or DATABASE_URL is required") + + if not args.library: + parser.error("--library or STORAGE_URL is required") + + if not args.cache: + parser.error("--cache or CACHE_URL is required") + + logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, + format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") + + # Get database connection + engine = get_db_engine(args.database) + + setup_thumb_user(engine) + + # Setup database in web framework + cherrypy.tools.db = SATool() + SAEnginePlugin(cherrypy.engine, engine).subscribe() + + # Create various internal tools + library_storage = uri_to_storage(args.library) + library_manager = LibraryManager(library_storage) + cache_storage = uri_to_storage(args.cache) + thumbnail_tool = ThumbGenerator(library_manager, cache_storage) + thumbnail_worker = ThumbWorker(engine, library_storage, cache_storage, thumbnail_tool) + thumbnail_worker.start() + + # Setup and mount web ui + web = ThumbServiceWeb(thumbnail_worker.queue.put) + cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False, + 'tools.db.on': True, }}) + + # Setup and mount API + api = PhotosApi(library_manager) + cherrypy.tree.mount(api, '/api', {'/': {'tools.sessions.on': False, + 'tools.trailing_slash.on': False, + 'tools.auth_basic.on': True, + 'tools.auth_basic.realm': 'photolib', + 'tools.auth_basic.checkpassword': validate_password, + 'tools.db.on': True}}) + + # General config options + cherrypy.config.update({ + 'tools.sessions.storage_class': DatabaseSession, + 'tools.sessions.on': True, + 'tools.sessions.locking': 'explicit', + 'tools.sessions.timeout': 525600, + 'request.show_tracebacks': True, + 'server.socket_port': args.port, + 'server.thread_pool': 5, + 'server.socket_host': '0.0.0.0', + 'server.show_tracebacks': True, + 'log.screen': False, + 'engine.autoreload.on': args.debug, + }) + + # Setup signal handling and run it. + def signal_handler(signum, stack): + logging.critical('Got sig {}, exiting...'.format(signum)) + cherrypy.engine.exit() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + cherrypy.engine.start() + cherrypy.engine.block() + finally: + logging.info("API has shut down") + cherrypy.engine.exit() + + +if __name__ == '__main__': + main() diff --git a/photoapp/users.py b/photoapp/users.py index 02ba21c..78fd63b 100644 --- a/photoapp/users.py +++ b/photoapp/users.py @@ -2,7 +2,7 @@ import os import argparse from photoapp.types import User from photoapp.common import pwhash -from photoapp.dbutils import get_db_session +from photoapp.dbutils import create_db_sessionmaker, get_db_engine from tabulate import tabulate @@ -45,7 +45,7 @@ def main(): if not args.database: parser.error("--database or DATABASE_URL is required") - session = get_db_session(args.database)() + session = create_db_sessionmaker(get_db_engine(args.database))() if args.action == "create": create_user(session, args.username, args.password) diff --git a/setup.py b/setup.py index e667a43..36e9340 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ setup(name='photoapp', entry_points={ "console_scripts": [ "photoappd = photoapp.daemon:main", + "photothumbd = photoapp.thumbserver:main", "photoinfo = photoapp.image:main", "photousers = photoapp.users:main", "photocli = photoapp.cli:main",