From 3bbe0f20ea42c28a94daa6b2fa8014057d451f12 Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 6 Jul 2019 11:25:15 -0700 Subject: [PATCH] use env vars for config --- .dockerignore | 3 +++ Dockerfile | 9 +++++---- README.md | 30 ++++++++++++++++++++++-------- photoapp/cli.py | 1 + photoapp/daemon.py | 20 ++++++++++++++++---- photoapp/dbutils.py | 19 +++++++++++++++---- photoapp/types.py | 24 ++++++++++++------------ photoapp/users.py | 7 ++++++- requirements.txt | 1 + 9 files changed, 81 insertions(+), 33 deletions(-) diff --git a/.dockerignore b/.dockerignore index a235ff1..13525a4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,6 @@ styles/css/ styles/dist/ styles/mincss/ testenv/ +source/ +source_copy/ +raws/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 54c246e..e3ab07f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:bionic RUN apt-get update && \ apt-get install -y wget software-properties-common && \ echo "deb https://deb.nodesource.com/node_10.x bionic main" | tee /etc/apt/sources.list.d/nodesource.list && \ - wget -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + wget -q -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ apt-get update && \ apt-get install -y nodejs @@ -13,7 +13,7 @@ RUN cd /tmp/code && \ npm install && \ ./node_modules/.bin/grunt -FROM ubuntu:bionic +FROM ubuntu:disco ADD . /tmp/code/ @@ -24,7 +24,7 @@ RUN apt-get update && \ RUN pip3 install -U pip && \ cd /tmp/code && \ - pip install -r requirements.txt && \ + pip3 install -r requirements.txt && \ python3 setup.py install && \ useradd --uid 1000 app @@ -33,5 +33,6 @@ VOLUME /srv/cache VOLUME /srv/db USER app +ENV CACHE_PATH=/tmp/cache -ENTRYPOINT ["photoappd", "--library", "/srv/library", "--database", "/srv/db/photos.db", "--cache", "/srv/cache"] +ENTRYPOINT ["photoappd"] diff --git a/README.md b/README.md index 119caf1..205a1f7 100644 --- a/README.md +++ b/README.md @@ -42,17 +42,25 @@ Usage After installation, run the server: -* `photoappd --port 8080 --library ./library --database ./photos.db --cache ./cache` +* `photoappd --port 8080 --library file://./library --database sqlite:///photos.db --cache ./cache` Arguments are as follows: -* `--library ./library` - store the hosted image files under this directory -* `--database ./photos.db` - store the SQLite database in this directory +* `--library file://./library` - file storage uri, in this case the relative path `./library` +* `--database sqlite:///photos.db` - [Sqlalchemy](https://docs.sqlalchemy.org/en/13/core/engines.html) connection uri * `--cache ./cache` - use this directory as a cache for things like thumbnails * `--port 8080` - listen on http on port 8080 +Supported library uri schemes are: + +* file - relative (as above) or absolute file paths - `file:///srv/library`. +* minio - `minio://username:password@host/bucket_name/path/prefix` + +Photolib is tested with [Minio](https://min.io/), but should work with S3. For the database, minio is tested with SQLite +and MySQL (using the `mysql+pymysql://` driver). + Next, the `photousers` command can be used to create a user account. Login information is necessary to see images marked -as private or upload images. +as private or upload images. You may want to run this before starting the server, but either order works. * `photousers create -u dave -p mypassword` @@ -112,13 +120,19 @@ Roadmap - ~~Standardize on API ingest~~ done - ~~Display and retrieval of images from the abstracted image store~~ done - ~~Thumbnail gen~~ done - - Database support + - ~~Database support~~ - ~~Get the web UI code (daemon.py) using the same db access method as the api~~ - - Support any connection URI sqlalchemy is happy with - - Tune certain databases if their uri is detected (sqlite and threads lol) + - ~~Support any connection URI sqlalchemy is happy with~~ + - ~~Tune certain databases if their uri is detected (sqlite and threads lol)~~ - ~~Cache~~ done - ~~Using the local fs seems fine?~~ done -- Option to cherrypy sessions - Redis? + - Migration path + - open database + - copy files table to memory + - recreate files table + - insert into the new table, with replaced paths, generating a list of files moves at the same time + - migrate files to the new storage according to the list +- Storage option for cherrypy sessions - Redis? - Flesh out CLI: - Config that is saved somewhere - Support additional fields on upload like title description tags etc diff --git a/photoapp/cli.py b/photoapp/cli.py index 9b9b217..9067cd9 100644 --- a/photoapp/cli.py +++ b/photoapp/cli.py @@ -214,6 +214,7 @@ def main(): e = future.exception() if e: results.append([set_fnames, "exception", repr(e)]) + numerrors += 1 else: result = future.result() if result[0] != "success": diff --git a/photoapp/daemon.py b/photoapp/daemon.py index beff585..a27c0d4 100644 --- a/photoapp/daemon.py +++ b/photoapp/daemon.py @@ -422,14 +422,26 @@ def main(): parser = argparse.ArgumentParser(description="Photod photo server") - parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on") - parser.add_argument('-l', '--library', default="./library", help="library path") - parser.add_argument('-c', '--cache', default="./cache", help="cache path") - parser.add_argument('-s', '--database', default="./photos.db", help="path to persistent sqlite database") + parser.add_argument('-p', '--port', help="tcp port to listen on", + default=int(os.environ.get("PHOTOLIB_PORT", 8080)), type=int) + parser.add_argument('-l', '--library', default=os.environ.get("STORAGE_URL"), help="library path") + parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_PATH"), help="cache path") + # https://docs.sqlalchemy.org/en/13/core/engines.html + parser.add_argument('-s', '--database', help="sqlalchemy database connection uri", + default=os.environ.get("DATABASE_URL")), parser.add_argument('--debug', action="store_true", help="enable development options") args = parser.parse_args() + if not args.database: + parser.error("--database or DATABASE_URL is required") + + if not args.library: + parser.error("--library or STORAGE_URL is required") + + if not args.cache: + parser.error("--cache or CACHE_PATH is required") + logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") diff --git a/photoapp/dbutils.py b/photoapp/dbutils.py index 3f6c29a..34adef0 100644 --- a/photoapp/dbutils.py +++ b/photoapp/dbutils.py @@ -2,17 +2,28 @@ import sqlalchemy import cherrypy from cherrypy.process import plugins from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.pool import StaticPool, AssertionPool, NullPool +from sqlalchemy.pool import NullPool from sqlalchemy.orm import sessionmaker Base = declarative_base() +engine_specific_options = {"sqlite": dict(connect_args={'check_same_thread': False}, + poolclass=NullPool, + pool_pre_ping=True), + "mysql": dict(pool_pre_ping=True)} + + +def get_engine_options(uri): + for engine_prefix, options in engine_specific_options.items(): + if uri.startswith(engine_prefix): + return options + return {} + + def get_db_engine(uri): - # TODO handle more uris - engine = sqlalchemy.create_engine('sqlite:///{}'.format(uri), - connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True) + engine = sqlalchemy.create_engine(uri, **get_engine_options(uri)) Base.metadata.create_all(engine) return engine diff --git a/photoapp/types.py b/photoapp/types.py index 5b5212d..ddcee41 100644 --- a/photoapp/types.py +++ b/photoapp/types.py @@ -50,7 +50,7 @@ class PhotoSet(Base): __tablename__ = 'photos' id = Column(Integer, primary_key=True) - uuid = Column(Unicode, unique=True, default=lambda: str(uuid.uuid4())) + uuid = Column(Unicode(length=36), unique=True, default=lambda: str(uuid.uuid4())) date = Column(DateTime) date_real = Column(DateTime) date_offset = Column(Integer, default=0) # minutes @@ -60,9 +60,9 @@ class PhotoSet(Base): files = relationship("Photo", back_populates="set") tags = relationship("TagItem", back_populates="set") - title = Column(String) - description = Column(String) - slug = Column(String) + title = Column(String(length=128)) + description = Column(String(length=1024)) + slug = Column(String(length=128)) status = Column(Enum(PhotoStatus), default=PhotoStatus.private) @@ -83,7 +83,7 @@ class Photo(Base): id = Column(Integer, primary_key=True) set_id = Column(Integer, ForeignKey("photos.id")) - uuid = Column(Unicode, unique=True, default=genuuid) + uuid = Column(Unicode(length=36), unique=True, default=genuuid) set = relationship("PhotoSet", back_populates="files", foreign_keys=[set_id]) @@ -92,8 +92,8 @@ class Photo(Base): height = Column(Integer) orientation = Column(Integer, default=0) hash = Column(String(length=64), unique=True) - path = Column(Unicode) - format = Column(String(length=64)) # TODO how long can a mime string be + path = Column(Unicode(length=64)) + format = Column(String(length=32)) # TODO how long can a mime string be fname = Column(String(length=64)) # seems generous enough def to_json(self): @@ -107,18 +107,18 @@ class Tag(Base): __tablename__ = 'tags' id = Column(Integer, primary_key=True) - uuid = Column(Unicode, unique=True, default=lambda: str(uuid.uuid4())) + uuid = Column(Unicode(length=36), unique=True, default=lambda: str(uuid.uuid4())) created = Column(DateTime, default=lambda: datetime.now()) modified = Column(DateTime, default=lambda: datetime.now()) is_album = Column(Boolean, default=False) # slug-like short name such as "iomtrip" - name = Column(String, unique=True) + name = Column(String(length=32), unique=True) # longer human-format title like "Isle of Man trip" - title = Column(String) + title = Column(String(length=32)) # url slug like "isle-of-man-trip" - slug = Column(String, unique=True) + slug = Column(String(length=32), unique=True) # fulltext description - description = Column(String) + description = Column(String(length=256)) entries = relationship("TagItem", back_populates="tag") diff --git a/photoapp/users.py b/photoapp/users.py index f7ea901..02ba21c 100644 --- a/photoapp/users.py +++ b/photoapp/users.py @@ -1,3 +1,4 @@ +import os import argparse from photoapp.types import User from photoapp.common import pwhash @@ -25,7 +26,8 @@ def delete_user(s, username): def main(): parser = argparse.ArgumentParser(description="User manipulation tool") - parser.add_argument("-d", "--database", help="database uri") + parser.add_argument('-s', '--database', help="sqlalchemy database connection uri", + default=os.environ.get("DATABASE_URL")), p_mode = parser.add_subparsers(dest='action', help='action to take') @@ -40,6 +42,9 @@ def main(): args = parser.parse_args() + if not args.database: + parser.error("--database or DATABASE_URL is required") + session = get_db_session(args.database)() if args.action == "create": diff --git a/requirements.txt b/requirements.txt index 23bef51..6dd9144 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ MarkupSafe==1.0 more-itertools==4.3.0 Pillow==5.2.0 portend==2.3 +PyMySQL==0.9.3 python-dateutil==2.8.0 python-magic==0.4.15 pytz==2018.5