use env vars for config

.dockerignore

@@ -10,3 +10,6 @@ styles/css/
 styles/dist/
 styles/mincss/
 testenv/
+source/
+source_copy/
+raws/

Dockerfile

@@ -3,7 +3,7 @@ FROM ubuntu:bionic
 RUN apt-get update && \
     apt-get install -y wget software-properties-common && \
     echo "deb https://deb.nodesource.com/node_10.x bionic main" | tee /etc/apt/sources.list.d/nodesource.list && \
-    wget -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
+    wget -q -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
     apt-get update && \
     apt-get install -y nodejs
 
@@ -13,7 +13,7 @@ RUN cd /tmp/code && \
     npm install && \
     ./node_modules/.bin/grunt
 
-FROM ubuntu:bionic
+FROM ubuntu:disco
 
 ADD . /tmp/code/
 
@@ -24,7 +24,7 @@ RUN apt-get update && \
 
 RUN pip3 install -U pip && \
     cd /tmp/code && \
-    pip install -r requirements.txt && \
+    pip3 install -r requirements.txt && \
     python3 setup.py install && \
     useradd --uid 1000 app
 
@@ -33,5 +33,6 @@ VOLUME /srv/cache
 VOLUME /srv/db
 
 USER app
+ENV CACHE_PATH=/tmp/cache
 
-ENTRYPOINT ["photoappd", "--library", "/srv/library", "--database", "/srv/db/photos.db", "--cache", "/srv/cache"]
+ENTRYPOINT ["photoappd"]

README.md

@@ -42,17 +42,25 @@ Usage
 
 After installation, run the server:
 
-* `photoappd --port 8080 --library ./library --database ./photos.db --cache ./cache`
+* `photoappd --port 8080 --library file://./library --database sqlite:///photos.db --cache ./cache`
 
 Arguments are as follows:
 
-* `--library ./library` - store the hosted image files under this directory
-* `--database ./photos.db` - store the SQLite database in this directory
+* `--library file://./library` - file storage uri, in this case the relative path `./library`
+* `--database sqlite:///photos.db` - [Sqlalchemy](https://docs.sqlalchemy.org/en/13/core/engines.html) connection uri
 * `--cache ./cache` - use this directory as a cache for things like thumbnails
 * `--port 8080` - listen on http on port 8080
 
+Supported library uri schemes are:
+
+* file - relative (as above) or absolute file paths - `file:///srv/library`.
+* minio - `minio://username:password@host/bucket_name/path/prefix`
+
+Photolib is tested with [Minio](https://min.io/), but should work with S3. For the database, minio is tested with SQLite
+and MySQL (using the `mysql+pymysql://` driver).
+
 Next, the `photousers` command can be used to create a user account. Login information is necessary to see images marked
-as private or upload images.
+as private or upload images. You may want to run this before starting the server, but either order works.
 
 * `photousers create -u dave -p mypassword`
 
@@ -112,13 +120,19 @@ Roadmap
         - ~~Standardize on API ingest~~ done
         - ~~Display and retrieval of images from the abstracted image store~~ done
             - ~~Thumbnail gen~~ done
-    - Database support
+    - ~~Database support~~
         - ~~Get the web UI code (daemon.py) using the same db access method as the api~~
-        - Support any connection URI sqlalchemy is happy with
-        - Tune certain databases if their uri is detected (sqlite and threads lol)
+        - ~~Support any connection URI sqlalchemy is happy with~~
+        - ~~Tune certain databases if their uri is detected (sqlite and threads lol)~~
     - ~~Cache~~ done
         - ~~Using the local fs seems fine?~~ done
-- Option to cherrypy sessions - Redis?
+    - Migration path
+        - open database
+        - copy files table to memory
+        - recreate files table
+        - insert into the new table, with replaced paths, generating a list of files moves at the same time
+        - migrate files to the new storage according to the list
+- Storage option for cherrypy sessions - Redis?
 - Flesh out CLI:
     - Config that is saved somewhere
     - Support additional fields on upload like title description tags etc

photoapp/cli.py

@@ -214,6 +214,7 @@ def main():
                 e = future.exception()
                 if e:
                     results.append([set_fnames, "exception", repr(e)])
+                    numerrors += 1
                 else:
                     result = future.result()
                     if result[0] != "success":

photoapp/daemon.py

@@ -422,14 +422,26 @@ def main():
 
     parser = argparse.ArgumentParser(description="Photod photo server")
 
-    parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on")
-    parser.add_argument('-l', '--library', default="./library", help="library path")
-    parser.add_argument('-c', '--cache', default="./cache", help="cache path")
-    parser.add_argument('-s', '--database', default="./photos.db", help="path to persistent sqlite database")
+    parser.add_argument('-p', '--port', help="tcp port to listen on",
+                        default=int(os.environ.get("PHOTOLIB_PORT", 8080)), type=int)
+    parser.add_argument('-l', '--library', default=os.environ.get("STORAGE_URL"), help="library path")
+    parser.add_argument('-c', '--cache', default=os.environ.get("CACHE_PATH"), help="cache path")
+    # https://docs.sqlalchemy.org/en/13/core/engines.html
+    parser.add_argument('-s', '--database', help="sqlalchemy database connection uri",
+                        default=os.environ.get("DATABASE_URL")),
     parser.add_argument('--debug', action="store_true", help="enable development options")
 
     args = parser.parse_args()
 
+    if not args.database:
+        parser.error("--database or DATABASE_URL is required")
+
+    if not args.library:
+        parser.error("--library or STORAGE_URL is required")
+
+    if not args.cache:
+        parser.error("--cache or CACHE_PATH is required")
+
     logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
                         format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
 

photoapp/dbutils.py

@@ -2,17 +2,28 @@ import sqlalchemy
 import cherrypy
 from cherrypy.process import plugins
 from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.pool import StaticPool, AssertionPool, NullPool
+from sqlalchemy.pool import NullPool
 from sqlalchemy.orm import sessionmaker
 
 
 Base = declarative_base()
 
 
+engine_specific_options = {"sqlite": dict(connect_args={'check_same_thread': False},
+                                          poolclass=NullPool,
+                                          pool_pre_ping=True),
+                           "mysql": dict(pool_pre_ping=True)}
+
+
+def get_engine_options(uri):
+    for engine_prefix, options in engine_specific_options.items():
+        if uri.startswith(engine_prefix):
+            return options
+    return {}
+
+
 def get_db_engine(uri):
-    # TODO handle more uris
-    engine = sqlalchemy.create_engine('sqlite:///{}'.format(uri),
-                                      connect_args={'check_same_thread': False}, poolclass=NullPool, pool_pre_ping=True)
+    engine = sqlalchemy.create_engine(uri, **get_engine_options(uri))
     Base.metadata.create_all(engine)
     return engine
 

photoapp/types.py

@@ -50,7 +50,7 @@ class PhotoSet(Base):
     __tablename__ = 'photos'
 
     id = Column(Integer, primary_key=True)
-    uuid = Column(Unicode, unique=True, default=lambda: str(uuid.uuid4()))
+    uuid = Column(Unicode(length=36), unique=True, default=lambda: str(uuid.uuid4()))
     date = Column(DateTime)
     date_real = Column(DateTime)
     date_offset = Column(Integer, default=0)  # minutes
@@ -60,9 +60,9 @@ class PhotoSet(Base):
     files = relationship("Photo", back_populates="set")
     tags = relationship("TagItem", back_populates="set")
 
-    title = Column(String)
-    description = Column(String)
-    slug = Column(String)
+    title = Column(String(length=128))
+    description = Column(String(length=1024))
+    slug = Column(String(length=128))
 
     status = Column(Enum(PhotoStatus), default=PhotoStatus.private)
 
@@ -83,7 +83,7 @@ class Photo(Base):
 
     id = Column(Integer, primary_key=True)
     set_id = Column(Integer, ForeignKey("photos.id"))
-    uuid = Column(Unicode, unique=True, default=genuuid)
+    uuid = Column(Unicode(length=36), unique=True, default=genuuid)
 
     set = relationship("PhotoSet", back_populates="files", foreign_keys=[set_id])
 
@@ -92,8 +92,8 @@ class Photo(Base):
     height = Column(Integer)
     orientation = Column(Integer, default=0)
     hash = Column(String(length=64), unique=True)
-    path = Column(Unicode)
-    format = Column(String(length=64))  # TODO how long can a mime string be
+    path = Column(Unicode(length=64))
+    format = Column(String(length=32))  # TODO how long can a mime string be
     fname = Column(String(length=64))  # seems generous enough
 
     def to_json(self):
@@ -107,18 +107,18 @@ class Tag(Base):
     __tablename__ = 'tags'
 
     id = Column(Integer, primary_key=True)
-    uuid = Column(Unicode, unique=True, default=lambda: str(uuid.uuid4()))
+    uuid = Column(Unicode(length=36), unique=True, default=lambda: str(uuid.uuid4()))
     created = Column(DateTime, default=lambda: datetime.now())
     modified = Column(DateTime, default=lambda: datetime.now())
     is_album = Column(Boolean, default=False)
     # slug-like short name such as "iomtrip"
-    name = Column(String, unique=True)
+    name = Column(String(length=32), unique=True)
     # longer human-format title like "Isle of Man trip"
-    title = Column(String)
+    title = Column(String(length=32))
     # url slug like "isle-of-man-trip"
-    slug = Column(String, unique=True)
+    slug = Column(String(length=32), unique=True)
     # fulltext description
-    description = Column(String)
+    description = Column(String(length=256))
 
     entries = relationship("TagItem", back_populates="tag")
 

photoapp/users.py

@@ -1,3 +1,4 @@
+import os
 import argparse
 from photoapp.types import User
 from photoapp.common import pwhash
@@ -25,7 +26,8 @@ def delete_user(s, username):
 
 def main():
     parser = argparse.ArgumentParser(description="User manipulation tool")
-    parser.add_argument("-d", "--database", help="database uri")
+    parser.add_argument('-s', '--database', help="sqlalchemy database connection uri",
+                        default=os.environ.get("DATABASE_URL")),
 
     p_mode = parser.add_subparsers(dest='action', help='action to take')
 
@@ -40,6 +42,9 @@ def main():
 
     args = parser.parse_args()
 
+    if not args.database:
+        parser.error("--database or DATABASE_URL is required")
+
     session = get_db_session(args.database)()
 
     if args.action == "create":

requirements.txt

@@ -15,6 +15,7 @@ MarkupSafe==1.0
 more-itertools==4.3.0
 Pillow==5.2.0
 portend==2.3
+PyMySQL==0.9.3
 python-dateutil==2.8.0
 python-magic==0.4.15
 pytz==2018.5