minio (s3) storage backend and backend uri support
This commit is contained in:
parent
a2e854568f
commit
83a0702341
|
@ -5,80 +5,12 @@ from datetime import datetime
|
||||||
from photoapp.types import Photo, PhotoSet, Tag, PhotoStatus, User, known_extensions, known_mimes, genuuid
|
from photoapp.types import Photo, PhotoSet, Tag, PhotoStatus, User, known_extensions, known_mimes, genuuid
|
||||||
from photoapp.utils import copysha, get_extension
|
from photoapp.utils import copysha, get_extension
|
||||||
from photoapp.image import special_magic_fobj
|
from photoapp.image import special_magic_fobj
|
||||||
|
from photoapp.storage import StorageAdapter
|
||||||
from photoapp.dbutils import db
|
from photoapp.dbutils import db
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
import traceback
|
import traceback
|
||||||
|
|
||||||
|
|
||||||
class StorageAdapter(object):
|
|
||||||
"""
|
|
||||||
Abstract interface for working with photo file storage. All paths are relative to the storage adapter's root param.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def exists(self, path):
|
|
||||||
# TODO return true/false if the file path exists
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def open(self, path, mode):
|
|
||||||
# TODO return a handle to the path
|
|
||||||
# TODO this should work as a context manager
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def delete(self, path):
|
|
||||||
# TODO erase the path
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def getsize(self, path):
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
|
|
||||||
class FilesystemAdapter(StorageAdapter):
|
|
||||||
def __init__(self, root):
|
|
||||||
super().__init__()
|
|
||||||
self.root = root # root path
|
|
||||||
|
|
||||||
def exists(self, path):
|
|
||||||
# TODO return true/false if the file path exists
|
|
||||||
return os.path.exists(self._abspath(path))
|
|
||||||
|
|
||||||
def open(self, path, mode):
|
|
||||||
# TODO return a handle to the path. this should work as a context manager
|
|
||||||
os.makedirs(os.path.dirname(self._abspath(path)), exist_ok=True)
|
|
||||||
return open(self._abspath(path), mode)
|
|
||||||
|
|
||||||
def delete(self, path):
|
|
||||||
# TODO delete the file
|
|
||||||
# TODO prune empty directories that were components of $path
|
|
||||||
os.unlink(self._abspath(path))
|
|
||||||
|
|
||||||
def getsize(self, path):
|
|
||||||
return os.path.getsize(self._abspath(path))
|
|
||||||
|
|
||||||
def _abspath(self, path):
|
|
||||||
return os.path.join(self.root, path)
|
|
||||||
|
|
||||||
|
|
||||||
class S3Adapter(StorageAdapter):
|
|
||||||
def exists(self, path):
|
|
||||||
# TODO return true/false if the file path exists
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def open(self, path, mode):
|
|
||||||
# TODO return a handle to the path. this should work as a context manager
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def delete(self, path):
|
|
||||||
# TODO erase the path
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
def getsize(self, path):
|
|
||||||
raise NotImplementedError()
|
|
||||||
|
|
||||||
|
|
||||||
class GfapiAdapter(StorageAdapter):
|
|
||||||
pass # TODO gluster storage backend
|
|
||||||
|
|
||||||
|
|
||||||
class LibraryManager(object):
|
class LibraryManager(object):
|
||||||
def __init__(self, storage):
|
def __init__(self, storage):
|
||||||
assert isinstance(storage, StorageAdapter)
|
assert isinstance(storage, StorageAdapter)
|
||||||
|
|
|
@ -7,9 +7,11 @@ from datetime import datetime, timedelta
|
||||||
from photoapp.thumb import ThumbGenerator
|
from photoapp.thumb import ThumbGenerator
|
||||||
from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
|
from photoapp.types import Photo, PhotoSet, Tag, TagItem, PhotoStatus, User
|
||||||
from photoapp.common import pwhash
|
from photoapp.common import pwhash
|
||||||
from photoapp.api import PhotosApi, LibraryManager, FilesystemAdapter
|
from photoapp.api import PhotosApi, LibraryManager
|
||||||
|
from photoapp.storage import FilesystemAdapter
|
||||||
from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine
|
from photoapp.dbutils import SAEnginePlugin, SATool, db, get_db_engine
|
||||||
from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify
|
from photoapp.utils import mime2ext, auth, require_auth, photoset_auth_filter, slugify
|
||||||
|
from photoapp.storage import uri_to_storage
|
||||||
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||||
from sqlalchemy import desc, func, and_, or_
|
from sqlalchemy import desc, func, and_, or_
|
||||||
|
|
||||||
|
@ -438,7 +440,7 @@ def main():
|
||||||
SAEnginePlugin(cherrypy.engine, engine).subscribe()
|
SAEnginePlugin(cherrypy.engine, engine).subscribe()
|
||||||
|
|
||||||
# Create various internal tools
|
# Create various internal tools
|
||||||
library_storage = FilesystemAdapter(args.library)
|
library_storage = uri_to_storage(args.library)
|
||||||
library_manager = LibraryManager(library_storage)
|
library_manager = LibraryManager(library_storage)
|
||||||
thumbnail_tool = ThumbGenerator(library_manager, args.cache)
|
thumbnail_tool = ThumbGenerator(library_manager, args.cache)
|
||||||
|
|
||||||
|
|
|
@ -121,7 +121,6 @@ def special_magic_fobj(fobj, fname):
|
||||||
if fname.split(".")[-1].lower() == "xmp":
|
if fname.split(".")[-1].lower() == "xmp":
|
||||||
return "application/octet-stream-xmp"
|
return "application/octet-stream-xmp"
|
||||||
else:
|
else:
|
||||||
fobj.seek(0)
|
|
||||||
return magic.from_buffer(fobj.read(1024), mime=True)
|
return magic.from_buffer(fobj.read(1024), mime=True)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,210 @@
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
import boto3
|
||||||
|
from botocore.client import Config as BotoConfig
|
||||||
|
import os
|
||||||
|
from botocore.exceptions import ClientError
|
||||||
|
from queue import Queue
|
||||||
|
from threading import Thread
|
||||||
|
|
||||||
|
|
||||||
|
def uri_to_storage(uri):
|
||||||
|
parsed = urlparse(uri)
|
||||||
|
for schemes, backend in {("file", ): FilesystemAdapter,
|
||||||
|
("minio", "minios", ): MinioAdapter}.items():
|
||||||
|
if parsed.scheme in schemes:
|
||||||
|
return backend.from_uri(parsed)
|
||||||
|
raise Exception(f"Unknown storage backend '{parsed.scheme}'")
|
||||||
|
|
||||||
|
|
||||||
|
class StorageAdapter(object):
|
||||||
|
"""
|
||||||
|
Abstract interface for working with photo file storage. All paths are relative to the storage adapter's root param.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def exists(self, path):
|
||||||
|
# return true/false if the file path exists
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def open(self, path, mode):
|
||||||
|
# return a handle to the path
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def delete(self, path):
|
||||||
|
# erase the path
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def getsize(self, path):
|
||||||
|
# return the number of bytes contained at the path
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_uri(uri):
|
||||||
|
# return an instance of the storage class backed by the passed URI
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
class FilesystemAdapter(StorageAdapter):
|
||||||
|
def __init__(self, root):
|
||||||
|
super().__init__()
|
||||||
|
self.root = root # root path
|
||||||
|
|
||||||
|
def exists(self, path):
|
||||||
|
# TODO return true/false if the file path exists
|
||||||
|
return os.path.exists(self._abspath(path))
|
||||||
|
|
||||||
|
def open(self, path, mode):
|
||||||
|
# TODO return a handle to the path. this should work as a context manager
|
||||||
|
os.makedirs(os.path.dirname(self._abspath(path)), exist_ok=True)
|
||||||
|
return open(self._abspath(path), mode)
|
||||||
|
|
||||||
|
def delete(self, path):
|
||||||
|
# TODO delete the file
|
||||||
|
# TODO prune empty directories that were components of $path
|
||||||
|
os.unlink(self._abspath(path))
|
||||||
|
|
||||||
|
def getsize(self, path):
|
||||||
|
return os.path.getsize(self._abspath(path))
|
||||||
|
|
||||||
|
def _abspath(self, path):
|
||||||
|
return os.path.join(self.root, path)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_uri(uri):
|
||||||
|
relative = True if uri.netloc and uri.netloc in (".", "localhost") else False
|
||||||
|
path = uri.path[1:] if relative else (uri.netloc + uri.path)
|
||||||
|
return FilesystemAdapter(os.path.abspath(path))
|
||||||
|
|
||||||
|
|
||||||
|
class S3WriteProxy(object):
|
||||||
|
def __init__(self, storage, key):
|
||||||
|
"""
|
||||||
|
A file-like class that accepts writes (until the writer closes me) and accepts reads (until closed)
|
||||||
|
"""
|
||||||
|
self.storage = storage
|
||||||
|
self.key = key
|
||||||
|
self.q = Queue(maxsize=1024) # number of $os write sizes to buffer
|
||||||
|
self.leftovers = None
|
||||||
|
self.closed = False
|
||||||
|
self.eof = False
|
||||||
|
self.s3_upload = Thread(target=self._upload)
|
||||||
|
self.s3_upload.start()
|
||||||
|
|
||||||
|
def _upload(self):
|
||||||
|
self.storage.s3.upload_fileobj(Fileobj=self,
|
||||||
|
Bucket=self.storage.bucket,
|
||||||
|
Key=self.key)
|
||||||
|
|
||||||
|
def write(self, data):
|
||||||
|
if self.closed:
|
||||||
|
raise Exception(f"cant write to a closed {self.__class__}")
|
||||||
|
self.q.put(data)
|
||||||
|
|
||||||
|
def read(self, size=0):
|
||||||
|
if self.eof:
|
||||||
|
return b''
|
||||||
|
buf = self.leftovers if self.leftovers else b''
|
||||||
|
while True:
|
||||||
|
p = self.q.get()
|
||||||
|
if not p:
|
||||||
|
self.eof = True
|
||||||
|
break
|
||||||
|
buf += p
|
||||||
|
if len(buf) >= size:
|
||||||
|
break
|
||||||
|
if len(buf) > size:
|
||||||
|
self.leftovers = buf[size:]
|
||||||
|
buf = buf[0:size]
|
||||||
|
else:
|
||||||
|
self.leftovers = None
|
||||||
|
|
||||||
|
return buf
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.closed = True
|
||||||
|
self.q.put(None)
|
||||||
|
self.s3_upload.join()
|
||||||
|
|
||||||
|
def __enter__(self, *args):
|
||||||
|
raise Exception("who called me?")
|
||||||
|
|
||||||
|
|
||||||
|
class S3ReadProxy(object):
|
||||||
|
def __init__(self, s3obj):
|
||||||
|
self.obj = s3obj
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.obj.close()
|
||||||
|
|
||||||
|
def read(self, size=None):
|
||||||
|
return self.obj.read(size)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, type, value, traceback):
|
||||||
|
self.obj.close()
|
||||||
|
|
||||||
|
|
||||||
|
class MinioAdapter(StorageAdapter):
|
||||||
|
def __init__(self, s3, bucket, prefix):
|
||||||
|
self.s3 = s3
|
||||||
|
self.bucket = bucket
|
||||||
|
self.prefix = prefix
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.s3.get_bucket_location(Bucket=self.bucket)
|
||||||
|
except Exception:
|
||||||
|
self.s3.create_bucket(Bucket=self.bucket)
|
||||||
|
|
||||||
|
def exists(self, path):
|
||||||
|
try:
|
||||||
|
self.s3.get_object(Bucket=self.bucket, Key=os.path.join(self.prefix, path))
|
||||||
|
return True
|
||||||
|
except ClientError as ce:
|
||||||
|
if ce.response['Error']['Code'] == 'NoSuchKey':
|
||||||
|
return False
|
||||||
|
raise
|
||||||
|
|
||||||
|
def open(self, path, mode):
|
||||||
|
# TODO return a handle to the path. this should work as a context manager
|
||||||
|
if "r" in mode:
|
||||||
|
return S3ReadProxy(self.s3.get_object(Bucket=self.bucket, Key=os.path.join(self.prefix, path))["Body"])
|
||||||
|
elif "w" in mode:
|
||||||
|
return S3WriteProxy(self, os.path.join(self.prefix, path))
|
||||||
|
else:
|
||||||
|
raise Exception("unknown file open mode")
|
||||||
|
|
||||||
|
def delete(self, path):
|
||||||
|
# TODO handle verification of return status
|
||||||
|
self.s3.delete_object(Bucket=self.bucket, Key=os.path.join(self.prefix, path))
|
||||||
|
|
||||||
|
def getsize(self, path):
|
||||||
|
return self.s3.head_object(Bucket=self.bucket, Key=os.path.join(self.prefix, path))["ContentLength"]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def from_uri(uri):
|
||||||
|
secure = uri.scheme.endswith("s")
|
||||||
|
|
||||||
|
s3args = {"config": BotoConfig(signature_version='s3v4')}
|
||||||
|
|
||||||
|
endpoint_url = f"http{'s' if secure else ''}://{uri.hostname}"
|
||||||
|
if uri.port:
|
||||||
|
endpoint_url += f":{uri.port}"
|
||||||
|
s3args["endpoint_url"] = endpoint_url
|
||||||
|
|
||||||
|
if uri.username and uri.password:
|
||||||
|
s3args["aws_access_key_id"] = uri.username
|
||||||
|
s3args["aws_secret_access_key"] = uri.password
|
||||||
|
|
||||||
|
s3 = boto3.client('s3', **s3args)
|
||||||
|
bucketpath = uri.path[1:]
|
||||||
|
|
||||||
|
splitted = bucketpath.split("/", 1)
|
||||||
|
bucket = splitted.pop(0)
|
||||||
|
prefix = splitted[0].lstrip("/") if splitted else ''
|
||||||
|
|
||||||
|
return MinioAdapter(s3, bucket, prefix)
|
||||||
|
|
||||||
|
|
||||||
|
class GfapiAdapter(StorageAdapter):
|
||||||
|
pass # TODO gluster storage backend
|
|
@ -53,8 +53,9 @@ class ThumbGenerator(object):
|
||||||
# TODO have the subprocess download the file
|
# TODO have the subprocess download the file
|
||||||
with tempfile.TemporaryDirectory() as tmpdir:
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
fpath = os.path.join(tmpdir, "image")
|
fpath = os.path.join(tmpdir, "image")
|
||||||
with self.library.storage.open(photo.path, 'rb') as fsrc, open(fpath, 'wb') as fdest:
|
with self.library.storage.open(photo.path, 'rb') as fsrc:
|
||||||
copyfileobj(fsrc, fdest)
|
with open(fpath, 'wb') as fdest:
|
||||||
|
copyfileobj(fsrc, fdest)
|
||||||
|
|
||||||
p = Process(target=self.gen_thumb, args=(fpath, dest, thumb_width, thumb_height, photo.orientation))
|
p = Process(target=self.gen_thumb, args=(fpath, dest, thumb_width, thumb_height, photo.orientation))
|
||||||
p.start()
|
p.start()
|
||||||
|
|
|
@ -7,7 +7,7 @@ import hashlib
|
||||||
def copysha(fpin, fpout):
|
def copysha(fpin, fpout):
|
||||||
sha = hashlib.sha256()
|
sha = hashlib.sha256()
|
||||||
while True:
|
while True:
|
||||||
b = fpin.read(4096)
|
b = fpin.read(1024 * 256)
|
||||||
if not b:
|
if not b:
|
||||||
break
|
break
|
||||||
fpout.write(b)
|
fpout.write(b)
|
||||||
|
|
|
@ -1,19 +1,25 @@
|
||||||
backports.functools-lru-cache==1.5
|
backports.functools-lru-cache==1.5
|
||||||
|
boto3==1.9.183
|
||||||
|
botocore==1.12.183
|
||||||
certifi==2019.6.16
|
certifi==2019.6.16
|
||||||
chardet==3.0.4
|
chardet==3.0.4
|
||||||
cheroot==6.5.2
|
cheroot==6.5.2
|
||||||
CherryPy==18.1.2
|
CherryPy==18.1.2
|
||||||
contextlib2==0.5.5
|
contextlib2==0.5.5
|
||||||
|
docutils==0.14
|
||||||
idna==2.8
|
idna==2.8
|
||||||
jaraco.functools==1.20
|
jaraco.functools==1.20
|
||||||
Jinja2==2.10.1
|
Jinja2==2.10.1
|
||||||
|
jmespath==0.9.4
|
||||||
MarkupSafe==1.0
|
MarkupSafe==1.0
|
||||||
more-itertools==4.3.0
|
more-itertools==4.3.0
|
||||||
Pillow==5.2.0
|
Pillow==5.2.0
|
||||||
portend==2.3
|
portend==2.3
|
||||||
|
python-dateutil==2.8.0
|
||||||
python-magic==0.4.15
|
python-magic==0.4.15
|
||||||
pytz==2018.5
|
pytz==2018.5
|
||||||
requests==2.22.0
|
requests==2.22.0
|
||||||
|
s3transfer==0.2.1
|
||||||
six==1.11.0
|
six==1.11.0
|
||||||
SQLAlchemy==1.3.5
|
SQLAlchemy==1.3.5
|
||||||
tabulate==0.8.3
|
tabulate==0.8.3
|
||||||
|
|
Loading…
Reference in New Issue