Use zodb instead of flat files

Dockerfile

@@ -1,15 +1,26 @@
 FROM ubuntu:bionic
 
-RUN apt-get update && \
+RUN sed -i -E 's/(archive|security).ubuntu.com/192.168.1.142/' /etc/apt/sources.list && \
-    apt-get install -y python3-pip
+    sed -i -E 's/^deb-src/# deb-src/' /etc/apt/sources.list && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+        apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+            wget gpg git build-essential && \
+    wget -qO- http://artifact.scc.net.davepedu.com/repo/apt/extpython/dists/bionic/install | bash /dev/stdin && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive \
+        apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+            extpython-python3.7 && \
+    apt-get clean autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/{apt,dpkg,cache,log}/
 
-ADD . /tmp/code/
+ADD . /tmp/code
 
-RUN pip3 install -U pip && \
+RUN cd /tmp/code && \
-    cd /tmp/code && \
+    /opt/extpython/3.7/bin/pip3 install -r requirements.txt && \
-    python3 setup.py install && \
+    /opt/extpython/3.7/bin/python3 setup.py install && \
     useradd --uid 1000 app
 
-VOLUME /data/
 USER app
-ENTRYPOINT ["wastebind", "-d", "/data/"]
+ENTRYPOINT ["/opt/extpython/3.7/bin/wastebind"]

requirements.txt

@@ -1,16 +1,30 @@
 appdirs==1.4.3
 backports.functools-lru-cache==1.5
-certifi==2018.11.29
+BTrees==4.5.1
+certifi==2019.3.9
+cffi==1.12.3
 chardet==3.0.4
-cheroot==6.5.4
+cheroot==6.5.5
-CherryPy==18.1.0
+CherryPy==18.1.1
 idna==2.8
 jaraco.functools==2.0
-more-itertools==5.0.0
+more-itertools==7.0.0
-portend==2.3
+perfmetrics==2.0
-pytz==2018.9
+persistent==4.5.0
-requests==2.21.0
+portend==2.4
+pycparser==2.19
+PyMySQL==0.9.3
+pytz==2019.1
+RelStorage==2.1.1
+requests==2.22.0
 six==1.12.0
-tempora==1.14
+tempora==1.14.1
-urllib3==1.24.1
+transaction==2.4.0
+urllib3==1.25.3
 zc.lockfile==1.4
+ZConfig==3.4.0
+zdaemon==4.3
+ZEO==5.2.1
+ZODB==5.5.1
+zodbpickle==1.0.3
+zope.interface==4.6.0

setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 import os
 
 
-__version__ = "0.0.0"
+__version__ = "0.0.1"
 with open(os.path.join(os.path.dirname(__file__), "requirements.txt")) as f:
     __requirements__ = [line.strip() for line in f.readlines()]
 

wastebin/cli.py

@@ -41,7 +41,9 @@ def main():
     # parser.add_argument("-p", "--password", help="password")
 
     spr_action = parser.add_subparsers(dest="action", help="action to take")
-    spr_action.add_parser("list", help="show list of pastes")
+
+    spr_list = spr_action.add_parser("list", help="show list of pastes")
+    spr_list.add_argument("name", nargs="?", help="prefix to match")
 
     spr_new = spr_action.add_parser("new", help="create a paste")
     spr_new.add_argument("name", nargs="?", default="", help="name of paste to create")
@@ -89,7 +91,13 @@ def main():
         r.delete(host + args.name).raise_for_status()
 
     elif args.action == "list":
-        print(r.get(host + "search").text, end="") 
+        print(r.get(host + "search",
+                    params={"prefix": args.name} if args.name else None).text,
+              end="")
+
+    else:
+        parser.error('must specify an action')
+
 
 if __name__ == "__main__":
     main()

wastebin/daemon.py

@@ -1,9 +1,84 @@
 import os
 import cherrypy
 import logging
-import hashlib
 import re
-from threading import Thread
+from urllib.parse import urlparse
+import ZODB
+from relstorage.storage import RelStorage
+from relstorage.options import Options
+from relstorage.adapters.mysql import MySQLAdapter
+import persistent
+import persistent.list
+import ZODB.FileStorage
+import persistent.mapping
+import BTrees.OOBTree
+
+
+def pmap():
+    return persistent.mapping.PersistentMapping()
+
+
+class Database(object):
+    def __init__(self, storage):
+        self.db = ZODB.DB(storage)
+        self.init_db()
+
+    @staticmethod
+    def from_uri(uri):
+        """
+        Return a database backed by the storage specified by the passed uri. URIs containing a scheme (scheme://) will
+        be checked against installed adapters. Schemeless URIs are assumed to be a file path for flat file storage.
+        """
+        parsed = urlparse(uri)
+        storage = None
+
+        if parsed.scheme:
+            mysql = MySQLAdapter(host=parsed.hostname, port=parsed.port,
+                                 user=parsed.username, passwd=parsed.password,
+                                 db=parsed.path[1:], options=Options(keep_history=False))
+            storage = RelStorage(adapter=mysql)
+        else:
+            storage = ZODB.FileStorage.FileStorage(uri)
+
+        if storage is None:
+            raise Exception(f"Unsupported uri {uri}")
+
+        return Database(storage)
+
+    def init_db(self):
+        with self.db.transaction() as c:
+            if "pastes" not in c.root():
+                c.root.pastes = BTrees.OOBTree.BTree()
+
+    def loadpaste(self, name):
+        with self.db.transaction() as c:
+            return c.root.pastes[name].value
+
+    def writepaste(self, name, contents):
+        with self.db.transaction() as c:
+            try:
+                paste = c.root.pastes[name]
+                paste.value = contents
+            except KeyError:
+                paste = Paste(contents)
+                c.root.pastes[name] = paste
+
+    def delpaste(self, name):
+        with self.db.transaction() as c:
+            del c.root.pastes[name]
+
+    def iterpastes(self, prefix=None):
+        with self.db.transaction() as c:
+            for name, value in c.root.pastes.items():
+                if prefix and not name.startswith(prefix):
+                    continue
+                yield (name, value, )
+
+
+class Paste(persistent.Persistent):
+    def __init__(self, value):
+        self.value = value
+
 
 PAGE = """<!DOCTYPE html>
 <html lang="en">
@@ -22,119 +97,83 @@ PAGE = """<!DOCTYPE html>
 """
 
 
-RE_NAME = re.compile(r'^[a-z0-9_\-/]+$')
+RE_NAME_RAW = r'^[a-z0-9_\-/]+$'
-
+RE_NAME = re.compile(RE_NAME_RAW)
-
-def sha256(data):
-    h = hashlib.sha256()
-    h.update(data.encode("utf-8"))
-    return h.hexdigest()
 
 
 class WasteWeb(object):
-    def __init__(self, datadir):
+    def __init__(self, db):
-        self.datadir = datadir
+        self.db = db
-        self.namecache = set()
-        t = Thread(target=self.prep_cache)
-        t.daemon = True
-        t.start()
-
-    def prep_cache(self):
-        print("Populating index cache....")
-        for dirpath, dirnames, filenames in os.walk(self.datadir):
-            for fname in filenames:
-                with open(os.path.join(dirpath, fname)) as f:
-                    self.namecache.update([f.readline().strip()])
-        print("Indexed {} items".format(len(self.namecache)))
 
     @cherrypy.expose
     def index(self, load=None):
         data = ""
         if load:
-            assert RE_NAME.match(load)
+            try:
-            data = self.loadpaste(load)
+                data = self.db.loadpaste(load)
+            except KeyError:
+                raise cherrypy.HTTPError(404)
         yield PAGE.format(data=data.replace("<", "&lt;"), load=load or "")
 
     @cherrypy.expose
     def make(self, name, contents):
-        pname = name or sha256(contents)
+        if not RE_NAME.match(name):
-        assert RE_NAME.match(pname)
+            raise cherrypy.HTTPError(400, f"paste name must match {RE_NAME_RAW}")
-        self.writepaste(pname, contents)
+        self.db.writepaste(name, contents)
-        raise cherrypy.HTTPRedirect("/" + pname)
+        raise cherrypy.HTTPRedirect("/" + name)
 
     @cherrypy.expose
     def default(self, *args):
-        if cherrypy.request.method == "DELETE":
+        try:
-            self.delpaste(args[0])
+            if cherrypy.request.method == "DELETE":
-            return "OK"
+                self.db.delpaste(args[0])
-        else:
+                return "OK"
-            cherrypy.response.headers['Content-Type'] = 'text/plain'
+            else:
-            return self.loadpaste(args[0]).encode("utf-8")
+                cherrypy.response.headers['Content-Type'] = 'text/plain'
+                return self.db.loadpaste(args[0]).encode("utf-8")
+        except KeyError:
+            raise cherrypy.HTTPError(404)
 
     @cherrypy.expose
-    def search(self):
+    def search(self, prefix=""):
-        for entry in self.namecache:
+        cherrypy.response.headers['Content-Type'] = 'text/plain'
-            yield entry + "\n"
 
-    def loadpaste(self, name):
+        def _work():
-        path = self.pastepath(sha256(name))
+            for name, _ in self.db.iterpastes(prefix):
-        with open(path) as f:
+                yield name + "\n"
-            f.readline()  # the name
+        return _work()
-            return f.read()
-
-    def writepaste(self, name, contents):
-        hname = sha256(name)
-        path = self.pastepath(hname)
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        with open(path, "w") as f:
-            f.write(name)
-            f.write("\n")
-            f.write(contents)
-        self.namecache.update({name})
-
-    def delpaste(self, name):
-        self.namecache.remove(name)
-        path = self.pastepath(sha256(name))
-        os.unlink(path)
-        pdir = os.path.dirname(path)
-        try:
-            os.rmdir(os.path.normpath(pdir))
-            os.rmdir(os.path.normpath(os.path.join(pdir, "../")))
-        except:
-            pass
-
-    def pastepath(self, hashedname):
-        return os.path.join(self.datadir, hashedname[0], hashedname[1], hashedname + ".txt")
 
 
 def main():
     import argparse
     import signal
 
-    parser = argparse.ArgumentParser(description="")
+    parser = argparse.ArgumentParser(description="basic pastebin",
+                                     epilog="supprted databases are file paths and mysql://")
 
-    parser.add_argument('-p', '--port', default=8080, type=int, help="http port")
+    parser.add_argument('-p', '--port', default=int(os.environ.get("PASTE_PORT", 8080)), type=int, help="http port")
-    parser.add_argument('-d', '--data', default="./", help="data dir")
+    parser.add_argument('-d', '--database', default=os.environ.get("PASTE_DB", None), help="database uri")
     parser.add_argument('--debug', action="store_true", help="enable development options")
 
     args = parser.parse_args()
 
+    if not args.database:
+        parser.error("the following arguments are required: -d/--database")
+
     logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
                         format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
 
-    web = WasteWeb(args.data)
+    web = WasteWeb(Database.from_uri(args.database))
 
     cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False}})
 
     cherrypy.config.update({
-        'tools.sessions.on': False,
+        "tools.sessions.on": False,
-        'request.show_tracebacks': True,
+        "server.socket_host": "0.0.0.0",
-        'server.socket_port': args.port,
+        "server.socket_port": args.port,
-        'server.thread_pool': 5,
+        "server.thread_pool": 5,
-        'server.socket_host': '0.0.0.0',
+        "engine.autoreload.on": args.debug,
-        'server.show_tracebacks': args.debug,
+        "log.screen": True
-        'log.screen': False,
-        'engine.autoreload.on': args.debug
     })
 
     def signal_handler(signum, stack):