initial commit

.dockerignore

@@ -0,0 +1,9 @@
+apt/*
+data/*
+dist/*
+out/*
+out2/*
+testenv/*
+testenv2/*
+test/*
+aptly/*

.gitignore

@@ -0,0 +1,7 @@
+/testenv
+/test
+__pycache__
+/repobot.egg-info
+/build
+/dist
+/repos.db*

Dockerfile

@@ -0,0 +1,21 @@
+FROM ubuntu:bionic
+
+RUN apt-get update && \
+    apt-get install -y python3-pip gpgv1 gnupg1 gpg sudo wget
+
+RUN cd /tmp && \
+    wget -qO aptly.tgz https://bintray.com/artifact/download/smira/aptly/aptly_1.3.0_linux_amd64.tar.gz && \
+    tar xvf aptly.tgz aptly_1.3.0_linux_amd64/aptly && \
+    mv aptly_1.3.0_linux_amd64/aptly /usr/bin/ && \
+    rm -rf aptly.tgz aptly_1.3.0_linux_amd64
+
+ADD . /tmp/code
+
+RUN cd /tmp/code && \
+    python3 setup.py install && \
+    useradd repobot && \
+    rm -rf /tmp/code
+
+ADD start /start
+
+ENTRYPOINT ["/start"]

README.md

@@ -0,0 +1,60 @@
+docker-artifact
+===============
+
+Software repository server
+
+Artifact provides an HTTP API for repository management. Currently, Python and Apt repositories are supported.
+
+
+Quickstart
+----------
+
+* Pull or build the image
+* `docker run -it --rm -v /some/host/dir:/data -p 80:8080 artifact`
+
+Persistent data will be placed in `/data`. The webserver will listen on port 8080 by default.
+
+
+Examples
+--------
+
+Upload python package:
+
+`curl -vv -F 'f=@pyircbot-4.0.0.post3-py3.5.egg' 'http://host/addpkg?provider=pypi&reponame=main&name=pyircbot&version=4.0.0'`
+
+
+Install python packages:
+
+`pip3 install -f http://host/repo/pypi/main/ --trusted-host host repobot`
+
+
+Upload apt package:
+
+`curl -vv -F 'f=@extpython-python3.7_3.7.0_amd64.deb' 'http://host/addpkg?provider=apt&reponame=main&name=extpython-python3.7&version=3.7.0&dist=bionic'`
+
+
+Install apt packages:
+
+```
+wget -qO- http://host/repo/apt/main/repo.key | apt-key add - && \
+echo "deb http://host/repo/apt/main bionic main" | tee -a /etc/apt/sources.list && \
+apt-get update && \
+apt-get install -y extpython-python3.6
+```
+
+
+Notes
+-----
+
+* Repos are created automatically when a package is added to them.
+* Repo URLs are structured as: `/repo/<provider>/<name>`. Deeper URLs are handled directly by the provider.
+* The apt provider will generate a gpg key per repo upon repo creation
+
+
+Todo
+----
+
+* Auth
+* Delete packages
+* Human-readable package listing
+* Support using existing GPG keys

repobot/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

repobot/common.py

@@ -0,0 +1,10 @@
+import persistent.list
+import persistent.mapping
+
+
+def plist():
+    return persistent.list.PersistentList()
+
+
+def pmap():
+    return persistent.mapping.PersistentMapping()

repobot/provider.py

@@ -0,0 +1,199 @@
+import os
+import shutil
+from repobot.common import plist, pmap
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+import cherrypy
+
+
+class PkgProvider(object):
+    def __init__(self, db, repo, datadir):
+        """
+        Base package provider class
+        """
+        self.db = db
+        self.repo = repo
+        self.dir = datadir
+
+    def render(self):
+        """
+        Respond to requests to browse the repo
+        """
+        raise NotImplementedError()
+
+    def add_package(self, pkobj, fname, fobj, params):
+        """
+        Add a package to the repo
+        """
+        raise NotImplementedError()
+
+
+class PyPiProvider(PkgProvider):
+    def add_package(self, pkgobj, fname, fobj, params):
+        if "files" not in pkgobj.data:
+            pkgobj.data["files"] = plist()
+
+        if fname in pkgobj.data["files"]:
+            raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version))
+
+        pkgdir = os.path.join(self.dir, pkgobj.name)
+        os.makedirs(pkgdir, exist_ok=True)
+        # TODO handle duplicate files better
+        pkgfilepath = os.path.join(pkgdir, fname)
+
+        with open(pkgfilepath, "wb") as fdest:
+            shutil.copyfileobj(fobj, fdest)
+
+        pkgobj.data["files"].append(fname)
+
+    def browse(self, args):
+        tpl = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape(['html', 'xml']))
+        if len(args) == 0:  # repo root
+            return tpl.get_template("pypi/root.html"). \
+                render(reponame=self.repo.name,
+                       packages=self.repo.packages.keys())
+        elif len(args) == 1:  # single module dir
+            files = []
+            if args[0] not in self.repo.packages:
+                raise cherrypy.HTTPError(404, 'Invalid package')
+            for _, version in self.repo.packages[args[0]].items():
+                files += version.data["files"]
+            return tpl.get_template("pypi/project.html"). \
+                render(reponame=self.repo.name,
+                       modulename=args[0],
+                       files=files)
+        elif len(args) == 2:  # fetch file
+            fpath = os.path.join(self.dir, args[0], args[1])
+            return cherrypy.lib.static.serve_file(os.path.abspath(fpath), "application/octet-stream")
+
+
+from subprocess import check_call, check_output, Popen, PIPE
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+import json
+
+
+class AptlyConfig(object):
+    """
+    Context manager providing an aptly config file
+    """
+    def __init__(self, rootdir):
+        self.conf = {"rootDir": rootdir}  # , "gpgDisableSign": True, "gpgDisableVerify": True}
+        self.file = None
+
+    def __enter__(self):
+        self.file = NamedTemporaryFile()
+        with open(self.file.name, "w") as f:
+            f.write(json.dumps(self.conf))
+        return self.file.name
+
+    def __exit__(self, *args):
+        self.file.close()
+
+
+class AptProvider(PkgProvider):
+    def add_package(self, pkgobj, fname, fobj, params):
+        # first package added sets the Distribution of the repo
+        # subsequent package add MUST specify the same dist
+        if "dist" not in self.repo.data:
+            self.repo.data["dist"] = params["dist"]
+        assert self.repo.data["dist"] == params["dist"]
+
+        # Generate a GPG key to sign packages in this repo
+        # TODO support passing keypath=... param to import existing keys and maybe other key generation options
+        if not os.path.exists(self._gpg_dir):
+            self._generate_gpg_key()
+
+        if "files" not in pkgobj.data:
+            pkgobj.data["files"] = plist()
+        if fname in pkgobj.data["files"]:
+            # raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version))
+            pass
+
+        with AptlyConfig(self.dir) as conf:
+            if not os.path.exists(os.path.join(self.dir, "db")):
+                os.makedirs(self.dir, exist_ok=True)
+                check_call(["aptly", "-config", conf, "repo", "create",
+                            "-distribution", self.repo.data["dist"], "main"])  # TODO dist param
+            # put the file somewhere for now
+            with TemporaryDirectory() as tdir:
+                tmppkgpath = os.path.join(tdir, fname)
+                with open(tmppkgpath, "wb") as fdest:
+                    shutil.copyfileobj(fobj, fdest)
+                check_call(["aptly", "-config", conf, "repo", "add", "main", tmppkgpath])
+            if not os.path.exists(os.path.join(self.dir, "public")):
+                check_call(["aptly", "-config", conf, "publish", "repo", "main"],
+                           env=self._env)
+            else:
+                check_call(["aptly", "-config", conf, "publish", "update",
+                            "-force-overwrite", self.repo.data["dist"]],
+                           env=self._env)
+
+        # Make the public key available for clients
+        self._export_pubkey()
+
+        pkgobj.data["files"].append(fname)
+
+        # TODO validate deb file name version against user passed version
+
+    def browse(self, args):
+        fpath = os.path.abspath(os.path.join(self.dir, "public", *args))
+        if not os.path.exists(fpath):
+            raise cherrypy.HTTPError(404)
+        return cherrypy.lib.static.serve_file(fpath)
+
+    def _generate_gpg_key(self):
+        """
+        Generate a GPG key for signing packages in this repo. Because only gpg2 supports unattended generation of
+        passwordless keys we generate the key with gpg2 then export/import it into gpg1.
+        """
+        # Generate the key
+        os.makedirs(self._gpg_dir)
+        proc = Popen(["gpg", "--batch", "--gen-key"], stdin=PIPE, env=self._env)
+        proc.stdin.write("""%no-protection
+Key-Type: rsa
+Key-Length: 1024
+Subkey-Type: default
+Subkey-Length: 1024
+Name-Real: Apt Master
+Name-Comment: Apt signing key
+Name-Email: aptmaster@localhost
+Expire-Date: 0
+%commit""".encode("ascii"))
+        proc.stdin.close()
+        proc.wait()
+        assert proc.returncode == 0
+
+        # Export the private key
+        keydata = check_output(["gpg", "--export-secret-key", "--armor", "aptmaster@localhost"], env=self._env)
+        shutil.rmtree(self._gpg_dir)
+        os.makedirs(self._gpg_dir)
+
+        # Import the private key
+        proc = Popen(["gpg1", "--import"], stdin=PIPE, env=self._env)
+        proc.stdin.write(keydata)
+        proc.stdin.close()
+        proc.wait()
+        assert proc.returncode == 0
+
+    def _export_pubkey(self):
+        keypath = os.path.join(self.dir, "public", "repo.key")
+        if not os.path.exists(keypath):
+            keydata = check_output(["gpg", "--export", "--armor", "aptmaster@localhost"], env=self._env)
+            with open(keypath, "wb") as f:
+                f.write(keydata)
+
+    @property
+    def _env(self):
+        """
+        Return env vars to be used for subprocesses of this module
+        """
+        print(os.environ["PATH"])
+        return {"GNUPGHOME": self._gpg_dir,
+                "PATH": os.environ["PATH"]}
+
+    @property
+    def _gpg_dir(self):
+        return os.path.join(self.dir, "gpg")
+
+
+providers = {"pypi": PyPiProvider,
+             "apt": AptProvider}

repobot/repos.py

@@ -0,0 +1,61 @@
+import ZODB
+import ZODB.FileStorage
+import persistent
+import BTrees.OOBTree
+from repobot.provider import providers
+import os
+from repobot.common import plist, pmap
+
+
+class Repo(persistent.Persistent):
+    def __init__(self, name, provider):
+        self.name = name
+        self.provider = provider
+        self.packages = pmap()
+        self.data = pmap()
+
+    def get_package(self, name, version):
+        if name not in self.packages:
+            self.packages[name] = pmap()
+        if version not in self.packages[name]:
+            self.packages[name][version] = RepoPackage(name, version)
+        return self.packages[name][version]
+
+
+class RepoPackage(persistent.Persistent):
+    def __init__(self, name, version):
+        self.name = name
+        self.version = version
+        self.data = pmap()
+
+
+class RepoDb(object):
+    def __init__(self, db_path, data_root):
+        self.storage = ZODB.FileStorage.FileStorage(db_path)
+        self.db = ZODB.DB(self.storage)
+        self.data_root = data_root
+
+        with self.db.transaction() as c:
+            if "repos" not in c.root():
+                c.root.repos = BTrees.OOBTree.BTree()
+
+    def add_package(self, provider, reponame, pkgname, pkgversion, fname, fobj, params):
+        with self.db.transaction() as c:
+            repo = self._get_repo(c, provider, reponame)
+            datadir = os.path.join(self.data_root, provider, reponame)
+            provider = providers[repo.provider](self.db, repo, datadir)
+            provider.add_package(repo.get_package(pkgname, pkgversion), fname, fobj, params)
+
+    def _get_repo(self, c, provider, name):
+        if provider not in c.root.repos:
+            c.root.repos[provider] = pmap()
+        if name not in c.root.repos[provider]:
+            c.root.repos[provider][name] = Repo(name, provider)
+        return c.root.repos[provider][name]
+
+    def browse_repo(self, provider, reponame, args):
+        with self.db.transaction() as c:
+            repo = c.root.repos[provider][reponame]
+            datadir = os.path.join(self.data_root, provider, reponame)
+            provider = providers[repo.provider](self.db, repo, datadir)
+            return provider.browse(args)

repobot/server.py

@@ -0,0 +1,99 @@
+import cherrypy
+import logging
+from repobot.repos import RepoDb
+
+
+class AppWeb(object):
+    def __init__(self, db):
+        self.db = db
+
+    @cherrypy.expose
+    def addpkg(self, provider, reponame, name, version, f, **params):
+        self.db.add_package(provider, reponame, name, version, f.filename, f.file, params)
+
+    @cherrypy.expose
+    def repo(self, provider, repo, *args):
+        return self.db.browse_repo(provider, repo, args)
+
+
+class FlatDispatch(cherrypy.dispatch.Dispatcher):
+    def __init__(self, method):
+        """
+        Route all sub urls of this one to the single passed method
+        """
+        super().__init__(self)
+        self.method = method
+
+    def find_handler(self, path):
+        # Hack, it does not respect settings of parent nodes
+        cherrypy.serving.request.config = cherrypy.config
+        return self.method, [i for i in filter(lambda o: len(o) > 0, path.split("/")[2:])]
+
+
+def main():
+    import argparse
+    import signal
+
+    parser = argparse.ArgumentParser(description="Repobot daemon")
+    parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on")
+    parser.add_argument('-s', '--database', default="./repos.db", help="path to persistent database")
+    parser.add_argument('-d', '--data-root', default="./data/", help="data storage dir")
+    parser.add_argument('--debug', action="store_true", help="enable development options")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
+                        format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
+
+    db = RepoDb(args.database, args.data_root)
+
+    web = AppWeb(db)
+
+    def validate_password(realm, username, password):
+        s = library.session()
+        if s.query(User).filter(User.name == username, User.password == pwhash(password)).first():
+            return True
+        return False
+
+    cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
+                                         # 'error_page.403': web.error,
+                                         # 'error_page.404': web.error
+                                         },
+                                   '/repo': {'request.dispatch': FlatDispatch(web.repo)},
+                                   #'/static': {"tools.staticdir.on": True,
+                                   #            "tools.staticdir.dir": os.path.join(APPROOT, "styles/dist")
+                                   #            if not args.debug else os.path.abspath("styles/dist")},
+                                   '/login': {'tools.auth_basic.on': True,
+                                              'tools.auth_basic.realm': 'webapp',
+                                              'tools.auth_basic.checkpassword': validate_password}})
+
+    cherrypy.config.update({
+        'tools.sessions.on': True,
+        'tools.sessions.locking': 'explicit',
+        'tools.sessions.timeout': 525600,
+        'request.show_tracebacks': True,
+        'server.socket_port': args.port,
+        'server.thread_pool': 25,
+        'server.socket_host': '0.0.0.0',
+        'server.show_tracebacks': True,
+        'log.screen': False,
+        'engine.autoreload.on': args.debug
+    })
+
+    def signal_handler(signum, stack):
+        logging.critical('Got sig {}, exiting...'.format(signum))
+        cherrypy.engine.exit()
+
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+    try:
+        cherrypy.engine.start()
+        cherrypy.engine.block()
+    finally:
+        logging.info("API has shut down")
+        cherrypy.engine.exit()
+
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+from setuptools import setup
+from repobot import __version__
+
+
+setup(name='repobot',
+      version=__version__,
+      description='server for build artifact storage',
+      url='',
+      author='dpedu',
+      author_email='dave@davepedu.com',
+      packages=['repobot'],
+      entry_points={
+          "console_scripts": [
+              "repobotd = repobot.server:main"
+          ]
+      },
+      include_package_data=True,
+      package_data={'repobot': ['../templates/pypi/*.html']},
+      install_requires=[
+          'ZODB==5.5.0',
+          'CherryPy==18.0.1',
+          'Jinja2==2.10'
+      ],
+      zip_safe=False)

start

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -x
+
+chown -R repobot:repobot /data
+install -d /data/db -o repobot -g repobot || true
+
+exec sudo -Hu repobot repobotd --data-root /data --database /data/db/repos.db $@

templates/pypi/project.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Simple index</title>
+    <style type="text/css">
+        a {
+          display: block;
+        }
+    </style>
+  </head>
+  <body>
+    {%- for file in files %}
+    <a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a>
+    {%- endfor %}
+  </body>
+</html>

templates/pypi/root.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Simple index</title>
+    <style type="text/css">
+        a {
+          display: block;
+        }
+    </style>
+  </head>
+  <body>
+    {%- for package in packages %}
+    <a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a>
+    {%- endfor %}
+  </body>
+</html>