From 185b638741b51eee4d95cf731561d336db27d49b Mon Sep 17 00:00:00 2001 From: dave Date: Sun, 21 Oct 2018 16:37:53 -0700 Subject: [PATCH] initial commit --- .dockerignore | 9 ++ .gitignore | 7 ++ Dockerfile | 21 ++++ README.md | 60 +++++++++++ repobot/__init__.py | 1 + repobot/common.py | 10 ++ repobot/provider.py | 199 ++++++++++++++++++++++++++++++++++++ repobot/repos.py | 61 +++++++++++ repobot/server.py | 99 ++++++++++++++++++ setup.py | 26 +++++ start | 7 ++ templates/mkdir | 0 templates/pypi/project.html | 16 +++ templates/pypi/root.html | 16 +++ 14 files changed, 532 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 repobot/__init__.py create mode 100644 repobot/common.py create mode 100644 repobot/provider.py create mode 100644 repobot/repos.py create mode 100644 repobot/server.py create mode 100644 setup.py create mode 100755 start create mode 100644 templates/mkdir create mode 100644 templates/pypi/project.html create mode 100644 templates/pypi/root.html diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2fd5f07 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +apt/* +data/* +dist/* +out/* +out2/* +testenv/* +testenv2/* +test/* +aptly/* diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8b91784 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/testenv +/test +__pycache__ +/repobot.egg-info +/build +/dist +/repos.db* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..53a79aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:bionic + +RUN apt-get update && \ + apt-get install -y python3-pip gpgv1 gnupg1 gpg sudo wget + +RUN cd /tmp && \ + wget -qO aptly.tgz https://bintray.com/artifact/download/smira/aptly/aptly_1.3.0_linux_amd64.tar.gz && \ + tar xvf aptly.tgz aptly_1.3.0_linux_amd64/aptly && \ + mv aptly_1.3.0_linux_amd64/aptly /usr/bin/ && \ + rm -rf aptly.tgz aptly_1.3.0_linux_amd64 + +ADD . /tmp/code + +RUN cd /tmp/code && \ + python3 setup.py install && \ + useradd repobot && \ + rm -rf /tmp/code + +ADD start /start + +ENTRYPOINT ["/start"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..0bf00ee --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +docker-artifact +=============== + +Software repository server + +Artifact provides an HTTP API for repository management. Currently, Python and Apt repositories are supported. + + +Quickstart +---------- + +* Pull or build the image +* `docker run -it --rm -v /some/host/dir:/data -p 80:8080 artifact` + +Persistent data will be placed in `/data`. The webserver will listen on port 8080 by default. + + +Examples +-------- + +Upload python package: + +`curl -vv -F 'f=@pyircbot-4.0.0.post3-py3.5.egg' 'http://host/addpkg?provider=pypi&reponame=main&name=pyircbot&version=4.0.0'` + + +Install python packages: + +`pip3 install -f http://host/repo/pypi/main/ --trusted-host host repobot` + + +Upload apt package: + +`curl -vv -F 'f=@extpython-python3.7_3.7.0_amd64.deb' 'http://host/addpkg?provider=apt&reponame=main&name=extpython-python3.7&version=3.7.0&dist=bionic'` + + +Install apt packages: + +``` +wget -qO- http://host/repo/apt/main/repo.key | apt-key add - && \ +echo "deb http://host/repo/apt/main bionic main" | tee -a /etc/apt/sources.list && \ +apt-get update && \ +apt-get install -y extpython-python3.6 +``` + + +Notes +----- + +* Repos are created automatically when a package is added to them. +* Repo URLs are structured as: `/repo//`. Deeper URLs are handled directly by the provider. +* The apt provider will generate a gpg key per repo upon repo creation + + +Todo +---- + +* Auth +* Delete packages +* Human-readable package listing +* Support using existing GPG keys diff --git a/repobot/__init__.py b/repobot/__init__.py new file mode 100644 index 0000000..f102a9c --- /dev/null +++ b/repobot/__init__.py @@ -0,0 +1 @@ +__version__ = "0.0.1" diff --git a/repobot/common.py b/repobot/common.py new file mode 100644 index 0000000..ab8932d --- /dev/null +++ b/repobot/common.py @@ -0,0 +1,10 @@ +import persistent.list +import persistent.mapping + + +def plist(): + return persistent.list.PersistentList() + + +def pmap(): + return persistent.mapping.PersistentMapping() diff --git a/repobot/provider.py b/repobot/provider.py new file mode 100644 index 0000000..14ad1af --- /dev/null +++ b/repobot/provider.py @@ -0,0 +1,199 @@ +import os +import shutil +from repobot.common import plist, pmap +from jinja2 import Environment, FileSystemLoader, select_autoescape +import cherrypy + + +class PkgProvider(object): + def __init__(self, db, repo, datadir): + """ + Base package provider class + """ + self.db = db + self.repo = repo + self.dir = datadir + + def render(self): + """ + Respond to requests to browse the repo + """ + raise NotImplementedError() + + def add_package(self, pkobj, fname, fobj, params): + """ + Add a package to the repo + """ + raise NotImplementedError() + + +class PyPiProvider(PkgProvider): + def add_package(self, pkgobj, fname, fobj, params): + if "files" not in pkgobj.data: + pkgobj.data["files"] = plist() + + if fname in pkgobj.data["files"]: + raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version)) + + pkgdir = os.path.join(self.dir, pkgobj.name) + os.makedirs(pkgdir, exist_ok=True) + # TODO handle duplicate files better + pkgfilepath = os.path.join(pkgdir, fname) + + with open(pkgfilepath, "wb") as fdest: + shutil.copyfileobj(fobj, fdest) + + pkgobj.data["files"].append(fname) + + def browse(self, args): + tpl = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape(['html', 'xml'])) + if len(args) == 0: # repo root + return tpl.get_template("pypi/root.html"). \ + render(reponame=self.repo.name, + packages=self.repo.packages.keys()) + elif len(args) == 1: # single module dir + files = [] + if args[0] not in self.repo.packages: + raise cherrypy.HTTPError(404, 'Invalid package') + for _, version in self.repo.packages[args[0]].items(): + files += version.data["files"] + return tpl.get_template("pypi/project.html"). \ + render(reponame=self.repo.name, + modulename=args[0], + files=files) + elif len(args) == 2: # fetch file + fpath = os.path.join(self.dir, args[0], args[1]) + return cherrypy.lib.static.serve_file(os.path.abspath(fpath), "application/octet-stream") + + +from subprocess import check_call, check_output, Popen, PIPE +from tempfile import NamedTemporaryFile, TemporaryDirectory +import json + + +class AptlyConfig(object): + """ + Context manager providing an aptly config file + """ + def __init__(self, rootdir): + self.conf = {"rootDir": rootdir} # , "gpgDisableSign": True, "gpgDisableVerify": True} + self.file = None + + def __enter__(self): + self.file = NamedTemporaryFile() + with open(self.file.name, "w") as f: + f.write(json.dumps(self.conf)) + return self.file.name + + def __exit__(self, *args): + self.file.close() + + +class AptProvider(PkgProvider): + def add_package(self, pkgobj, fname, fobj, params): + # first package added sets the Distribution of the repo + # subsequent package add MUST specify the same dist + if "dist" not in self.repo.data: + self.repo.data["dist"] = params["dist"] + assert self.repo.data["dist"] == params["dist"] + + # Generate a GPG key to sign packages in this repo + # TODO support passing keypath=... param to import existing keys and maybe other key generation options + if not os.path.exists(self._gpg_dir): + self._generate_gpg_key() + + if "files" not in pkgobj.data: + pkgobj.data["files"] = plist() + if fname in pkgobj.data["files"]: + # raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version)) + pass + + with AptlyConfig(self.dir) as conf: + if not os.path.exists(os.path.join(self.dir, "db")): + os.makedirs(self.dir, exist_ok=True) + check_call(["aptly", "-config", conf, "repo", "create", + "-distribution", self.repo.data["dist"], "main"]) # TODO dist param + # put the file somewhere for now + with TemporaryDirectory() as tdir: + tmppkgpath = os.path.join(tdir, fname) + with open(tmppkgpath, "wb") as fdest: + shutil.copyfileobj(fobj, fdest) + check_call(["aptly", "-config", conf, "repo", "add", "main", tmppkgpath]) + if not os.path.exists(os.path.join(self.dir, "public")): + check_call(["aptly", "-config", conf, "publish", "repo", "main"], + env=self._env) + else: + check_call(["aptly", "-config", conf, "publish", "update", + "-force-overwrite", self.repo.data["dist"]], + env=self._env) + + # Make the public key available for clients + self._export_pubkey() + + pkgobj.data["files"].append(fname) + + # TODO validate deb file name version against user passed version + + def browse(self, args): + fpath = os.path.abspath(os.path.join(self.dir, "public", *args)) + if not os.path.exists(fpath): + raise cherrypy.HTTPError(404) + return cherrypy.lib.static.serve_file(fpath) + + def _generate_gpg_key(self): + """ + Generate a GPG key for signing packages in this repo. Because only gpg2 supports unattended generation of + passwordless keys we generate the key with gpg2 then export/import it into gpg1. + """ + # Generate the key + os.makedirs(self._gpg_dir) + proc = Popen(["gpg", "--batch", "--gen-key"], stdin=PIPE, env=self._env) + proc.stdin.write("""%no-protection +Key-Type: rsa +Key-Length: 1024 +Subkey-Type: default +Subkey-Length: 1024 +Name-Real: Apt Master +Name-Comment: Apt signing key +Name-Email: aptmaster@localhost +Expire-Date: 0 +%commit""".encode("ascii")) + proc.stdin.close() + proc.wait() + assert proc.returncode == 0 + + # Export the private key + keydata = check_output(["gpg", "--export-secret-key", "--armor", "aptmaster@localhost"], env=self._env) + shutil.rmtree(self._gpg_dir) + os.makedirs(self._gpg_dir) + + # Import the private key + proc = Popen(["gpg1", "--import"], stdin=PIPE, env=self._env) + proc.stdin.write(keydata) + proc.stdin.close() + proc.wait() + assert proc.returncode == 0 + + def _export_pubkey(self): + keypath = os.path.join(self.dir, "public", "repo.key") + if not os.path.exists(keypath): + keydata = check_output(["gpg", "--export", "--armor", "aptmaster@localhost"], env=self._env) + with open(keypath, "wb") as f: + f.write(keydata) + + @property + def _env(self): + """ + Return env vars to be used for subprocesses of this module + """ + print(os.environ["PATH"]) + return {"GNUPGHOME": self._gpg_dir, + "PATH": os.environ["PATH"]} + + @property + def _gpg_dir(self): + return os.path.join(self.dir, "gpg") + + +providers = {"pypi": PyPiProvider, + "apt": AptProvider} diff --git a/repobot/repos.py b/repobot/repos.py new file mode 100644 index 0000000..5d132fd --- /dev/null +++ b/repobot/repos.py @@ -0,0 +1,61 @@ +import ZODB +import ZODB.FileStorage +import persistent +import BTrees.OOBTree +from repobot.provider import providers +import os +from repobot.common import plist, pmap + + +class Repo(persistent.Persistent): + def __init__(self, name, provider): + self.name = name + self.provider = provider + self.packages = pmap() + self.data = pmap() + + def get_package(self, name, version): + if name not in self.packages: + self.packages[name] = pmap() + if version not in self.packages[name]: + self.packages[name][version] = RepoPackage(name, version) + return self.packages[name][version] + + +class RepoPackage(persistent.Persistent): + def __init__(self, name, version): + self.name = name + self.version = version + self.data = pmap() + + +class RepoDb(object): + def __init__(self, db_path, data_root): + self.storage = ZODB.FileStorage.FileStorage(db_path) + self.db = ZODB.DB(self.storage) + self.data_root = data_root + + with self.db.transaction() as c: + if "repos" not in c.root(): + c.root.repos = BTrees.OOBTree.BTree() + + def add_package(self, provider, reponame, pkgname, pkgversion, fname, fobj, params): + with self.db.transaction() as c: + repo = self._get_repo(c, provider, reponame) + datadir = os.path.join(self.data_root, provider, reponame) + provider = providers[repo.provider](self.db, repo, datadir) + provider.add_package(repo.get_package(pkgname, pkgversion), fname, fobj, params) + + def _get_repo(self, c, provider, name): + if provider not in c.root.repos: + c.root.repos[provider] = pmap() + if name not in c.root.repos[provider]: + c.root.repos[provider][name] = Repo(name, provider) + return c.root.repos[provider][name] + + def browse_repo(self, provider, reponame, args): + with self.db.transaction() as c: + repo = c.root.repos[provider][reponame] + datadir = os.path.join(self.data_root, provider, reponame) + provider = providers[repo.provider](self.db, repo, datadir) + return provider.browse(args) diff --git a/repobot/server.py b/repobot/server.py new file mode 100644 index 0000000..fe57a9c --- /dev/null +++ b/repobot/server.py @@ -0,0 +1,99 @@ +import cherrypy +import logging +from repobot.repos import RepoDb + + +class AppWeb(object): + def __init__(self, db): + self.db = db + + @cherrypy.expose + def addpkg(self, provider, reponame, name, version, f, **params): + self.db.add_package(provider, reponame, name, version, f.filename, f.file, params) + + @cherrypy.expose + def repo(self, provider, repo, *args): + return self.db.browse_repo(provider, repo, args) + + +class FlatDispatch(cherrypy.dispatch.Dispatcher): + def __init__(self, method): + """ + Route all sub urls of this one to the single passed method + """ + super().__init__(self) + self.method = method + + def find_handler(self, path): + # Hack, it does not respect settings of parent nodes + cherrypy.serving.request.config = cherrypy.config + return self.method, [i for i in filter(lambda o: len(o) > 0, path.split("/")[2:])] + + +def main(): + import argparse + import signal + + parser = argparse.ArgumentParser(description="Repobot daemon") + parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on") + parser.add_argument('-s', '--database', default="./repos.db", help="path to persistent database") + parser.add_argument('-d', '--data-root', default="./data/", help="data storage dir") + parser.add_argument('--debug', action="store_true", help="enable development options") + + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, + format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") + + db = RepoDb(args.database, args.data_root) + + web = AppWeb(db) + + def validate_password(realm, username, password): + s = library.session() + if s.query(User).filter(User.name == username, User.password == pwhash(password)).first(): + return True + return False + + cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False, + # 'error_page.403': web.error, + # 'error_page.404': web.error + }, + '/repo': {'request.dispatch': FlatDispatch(web.repo)}, + #'/static': {"tools.staticdir.on": True, + # "tools.staticdir.dir": os.path.join(APPROOT, "styles/dist") + # if not args.debug else os.path.abspath("styles/dist")}, + '/login': {'tools.auth_basic.on': True, + 'tools.auth_basic.realm': 'webapp', + 'tools.auth_basic.checkpassword': validate_password}}) + + cherrypy.config.update({ + 'tools.sessions.on': True, + 'tools.sessions.locking': 'explicit', + 'tools.sessions.timeout': 525600, + 'request.show_tracebacks': True, + 'server.socket_port': args.port, + 'server.thread_pool': 25, + 'server.socket_host': '0.0.0.0', + 'server.show_tracebacks': True, + 'log.screen': False, + 'engine.autoreload.on': args.debug + }) + + def signal_handler(signum, stack): + logging.critical('Got sig {}, exiting...'.format(signum)) + cherrypy.engine.exit() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + try: + cherrypy.engine.start() + cherrypy.engine.block() + finally: + logging.info("API has shut down") + cherrypy.engine.exit() + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..98f5be5 --- /dev/null +++ b/setup.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +from setuptools import setup +from repobot import __version__ + + +setup(name='repobot', + version=__version__, + description='server for build artifact storage', + url='', + author='dpedu', + author_email='dave@davepedu.com', + packages=['repobot'], + entry_points={ + "console_scripts": [ + "repobotd = repobot.server:main" + ] + }, + include_package_data=True, + package_data={'repobot': ['../templates/pypi/*.html']}, + install_requires=[ + 'ZODB==5.5.0', + 'CherryPy==18.0.1', + 'Jinja2==2.10' + ], + zip_safe=False) diff --git a/start b/start new file mode 100755 index 0000000..67ad9ae --- /dev/null +++ b/start @@ -0,0 +1,7 @@ +#!/bin/sh +set -x + +chown -R repobot:repobot /data +install -d /data/db -o repobot -g repobot || true + +exec sudo -Hu repobot repobotd --data-root /data --database /data/db/repos.db $@ diff --git a/templates/mkdir b/templates/mkdir new file mode 100644 index 0000000..e69de29 diff --git a/templates/pypi/project.html b/templates/pypi/project.html new file mode 100644 index 0000000..9056a98 --- /dev/null +++ b/templates/pypi/project.html @@ -0,0 +1,16 @@ + + + + Simple index + + + + {%- for file in files %} + {{ file }} + {%- endfor %} + + diff --git a/templates/pypi/root.html b/templates/pypi/root.html new file mode 100644 index 0000000..794ac22 --- /dev/null +++ b/templates/pypi/root.html @@ -0,0 +1,16 @@ + + + + Simple index + + + + {%- for package in packages %} + {{ package }} + {%- endfor %} + +