@@ -0,0 +1,9 @@ | |||
apt/* | |||
data/* | |||
dist/* | |||
out/* | |||
out2/* | |||
testenv/* | |||
testenv2/* | |||
test/* | |||
aptly/* |
@@ -0,0 +1,7 @@ | |||
/testenv | |||
/test | |||
__pycache__ | |||
/repobot.egg-info | |||
/build | |||
/dist | |||
/repos.db* |
@@ -0,0 +1,21 @@ | |||
FROM ubuntu:bionic | |||
RUN apt-get update && \ | |||
apt-get install -y python3-pip gpgv1 gnupg1 gpg sudo wget | |||
RUN cd /tmp && \ | |||
wget -qO aptly.tgz https://bintray.com/artifact/download/smira/aptly/aptly_1.3.0_linux_amd64.tar.gz && \ | |||
tar xvf aptly.tgz aptly_1.3.0_linux_amd64/aptly && \ | |||
mv aptly_1.3.0_linux_amd64/aptly /usr/bin/ && \ | |||
rm -rf aptly.tgz aptly_1.3.0_linux_amd64 | |||
ADD . /tmp/code | |||
RUN cd /tmp/code && \ | |||
python3 setup.py install && \ | |||
useradd repobot && \ | |||
rm -rf /tmp/code | |||
ADD start /start | |||
ENTRYPOINT ["/start"] |
@@ -0,0 +1,60 @@ | |||
docker-artifact | |||
=============== | |||
Software repository server | |||
Artifact provides an HTTP API for repository management. Currently, Python and Apt repositories are supported. | |||
Quickstart | |||
---------- | |||
* Pull or build the image | |||
* `docker run -it --rm -v /some/host/dir:/data -p 80:8080 artifact` | |||
Persistent data will be placed in `/data`. The webserver will listen on port 8080 by default. | |||
Examples | |||
-------- | |||
Upload python package: | |||
`curl -vv -F 'f=@pyircbot-4.0.0.post3-py3.5.egg' 'http://host/addpkg?provider=pypi&reponame=main&name=pyircbot&version=4.0.0'` | |||
Install python packages: | |||
`pip3 install -f http://host/repo/pypi/main/ --trusted-host host repobot` | |||
Upload apt package: | |||
`curl -vv -F 'f=@extpython-python3.7_3.7.0_amd64.deb' 'http://host/addpkg?provider=apt&reponame=main&name=extpython-python3.7&version=3.7.0&dist=bionic'` | |||
Install apt packages: | |||
``` | |||
wget -qO- http://host/repo/apt/main/repo.key | apt-key add - && \ | |||
echo "deb http://host/repo/apt/main bionic main" | tee -a /etc/apt/sources.list && \ | |||
apt-get update && \ | |||
apt-get install -y extpython-python3.6 | |||
``` | |||
Notes | |||
----- | |||
* Repos are created automatically when a package is added to them. | |||
* Repo URLs are structured as: `/repo/<provider>/<name>`. Deeper URLs are handled directly by the provider. | |||
* The apt provider will generate a gpg key per repo upon repo creation | |||
Todo | |||
---- | |||
* Auth | |||
* Delete packages | |||
* Human-readable package listing | |||
* Support using existing GPG keys |
@@ -0,0 +1 @@ | |||
__version__ = "0.0.1" |
@@ -0,0 +1,10 @@ | |||
import persistent.list | |||
import persistent.mapping | |||
def plist(): | |||
return persistent.list.PersistentList() | |||
def pmap(): | |||
return persistent.mapping.PersistentMapping() |
@@ -0,0 +1,199 @@ | |||
import os | |||
import shutil | |||
from repobot.common import plist, pmap | |||
from jinja2 import Environment, FileSystemLoader, select_autoescape | |||
import cherrypy | |||
class PkgProvider(object): | |||
def __init__(self, db, repo, datadir): | |||
""" | |||
Base package provider class | |||
""" | |||
self.db = db | |||
self.repo = repo | |||
self.dir = datadir | |||
def render(self): | |||
""" | |||
Respond to requests to browse the repo | |||
""" | |||
raise NotImplementedError() | |||
def add_package(self, pkobj, fname, fobj, params): | |||
""" | |||
Add a package to the repo | |||
""" | |||
raise NotImplementedError() | |||
class PyPiProvider(PkgProvider): | |||
def add_package(self, pkgobj, fname, fobj, params): | |||
if "files" not in pkgobj.data: | |||
pkgobj.data["files"] = plist() | |||
if fname in pkgobj.data["files"]: | |||
raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version)) | |||
pkgdir = os.path.join(self.dir, pkgobj.name) | |||
os.makedirs(pkgdir, exist_ok=True) | |||
# TODO handle duplicate files better | |||
pkgfilepath = os.path.join(pkgdir, fname) | |||
with open(pkgfilepath, "wb") as fdest: | |||
shutil.copyfileobj(fobj, fdest) | |||
pkgobj.data["files"].append(fname) | |||
def browse(self, args): | |||
tpl = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape(['html', 'xml'])) | |||
if len(args) == 0: # repo root | |||
return tpl.get_template("pypi/root.html"). \ | |||
render(reponame=self.repo.name, | |||
packages=self.repo.packages.keys()) | |||
elif len(args) == 1: # single module dir | |||
files = [] | |||
if args[0] not in self.repo.packages: | |||
raise cherrypy.HTTPError(404, 'Invalid package') | |||
for _, version in self.repo.packages[args[0]].items(): | |||
files += version.data["files"] | |||
return tpl.get_template("pypi/project.html"). \ | |||
render(reponame=self.repo.name, | |||
modulename=args[0], | |||
files=files) | |||
elif len(args) == 2: # fetch file | |||
fpath = os.path.join(self.dir, args[0], args[1]) | |||
return cherrypy.lib.static.serve_file(os.path.abspath(fpath), "application/octet-stream") | |||
from subprocess import check_call, check_output, Popen, PIPE | |||
from tempfile import NamedTemporaryFile, TemporaryDirectory | |||
import json | |||
class AptlyConfig(object): | |||
""" | |||
Context manager providing an aptly config file | |||
""" | |||
def __init__(self, rootdir): | |||
self.conf = {"rootDir": rootdir} # , "gpgDisableSign": True, "gpgDisableVerify": True} | |||
self.file = None | |||
def __enter__(self): | |||
self.file = NamedTemporaryFile() | |||
with open(self.file.name, "w") as f: | |||
f.write(json.dumps(self.conf)) | |||
return self.file.name | |||
def __exit__(self, *args): | |||
self.file.close() | |||
class AptProvider(PkgProvider): | |||
def add_package(self, pkgobj, fname, fobj, params): | |||
# first package added sets the Distribution of the repo | |||
# subsequent package add MUST specify the same dist | |||
if "dist" not in self.repo.data: | |||
self.repo.data["dist"] = params["dist"] | |||
assert self.repo.data["dist"] == params["dist"] | |||
# Generate a GPG key to sign packages in this repo | |||
# TODO support passing keypath=... param to import existing keys and maybe other key generation options | |||
if not os.path.exists(self._gpg_dir): | |||
self._generate_gpg_key() | |||
if "files" not in pkgobj.data: | |||
pkgobj.data["files"] = plist() | |||
if fname in pkgobj.data["files"]: | |||
# raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version)) | |||
pass | |||
with AptlyConfig(self.dir) as conf: | |||
if not os.path.exists(os.path.join(self.dir, "db")): | |||
os.makedirs(self.dir, exist_ok=True) | |||
check_call(["aptly", "-config", conf, "repo", "create", | |||
"-distribution", self.repo.data["dist"], "main"]) # TODO dist param | |||
# put the file somewhere for now | |||
with TemporaryDirectory() as tdir: | |||
tmppkgpath = os.path.join(tdir, fname) | |||
with open(tmppkgpath, "wb") as fdest: | |||
shutil.copyfileobj(fobj, fdest) | |||
check_call(["aptly", "-config", conf, "repo", "add", "main", tmppkgpath]) | |||
if not os.path.exists(os.path.join(self.dir, "public")): | |||
check_call(["aptly", "-config", conf, "publish", "repo", "main"], | |||
env=self._env) | |||
else: | |||
check_call(["aptly", "-config", conf, "publish", "update", | |||
"-force-overwrite", self.repo.data["dist"]], | |||
env=self._env) | |||
# Make the public key available for clients | |||
self._export_pubkey() | |||
pkgobj.data["files"].append(fname) | |||
# TODO validate deb file name version against user passed version | |||
def browse(self, args): | |||
fpath = os.path.abspath(os.path.join(self.dir, "public", *args)) | |||
if not os.path.exists(fpath): | |||
raise cherrypy.HTTPError(404) | |||
return cherrypy.lib.static.serve_file(fpath) | |||
def _generate_gpg_key(self): | |||
""" | |||
Generate a GPG key for signing packages in this repo. Because only gpg2 supports unattended generation of | |||
passwordless keys we generate the key with gpg2 then export/import it into gpg1. | |||
""" | |||
# Generate the key | |||
os.makedirs(self._gpg_dir) | |||
proc = Popen(["gpg", "--batch", "--gen-key"], stdin=PIPE, env=self._env) | |||
proc.stdin.write("""%no-protection | |||
Key-Type: rsa | |||
Key-Length: 1024 | |||
Subkey-Type: default | |||
Subkey-Length: 1024 | |||
Name-Real: Apt Master | |||
Name-Comment: Apt signing key | |||
Name-Email: aptmaster@localhost | |||
Expire-Date: 0 | |||
%commit""".encode("ascii")) | |||
proc.stdin.close() | |||
proc.wait() | |||
assert proc.returncode == 0 | |||
# Export the private key | |||
keydata = check_output(["gpg", "--export-secret-key", "--armor", "aptmaster@localhost"], env=self._env) | |||
shutil.rmtree(self._gpg_dir) | |||
os.makedirs(self._gpg_dir) | |||
# Import the private key | |||
proc = Popen(["gpg1", "--import"], stdin=PIPE, env=self._env) | |||
proc.stdin.write(keydata) | |||
proc.stdin.close() | |||
proc.wait() | |||
assert proc.returncode == 0 | |||
def _export_pubkey(self): | |||
keypath = os.path.join(self.dir, "public", "repo.key") | |||
if not os.path.exists(keypath): | |||
keydata = check_output(["gpg", "--export", "--armor", "aptmaster@localhost"], env=self._env) | |||
with open(keypath, "wb") as f: | |||
f.write(keydata) | |||
@property | |||
def _env(self): | |||
""" | |||
Return env vars to be used for subprocesses of this module | |||
""" | |||
print(os.environ["PATH"]) | |||
return {"GNUPGHOME": self._gpg_dir, | |||
"PATH": os.environ["PATH"]} | |||
@property | |||
def _gpg_dir(self): | |||
return os.path.join(self.dir, "gpg") | |||
providers = {"pypi": PyPiProvider, | |||
"apt": AptProvider} |
@@ -0,0 +1,61 @@ | |||
import ZODB | |||
import ZODB.FileStorage | |||
import persistent | |||
import BTrees.OOBTree | |||
from repobot.provider import providers | |||
import os | |||
from repobot.common import plist, pmap | |||
class Repo(persistent.Persistent): | |||
def __init__(self, name, provider): | |||
self.name = name | |||
self.provider = provider | |||
self.packages = pmap() | |||
self.data = pmap() | |||
def get_package(self, name, version): | |||
if name not in self.packages: | |||
self.packages[name] = pmap() | |||
if version not in self.packages[name]: | |||
self.packages[name][version] = RepoPackage(name, version) | |||
return self.packages[name][version] | |||
class RepoPackage(persistent.Persistent): | |||
def __init__(self, name, version): | |||
self.name = name | |||
self.version = version | |||
self.data = pmap() | |||
class RepoDb(object): | |||
def __init__(self, db_path, data_root): | |||
self.storage = ZODB.FileStorage.FileStorage(db_path) | |||
self.db = ZODB.DB(self.storage) | |||
self.data_root = data_root | |||
with self.db.transaction() as c: | |||
if "repos" not in c.root(): | |||
c.root.repos = BTrees.OOBTree.BTree() | |||
def add_package(self, provider, reponame, pkgname, pkgversion, fname, fobj, params): | |||
with self.db.transaction() as c: | |||
repo = self._get_repo(c, provider, reponame) | |||
datadir = os.path.join(self.data_root, provider, reponame) | |||
provider = providers[repo.provider](self.db, repo, datadir) | |||
provider.add_package(repo.get_package(pkgname, pkgversion), fname, fobj, params) | |||
def _get_repo(self, c, provider, name): | |||
if provider not in c.root.repos: | |||
c.root.repos[provider] = pmap() | |||
if name not in c.root.repos[provider]: | |||
c.root.repos[provider][name] = Repo(name, provider) | |||
return c.root.repos[provider][name] | |||
def browse_repo(self, provider, reponame, args): | |||
with self.db.transaction() as c: | |||
repo = c.root.repos[provider][reponame] | |||
datadir = os.path.join(self.data_root, provider, reponame) | |||
provider = providers[repo.provider](self.db, repo, datadir) | |||
return provider.browse(args) |
@@ -0,0 +1,99 @@ | |||
import cherrypy | |||
import logging | |||
from repobot.repos import RepoDb | |||
class AppWeb(object): | |||
def __init__(self, db): | |||
self.db = db | |||
@cherrypy.expose | |||
def addpkg(self, provider, reponame, name, version, f, **params): | |||
self.db.add_package(provider, reponame, name, version, f.filename, f.file, params) | |||
@cherrypy.expose | |||
def repo(self, provider, repo, *args): | |||
return self.db.browse_repo(provider, repo, args) | |||
class FlatDispatch(cherrypy.dispatch.Dispatcher): | |||
def __init__(self, method): | |||
""" | |||
Route all sub urls of this one to the single passed method | |||
""" | |||
super().__init__(self) | |||
self.method = method | |||
def find_handler(self, path): | |||
# Hack, it does not respect settings of parent nodes | |||
cherrypy.serving.request.config = cherrypy.config | |||
return self.method, [i for i in filter(lambda o: len(o) > 0, path.split("/")[2:])] | |||
def main(): | |||
import argparse | |||
import signal | |||
parser = argparse.ArgumentParser(description="Repobot daemon") | |||
parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on") | |||
parser.add_argument('-s', '--database', default="./repos.db", help="path to persistent database") | |||
parser.add_argument('-d', '--data-root', default="./data/", help="data storage dir") | |||
parser.add_argument('--debug', action="store_true", help="enable development options") | |||
args = parser.parse_args() | |||
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, | |||
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") | |||
db = RepoDb(args.database, args.data_root) | |||
web = AppWeb(db) | |||
def validate_password(realm, username, password): | |||
s = library.session() | |||
if s.query(User).filter(User.name == username, User.password == pwhash(password)).first(): | |||
return True | |||
return False | |||
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False, | |||
# 'error_page.403': web.error, | |||
# 'error_page.404': web.error | |||
}, | |||
'/repo': {'request.dispatch': FlatDispatch(web.repo)}, | |||
#'/static': {"tools.staticdir.on": True, | |||
# "tools.staticdir.dir": os.path.join(APPROOT, "styles/dist") | |||
# if not args.debug else os.path.abspath("styles/dist")}, | |||
'/login': {'tools.auth_basic.on': True, | |||
'tools.auth_basic.realm': 'webapp', | |||
'tools.auth_basic.checkpassword': validate_password}}) | |||
cherrypy.config.update({ | |||
'tools.sessions.on': True, | |||
'tools.sessions.locking': 'explicit', | |||
'tools.sessions.timeout': 525600, | |||
'request.show_tracebacks': True, | |||
'server.socket_port': args.port, | |||
'server.thread_pool': 25, | |||
'server.socket_host': '0.0.0.0', | |||
'server.show_tracebacks': True, | |||
'log.screen': False, | |||
'engine.autoreload.on': args.debug | |||
}) | |||
def signal_handler(signum, stack): | |||
logging.critical('Got sig {}, exiting...'.format(signum)) | |||
cherrypy.engine.exit() | |||
signal.signal(signal.SIGINT, signal_handler) | |||
signal.signal(signal.SIGTERM, signal_handler) | |||
try: | |||
cherrypy.engine.start() | |||
cherrypy.engine.block() | |||
finally: | |||
logging.info("API has shut down") | |||
cherrypy.engine.exit() | |||
if __name__ == '__main__': | |||
main() |
@@ -0,0 +1,26 @@ | |||
#!/usr/bin/env python3 | |||
from setuptools import setup | |||
from repobot import __version__ | |||
setup(name='repobot', | |||
version=__version__, | |||
description='server for build artifact storage', | |||
url='', | |||
author='dpedu', | |||
author_email='dave@davepedu.com', | |||
packages=['repobot'], | |||
entry_points={ | |||
"console_scripts": [ | |||
"repobotd = repobot.server:main" | |||
] | |||
}, | |||
include_package_data=True, | |||
package_data={'repobot': ['../templates/pypi/*.html']}, | |||
install_requires=[ | |||
'ZODB==5.5.0', | |||
'CherryPy==18.0.1', | |||
'Jinja2==2.10' | |||
], | |||
zip_safe=False) |
@@ -0,0 +1,7 @@ | |||
#!/bin/sh | |||
set -x | |||
chown -R repobot:repobot /data | |||
install -d /data/db -o repobot -g repobot || true | |||
exec sudo -Hu repobot repobotd --data-root /data --database /data/db/repos.db $@ |
@@ -0,0 +1,16 @@ | |||
<!DOCTYPE html> | |||
<html> | |||
<head> | |||
<title>Simple index</title> | |||
<style type="text/css"> | |||
a { | |||
display: block; | |||
} | |||
</style> | |||
</head> | |||
<body> | |||
{%- for file in files %} | |||
<a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a> | |||
{%- endfor %} | |||
</body> | |||
</html> |
@@ -0,0 +1,16 @@ | |||
<!DOCTYPE html> | |||
<html> | |||
<head> | |||
<title>Simple index</title> | |||
<style type="text/css"> | |||
a { | |||
display: block; | |||
} | |||
</style> | |||
</head> | |||
<body> | |||
{%- for package in packages %} | |||
<a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a> | |||
{%- endfor %} | |||
</body> | |||
</html> |