initial commit

This commit is contained in:
dave 2018-10-21 16:37:53 -07:00
commit 185b638741
14 changed files with 532 additions and 0 deletions

9
.dockerignore Normal file
View File

@ -0,0 +1,9 @@
apt/*
data/*
dist/*
out/*
out2/*
testenv/*
testenv2/*
test/*
aptly/*

7
.gitignore vendored Normal file
View File

@ -0,0 +1,7 @@
/testenv
/test
__pycache__
/repobot.egg-info
/build
/dist
/repos.db*

21
Dockerfile Normal file
View File

@ -0,0 +1,21 @@
FROM ubuntu:bionic
RUN apt-get update && \
apt-get install -y python3-pip gpgv1 gnupg1 gpg sudo wget
RUN cd /tmp && \
wget -qO aptly.tgz https://bintray.com/artifact/download/smira/aptly/aptly_1.3.0_linux_amd64.tar.gz && \
tar xvf aptly.tgz aptly_1.3.0_linux_amd64/aptly && \
mv aptly_1.3.0_linux_amd64/aptly /usr/bin/ && \
rm -rf aptly.tgz aptly_1.3.0_linux_amd64
ADD . /tmp/code
RUN cd /tmp/code && \
python3 setup.py install && \
useradd repobot && \
rm -rf /tmp/code
ADD start /start
ENTRYPOINT ["/start"]

60
README.md Normal file
View File

@ -0,0 +1,60 @@
docker-artifact
===============
Software repository server
Artifact provides an HTTP API for repository management. Currently, Python and Apt repositories are supported.
Quickstart
----------
* Pull or build the image
* `docker run -it --rm -v /some/host/dir:/data -p 80:8080 artifact`
Persistent data will be placed in `/data`. The webserver will listen on port 8080 by default.
Examples
--------
Upload python package:
`curl -vv -F 'f=@pyircbot-4.0.0.post3-py3.5.egg' 'http://host/addpkg?provider=pypi&reponame=main&name=pyircbot&version=4.0.0'`
Install python packages:
`pip3 install -f http://host/repo/pypi/main/ --trusted-host host repobot`
Upload apt package:
`curl -vv -F 'f=@extpython-python3.7_3.7.0_amd64.deb' 'http://host/addpkg?provider=apt&reponame=main&name=extpython-python3.7&version=3.7.0&dist=bionic'`
Install apt packages:
```
wget -qO- http://host/repo/apt/main/repo.key | apt-key add - && \
echo "deb http://host/repo/apt/main bionic main" | tee -a /etc/apt/sources.list && \
apt-get update && \
apt-get install -y extpython-python3.6
```
Notes
-----
* Repos are created automatically when a package is added to them.
* Repo URLs are structured as: `/repo/<provider>/<name>`. Deeper URLs are handled directly by the provider.
* The apt provider will generate a gpg key per repo upon repo creation
Todo
----
* Auth
* Delete packages
* Human-readable package listing
* Support using existing GPG keys

1
repobot/__init__.py Normal file
View File

@ -0,0 +1 @@
__version__ = "0.0.1"

10
repobot/common.py Normal file
View File

@ -0,0 +1,10 @@
import persistent.list
import persistent.mapping
def plist():
return persistent.list.PersistentList()
def pmap():
return persistent.mapping.PersistentMapping()

199
repobot/provider.py Normal file
View File

@ -0,0 +1,199 @@
import os
import shutil
from repobot.common import plist, pmap
from jinja2 import Environment, FileSystemLoader, select_autoescape
import cherrypy
class PkgProvider(object):
def __init__(self, db, repo, datadir):
"""
Base package provider class
"""
self.db = db
self.repo = repo
self.dir = datadir
def render(self):
"""
Respond to requests to browse the repo
"""
raise NotImplementedError()
def add_package(self, pkobj, fname, fobj, params):
"""
Add a package to the repo
"""
raise NotImplementedError()
class PyPiProvider(PkgProvider):
def add_package(self, pkgobj, fname, fobj, params):
if "files" not in pkgobj.data:
pkgobj.data["files"] = plist()
if fname in pkgobj.data["files"]:
raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version))
pkgdir = os.path.join(self.dir, pkgobj.name)
os.makedirs(pkgdir, exist_ok=True)
# TODO handle duplicate files better
pkgfilepath = os.path.join(pkgdir, fname)
with open(pkgfilepath, "wb") as fdest:
shutil.copyfileobj(fobj, fdest)
pkgobj.data["files"].append(fname)
def browse(self, args):
tpl = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape(['html', 'xml']))
if len(args) == 0: # repo root
return tpl.get_template("pypi/root.html"). \
render(reponame=self.repo.name,
packages=self.repo.packages.keys())
elif len(args) == 1: # single module dir
files = []
if args[0] not in self.repo.packages:
raise cherrypy.HTTPError(404, 'Invalid package')
for _, version in self.repo.packages[args[0]].items():
files += version.data["files"]
return tpl.get_template("pypi/project.html"). \
render(reponame=self.repo.name,
modulename=args[0],
files=files)
elif len(args) == 2: # fetch file
fpath = os.path.join(self.dir, args[0], args[1])
return cherrypy.lib.static.serve_file(os.path.abspath(fpath), "application/octet-stream")
from subprocess import check_call, check_output, Popen, PIPE
from tempfile import NamedTemporaryFile, TemporaryDirectory
import json
class AptlyConfig(object):
"""
Context manager providing an aptly config file
"""
def __init__(self, rootdir):
self.conf = {"rootDir": rootdir} # , "gpgDisableSign": True, "gpgDisableVerify": True}
self.file = None
def __enter__(self):
self.file = NamedTemporaryFile()
with open(self.file.name, "w") as f:
f.write(json.dumps(self.conf))
return self.file.name
def __exit__(self, *args):
self.file.close()
class AptProvider(PkgProvider):
def add_package(self, pkgobj, fname, fobj, params):
# first package added sets the Distribution of the repo
# subsequent package add MUST specify the same dist
if "dist" not in self.repo.data:
self.repo.data["dist"] = params["dist"]
assert self.repo.data["dist"] == params["dist"]
# Generate a GPG key to sign packages in this repo
# TODO support passing keypath=... param to import existing keys and maybe other key generation options
if not os.path.exists(self._gpg_dir):
self._generate_gpg_key()
if "files" not in pkgobj.data:
pkgobj.data["files"] = plist()
if fname in pkgobj.data["files"]:
# raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version))
pass
with AptlyConfig(self.dir) as conf:
if not os.path.exists(os.path.join(self.dir, "db")):
os.makedirs(self.dir, exist_ok=True)
check_call(["aptly", "-config", conf, "repo", "create",
"-distribution", self.repo.data["dist"], "main"]) # TODO dist param
# put the file somewhere for now
with TemporaryDirectory() as tdir:
tmppkgpath = os.path.join(tdir, fname)
with open(tmppkgpath, "wb") as fdest:
shutil.copyfileobj(fobj, fdest)
check_call(["aptly", "-config", conf, "repo", "add", "main", tmppkgpath])
if not os.path.exists(os.path.join(self.dir, "public")):
check_call(["aptly", "-config", conf, "publish", "repo", "main"],
env=self._env)
else:
check_call(["aptly", "-config", conf, "publish", "update",
"-force-overwrite", self.repo.data["dist"]],
env=self._env)
# Make the public key available for clients
self._export_pubkey()
pkgobj.data["files"].append(fname)
# TODO validate deb file name version against user passed version
def browse(self, args):
fpath = os.path.abspath(os.path.join(self.dir, "public", *args))
if not os.path.exists(fpath):
raise cherrypy.HTTPError(404)
return cherrypy.lib.static.serve_file(fpath)
def _generate_gpg_key(self):
"""
Generate a GPG key for signing packages in this repo. Because only gpg2 supports unattended generation of
passwordless keys we generate the key with gpg2 then export/import it into gpg1.
"""
# Generate the key
os.makedirs(self._gpg_dir)
proc = Popen(["gpg", "--batch", "--gen-key"], stdin=PIPE, env=self._env)
proc.stdin.write("""%no-protection
Key-Type: rsa
Key-Length: 1024
Subkey-Type: default
Subkey-Length: 1024
Name-Real: Apt Master
Name-Comment: Apt signing key
Name-Email: aptmaster@localhost
Expire-Date: 0
%commit""".encode("ascii"))
proc.stdin.close()
proc.wait()
assert proc.returncode == 0
# Export the private key
keydata = check_output(["gpg", "--export-secret-key", "--armor", "aptmaster@localhost"], env=self._env)
shutil.rmtree(self._gpg_dir)
os.makedirs(self._gpg_dir)
# Import the private key
proc = Popen(["gpg1", "--import"], stdin=PIPE, env=self._env)
proc.stdin.write(keydata)
proc.stdin.close()
proc.wait()
assert proc.returncode == 0
def _export_pubkey(self):
keypath = os.path.join(self.dir, "public", "repo.key")
if not os.path.exists(keypath):
keydata = check_output(["gpg", "--export", "--armor", "aptmaster@localhost"], env=self._env)
with open(keypath, "wb") as f:
f.write(keydata)
@property
def _env(self):
"""
Return env vars to be used for subprocesses of this module
"""
print(os.environ["PATH"])
return {"GNUPGHOME": self._gpg_dir,
"PATH": os.environ["PATH"]}
@property
def _gpg_dir(self):
return os.path.join(self.dir, "gpg")
providers = {"pypi": PyPiProvider,
"apt": AptProvider}

61
repobot/repos.py Normal file
View File

@ -0,0 +1,61 @@
import ZODB
import ZODB.FileStorage
import persistent
import BTrees.OOBTree
from repobot.provider import providers
import os
from repobot.common import plist, pmap
class Repo(persistent.Persistent):
def __init__(self, name, provider):
self.name = name
self.provider = provider
self.packages = pmap()
self.data = pmap()
def get_package(self, name, version):
if name not in self.packages:
self.packages[name] = pmap()
if version not in self.packages[name]:
self.packages[name][version] = RepoPackage(name, version)
return self.packages[name][version]
class RepoPackage(persistent.Persistent):
def __init__(self, name, version):
self.name = name
self.version = version
self.data = pmap()
class RepoDb(object):
def __init__(self, db_path, data_root):
self.storage = ZODB.FileStorage.FileStorage(db_path)
self.db = ZODB.DB(self.storage)
self.data_root = data_root
with self.db.transaction() as c:
if "repos" not in c.root():
c.root.repos = BTrees.OOBTree.BTree()
def add_package(self, provider, reponame, pkgname, pkgversion, fname, fobj, params):
with self.db.transaction() as c:
repo = self._get_repo(c, provider, reponame)
datadir = os.path.join(self.data_root, provider, reponame)
provider = providers[repo.provider](self.db, repo, datadir)
provider.add_package(repo.get_package(pkgname, pkgversion), fname, fobj, params)
def _get_repo(self, c, provider, name):
if provider not in c.root.repos:
c.root.repos[provider] = pmap()
if name not in c.root.repos[provider]:
c.root.repos[provider][name] = Repo(name, provider)
return c.root.repos[provider][name]
def browse_repo(self, provider, reponame, args):
with self.db.transaction() as c:
repo = c.root.repos[provider][reponame]
datadir = os.path.join(self.data_root, provider, reponame)
provider = providers[repo.provider](self.db, repo, datadir)
return provider.browse(args)

99
repobot/server.py Normal file
View File

@ -0,0 +1,99 @@
import cherrypy
import logging
from repobot.repos import RepoDb
class AppWeb(object):
def __init__(self, db):
self.db = db
@cherrypy.expose
def addpkg(self, provider, reponame, name, version, f, **params):
self.db.add_package(provider, reponame, name, version, f.filename, f.file, params)
@cherrypy.expose
def repo(self, provider, repo, *args):
return self.db.browse_repo(provider, repo, args)
class FlatDispatch(cherrypy.dispatch.Dispatcher):
def __init__(self, method):
"""
Route all sub urls of this one to the single passed method
"""
super().__init__(self)
self.method = method
def find_handler(self, path):
# Hack, it does not respect settings of parent nodes
cherrypy.serving.request.config = cherrypy.config
return self.method, [i for i in filter(lambda o: len(o) > 0, path.split("/")[2:])]
def main():
import argparse
import signal
parser = argparse.ArgumentParser(description="Repobot daemon")
parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on")
parser.add_argument('-s', '--database', default="./repos.db", help="path to persistent database")
parser.add_argument('-d', '--data-root', default="./data/", help="data storage dir")
parser.add_argument('--debug', action="store_true", help="enable development options")
args = parser.parse_args()
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
db = RepoDb(args.database, args.data_root)
web = AppWeb(db)
def validate_password(realm, username, password):
s = library.session()
if s.query(User).filter(User.name == username, User.password == pwhash(password)).first():
return True
return False
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
# 'error_page.403': web.error,
# 'error_page.404': web.error
},
'/repo': {'request.dispatch': FlatDispatch(web.repo)},
#'/static': {"tools.staticdir.on": True,
# "tools.staticdir.dir": os.path.join(APPROOT, "styles/dist")
# if not args.debug else os.path.abspath("styles/dist")},
'/login': {'tools.auth_basic.on': True,
'tools.auth_basic.realm': 'webapp',
'tools.auth_basic.checkpassword': validate_password}})
cherrypy.config.update({
'tools.sessions.on': True,
'tools.sessions.locking': 'explicit',
'tools.sessions.timeout': 525600,
'request.show_tracebacks': True,
'server.socket_port': args.port,
'server.thread_pool': 25,
'server.socket_host': '0.0.0.0',
'server.show_tracebacks': True,
'log.screen': False,
'engine.autoreload.on': args.debug
})
def signal_handler(signum, stack):
logging.critical('Got sig {}, exiting...'.format(signum))
cherrypy.engine.exit()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
try:
cherrypy.engine.start()
cherrypy.engine.block()
finally:
logging.info("API has shut down")
cherrypy.engine.exit()
if __name__ == '__main__':
main()

26
setup.py Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python3
from setuptools import setup
from repobot import __version__
setup(name='repobot',
version=__version__,
description='server for build artifact storage',
url='',
author='dpedu',
author_email='dave@davepedu.com',
packages=['repobot'],
entry_points={
"console_scripts": [
"repobotd = repobot.server:main"
]
},
include_package_data=True,
package_data={'repobot': ['../templates/pypi/*.html']},
install_requires=[
'ZODB==5.5.0',
'CherryPy==18.0.1',
'Jinja2==2.10'
],
zip_safe=False)

7
start Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -x
chown -R repobot:repobot /data
install -d /data/db -o repobot -g repobot || true
exec sudo -Hu repobot repobotd --data-root /data --database /data/db/repos.db $@

0
templates/mkdir Normal file
View File

View File

@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<title>Simple index</title>
<style type="text/css">
a {
display: block;
}
</style>
</head>
<body>
{%- for file in files %}
<a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a>
{%- endfor %}
</body>
</html>

16
templates/pypi/root.html Normal file
View File

@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<title>Simple index</title>
<style type="text/css">
a {
display: block;
}
</style>
</head>
<body>
{%- for package in packages %}
<a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a>
{%- endfor %}
</body>
</html>