From 7c5b8e5937b15c7f5d04672f63c1a8ac8a06da1f Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 4 May 2019 21:26:27 -0700 Subject: [PATCH] Update readme and dockerfile --- .dockerignore | 2 ++ Dockerfile | 16 +++++----------- README.md | 37 ++++++++++++++++++------------------- repobot/__init__.py | 2 +- repobot/pypiprovider.py | 4 ++-- requirements.txt | 3 ++- setup.py | 3 +-- start | 8 -------- 8 files changed, 31 insertions(+), 44 deletions(-) delete mode 100755 start diff --git a/.dockerignore b/.dockerignore index 2fd5f07..7f43f86 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,5 @@ testenv/* testenv2/* test/* aptly/* +wheel/* +dist/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 53a79aa..ba86f6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,15 @@ FROM ubuntu:bionic RUN apt-get update && \ - apt-get install -y python3-pip gpgv1 gnupg1 gpg sudo wget - -RUN cd /tmp && \ - wget -qO aptly.tgz https://bintray.com/artifact/download/smira/aptly/aptly_1.3.0_linux_amd64.tar.gz && \ - tar xvf aptly.tgz aptly_1.3.0_linux_amd64/aptly && \ - mv aptly_1.3.0_linux_amd64/aptly /usr/bin/ && \ - rm -rf aptly.tgz aptly_1.3.0_linux_amd64 + apt-get install -y python3-pip gpgv1 gnupg1 gpg sudo wget git ADD . /tmp/code RUN cd /tmp/code && \ + pip3 install -r requirements.txt && \ python3 setup.py install && \ - useradd repobot && \ - rm -rf /tmp/code + useradd repobot -ADD start /start +USER repobot -ENTRYPOINT ["/start"] +ENTRYPOINT ["repobotd"] diff --git a/README.md b/README.md index e9d16b7..8fbe46e 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,18 @@ docker-artifact Software repository server -Artifact provides an HTTP API for repository management. Currently, Python and Apt repositories are supported. +Artifactd provides an HTTP API for repository management. Currently, Python and Apt repositories are supported. Quickstart ---------- * Pull or build the image -* `docker run -it --rm -v /some/host/dir:/data -p 80:8080 artifact` +* `docker run -it --rm -e 'DATABASE_URL=mysql+pymysql://...' -e 'S3_URL=http://...' -p 8080:8080 artifact` -Persistent data will be placed in `/data`. The webserver will listen on port 8080 by default. +The in-container webserver will listen on port 8080 by default. Database url is passed directly to sqlalchemy, but only +mysql is tested. S3_URL is in the form of `https?://keyname:keysecret@endpoint_url/bucket_name`. Amazon S3 is supported +but minio is the preferred backend. Examples @@ -20,43 +22,40 @@ Examples Upload python package: -`curl -vv -F 'f=@pyircbot-4.0.0.post3-py3.5.egg' 'http://host/addpkg?provider=pypi&reponame=main&name=pyircbot&version=4.0.0'` +`curl -vv -F 'f=@pyircbot-4.0.0.post3-py3.5.egg' 'http://localhost:8080/addpkg?provider=pypi&reponame=main&name=pyircbot&version=4.0.0'` Install python packages: -`pip3 install -f http://host/repo/pypi/main/ --trusted-host host repobot` +`pip3 install -i http://host/repo/pypi/main/ --trusted-host host ` Upload apt package: -`curl -vv -F 'f=@extpython-python3.7_3.7.0_amd64.deb' 'http://host/addpkg?provider=apt&reponame=main&name=extpython-python3.7&version=3.7.0&dist=bionic'` +`curl -vv -F 'f=@python3_3.6.7-1~18.04_amd64.deb' 'http://host/addpkg?provider=apt&reponame=main&name=python3&version=3.6.7-1~18.04&dist=bionic'` Install apt packages: ``` -wget -qO- http://host/repo/apt/main/repo.key | apt-key add - && \ +wget -qO- http://host/repo/apt/main/pubkey | apt-key add - && \ echo "deb http://host/repo/apt/main bionic main" | tee -a /etc/apt/sources.list && \ -apt-get update && \ -apt-get install -y extpython-python3.6 +apt-get update ``` -Replication ------------ - -Repobot can automatically copy packages to neighbor repobot instances. Pass `-n` with one or more neighbor addresses in -the format of `http://1.2.3.4:8080`. When new packages are submitted, they will be queued for replication. The server -will periodically attempt to submit the package to each neighbor. - - Notes ----- * Repos are created automatically when a package is added to them. -* Repo URLs are structured as: `/repo//`. Deeper URLs are handled directly by the provider. +* Repo URLs are structured as: `/repo//`. URLs at and below this level are handled directly by + the provider. +* In the apt provider, only binary-amd64 packages are supported. No source, binary-386 or other groups +* In the apt provider, every repo has only one component, named "main" * The apt provider will generate a gpg key per repo upon repo creation +* The repo contents can be browsed on the web +* This uses my fork of python-dpkg, from [here](https://git.davepedu.com/dave/python-dpkg), which is not automatically + installed via `setup.py` due to pip limitations. Todo @@ -64,5 +63,5 @@ Todo * Auth * Delete packages -* Human-readable package listing * Support using existing GPG keys +* Nicer UI diff --git a/repobot/__init__.py b/repobot/__init__.py index d3ec452..493f741 100644 --- a/repobot/__init__.py +++ b/repobot/__init__.py @@ -1 +1 @@ -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/repobot/pypiprovider.py b/repobot/pypiprovider.py index e5deb4c..00eb132 100644 --- a/repobot/pypiprovider.py +++ b/repobot/pypiprovider.py @@ -195,7 +195,7 @@ class PypiProvider(object): # s3 path - repos//wheels/f/foo.wheel dpath = os.path.join(self.basepath, "repos", repo.name, "wheels", - metadata["wheelname"][0], metadata["wheelname"]) + metadata["wheelname"][0].lower(), metadata["wheelname"]) files = self.s3.list_objects(Bucket=self.bucket, Prefix=dpath).get("Contents") if files: @@ -278,7 +278,7 @@ class PipWeb(object): if not pkg: raise cherrypy.HTTPError(404) - dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0], pkg.fname) + dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0].lower(), pkg.fname) if str(cherrypy.request.method) == "DELETE": db().delete(pkg) diff --git a/requirements.txt b/requirements.txt index b114e8d..2997d6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ PGPy==0.4.1 portend==2.4 pyasn1==0.4.5 pycparser==2.19 -pydpkg==1.3.1 +-e git+https://git.davepedu.com/dave/python-dpkg.git#egg=pydpkg PyMySQL==0.9.3 python-dateutil==2.8.0 python-gnupg==0.4.4 @@ -28,4 +28,5 @@ six==1.12.0 SQLAlchemy==1.3.3 tempora==1.14.1 urllib3==1.24.2 +wheel==0.33.1 zc.lockfile==1.4 diff --git a/setup.py b/setup.py index 348f4be..40daf77 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from repobot import __version__ with open("requirements.txt") as f: - requirements = f.readlines() + requirements = [l for l in f.readlines() if not l.startswith("-")] setup(name='repobot', @@ -21,7 +21,6 @@ setup(name='repobot', ] }, include_package_data=True, - # package_data={'repobot': ['../templates/pypi/*.html']}, install_requires=requirements, package_data={'repobot': ['../templates/pypi/*.html']}, zip_safe=False) diff --git a/start b/start deleted file mode 100755 index 897b522..0000000 --- a/start +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -set -x - -install -d /data/db -o repobot -g repobot || true -install -d /data/data -o repobot -g repobot || true -chown -R repobot:repobot /data/db /data/data - -exec sudo -Hu repobot repobotd --data-root /data/data --database /data/db/repos.db $@