From 70d6d0d458b91764df589e6fd2f2b2294ad2d8c6 Mon Sep 17 00:00:00 2001 From: dave Date: Sat, 4 May 2019 19:29:49 -0700 Subject: [PATCH] pip working --- repobot/pypiprovider.py | 90 +++++++++++++++++----- requirements.txt | 3 +- templates/mkdir | 0 templates/pypi/dist.html | 16 ++++ templates/pypi/{project.html => repo.html} | 4 +- templates/pypi/root.html | 4 +- 6 files changed, 94 insertions(+), 23 deletions(-) delete mode 100644 templates/mkdir create mode 100644 templates/pypi/dist.html rename templates/pypi/{project.html => repo.html} (64%) diff --git a/repobot/pypiprovider.py b/repobot/pypiprovider.py index 8ee79e3..71434fc 100644 --- a/repobot/pypiprovider.py +++ b/repobot/pypiprovider.py @@ -2,9 +2,9 @@ import cherrypy import hashlib import json import os -import queue import re from email import message_from_string +from jinja2 import Environment, FileSystemLoader, select_autoescape from sqlalchemy import Column, ForeignKey, UniqueConstraint from sqlalchemy.orm import relationship from sqlalchemy.types import String, Integer, Text @@ -13,6 +13,9 @@ from wheel import wheelfile from repobot.tables import Base, db +APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) + + def parse_wheel(path): fsize = os.path.getsize(path) @@ -80,6 +83,10 @@ def parse_wheel(path): "size": fsize} +def normalize(name): + return re.sub(r"[-_.]+", "-", name).lower() + + # https://stackoverflow.com/a/5967539 def sort_atoi(text): return int(text) if text.isdigit() else text @@ -112,12 +119,13 @@ class PipPackage(Base): # see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl - dist = Column(String(length=128), nullable=False) # 'requests' - version = Column(String(length=64), nullable=False) # '2.14.2' - build = Column(String(length=64), nullable=True) # '1234' - python = Column(String(length=64), nullable=False) # 'cp37' - api = Column(String(length=64), nullable=False) # 'cp37m' - platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64' + dist = Column(String(length=128), nullable=False) # 'requests' + dist_norm = Column(String(length=128), nullable=False) # 'requests' + version = Column(String(length=64), nullable=False) # '2.14.2' + build = Column(String(length=64), nullable=True) # '1234' + python = Column(String(length=64), nullable=False) # 'cp37' + api = Column(String(length=64), nullable=False) # 'cp37m' + platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64' fname = Column(String(length=256), nullable=False) @@ -168,13 +176,9 @@ class PypiProvider(object): self.bucket = bucket """base path within the s3 bucket""" self.basepath = "data/provider/pip" - """queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for""" - self.queue = queue.Queue() - return - - cherrypy.tree.mount(PipWeb(self), "/repo/pop", {'/': {'tools.trailing_slash.on': False, - 'tools.db.on': True}}) + cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False, + 'tools.db.on': True}}) # ensure bucket exists #TODO bucket creation should happen in server.py @@ -206,6 +210,7 @@ class PypiProvider(object): # add to db pkg = PipPackage(repo=repo, dist=metadata["fields"]["dist"], + dist_norm=normalize(metadata["fields"]["dist"]), # index me ? version=metadata["fields"]["version"], build=metadata["fields"]["build"], python=metadata["fields"]["python"], @@ -230,13 +235,62 @@ class PypiProvider(object): yield json.dumps(metadata, indent=4) -@cherrypy.popargs("reponame") +@cherrypy.popargs("reponame", "distname", "filename") class PipWeb(object): def __init__(self, base): self.base = base - # self.dists = AptDists(base) - # self.packages = AptFiles(base) + + template_dir = "templates" if os.path.exists("templates") else os.path.join(APPROOT, "templates") + self.tpl = Environment(loader=FileSystemLoader(template_dir), + autoescape=select_autoescape(['html', 'xml'])) + self.tpl.filters.update(normalize=normalize) @cherrypy.expose - def index(self, reponame=None): - yield "viewing repo {}".format(reponame) + def index(self, reponame=None, distname=None, filename=None): + if filename: + return self.handle_download(reponame, distname, filename) + else: + return self.handle_navigation(reponame, distname, filename) + + def handle_navigation(self, reponame=None, distname=None, filename=None): + if reponame: + repo = get_repo(db(), reponame, create_ok=False) + if distname: + yield self.tpl.get_template("pypi/dist.html") \ + .render(repo=repo, + pkgs=db().query(PipPackage).filter(PipPackage.repo == repo, + PipPackage.dist_norm == distname). + order_by(PipPackage.version).all(), + distname=normalize(distname)) + return + + yield self.tpl.get_template("pypi/repo.html") \ + .render(repo=repo, + dists=db().query(PipPackage).filter(PipPackage.repo == repo).order_by(PipPackage.dist).all()) + return + + yield self.tpl.get_template("pypi/root.html") \ + .render(repos=db().query(PipRepo).order_by(PipRepo.name).all()) + + def handle_download(self, reponame, distname, filename): + repo = get_repo(db(), reponame, create_ok=False) + pkg = db().query(PipPackage).filter(PipPackage.repo == repo, PipPackage.fname == filename).first() + if not pkg: + raise cherrypy.HTTPError(404) + dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0], pkg.fname) + + response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath) + + cherrypy.response.headers["Content-Type"] = "binary/octet-stream" + cherrypy.response.headers["Content-Length"] = response["ContentLength"] + + def stream(): + while True: + data = response["Body"].read(65535) + if not data: + return + yield data + + return stream() + + handle_download._cp_config = {'response.stream': True} diff --git a/requirements.txt b/requirements.txt index 9d32b86..b114e8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,9 @@ CherryPy==18.1.1 cryptography==2.6.1 docutils==0.14 jaraco.functools==2.0 +Jinja2==2.10.1 jmespath==0.9.4 +MarkupSafe==1.1.1 more-itertools==7.0.0 PGPy==0.4.1 portend==2.4 @@ -20,7 +22,6 @@ PyMySQL==0.9.3 python-dateutil==2.8.0 python-gnupg==0.4.4 pytz==2019.1 --e git+ssh://git@git.davepedu.com:223/dave/docker-artifact.git@48de79c18776e77bbd9b956afc27a872efeb0e9f#egg=repobot s3transfer==0.2.0 singledispatch==3.4.0.3 six==1.12.0 diff --git a/templates/mkdir b/templates/mkdir deleted file mode 100644 index e69de29..0000000 diff --git a/templates/pypi/dist.html b/templates/pypi/dist.html new file mode 100644 index 0000000..c7f123c --- /dev/null +++ b/templates/pypi/dist.html @@ -0,0 +1,16 @@ + + + + Simple index + + + + {%- for pkg in pkgs %} + {{ pkg.fname }} + {%- endfor %} + + diff --git a/templates/pypi/project.html b/templates/pypi/repo.html similarity index 64% rename from templates/pypi/project.html rename to templates/pypi/repo.html index 9056a98..f9ad573 100644 --- a/templates/pypi/project.html +++ b/templates/pypi/repo.html @@ -9,8 +9,8 @@ - {%- for file in files %} - {{ file }} + {%- for dist in dists %} + {{ dist.dist_norm }} {%- endfor %} diff --git a/templates/pypi/root.html b/templates/pypi/root.html index 794ac22..6c9c2b3 100644 --- a/templates/pypi/root.html +++ b/templates/pypi/root.html @@ -9,8 +9,8 @@ - {%- for package in packages %} - {{ package }} + {%- for repo in repos %} + {{ repo.name }} {%- endfor %}