pip working

repobot/pypiprovider.py

@@ -2,9 +2,9 @@ import cherrypy
 import hashlib
 import json
 import os
-import queue
 import re
 from email import message_from_string
+from jinja2 import Environment, FileSystemLoader, select_autoescape
 from sqlalchemy import Column, ForeignKey, UniqueConstraint
 from sqlalchemy.orm import relationship
 from sqlalchemy.types import String, Integer, Text
@@ -13,6 +13,9 @@ from wheel import wheelfile
 from repobot.tables import Base, db
 
 
+APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+
+
 def parse_wheel(path):
     fsize = os.path.getsize(path)
 
@@ -80,6 +83,10 @@ def parse_wheel(path):
             "size": fsize}
 
 
+def normalize(name):
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
 # https://stackoverflow.com/a/5967539
 def sort_atoi(text):
     return int(text) if text.isdigit() else text
@@ -112,12 +119,13 @@ class PipPackage(Base):
 
     # see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py
     # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
-    dist = Column(String(length=128), nullable=False)      # 'requests'
-    version = Column(String(length=64), nullable=False)    # '2.14.2'
-    build = Column(String(length=64), nullable=True)       # '1234'
-    python = Column(String(length=64), nullable=False)     # 'cp37'
-    api = Column(String(length=64), nullable=False)        # 'cp37m'
-    platform = Column(String(length=256), nullable=False)  # 'manylinux1_x86_64'
+    dist = Column(String(length=128), nullable=False)       # 'requests'
+    dist_norm = Column(String(length=128), nullable=False)  # 'requests'
+    version = Column(String(length=64), nullable=False)     # '2.14.2'
+    build = Column(String(length=64), nullable=True)        # '1234'
+    python = Column(String(length=64), nullable=False)      # 'cp37'
+    api = Column(String(length=64), nullable=False)         # 'cp37m'
+    platform = Column(String(length=256), nullable=False)   # 'manylinux1_x86_64'
 
     fname = Column(String(length=256), nullable=False)
 
@@ -168,13 +176,9 @@ class PypiProvider(object):
         self.bucket = bucket
         """base path within the s3 bucket"""
         self.basepath = "data/provider/pip"
-        """queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for"""
-        self.queue = queue.Queue()
-
-        return
 
-        cherrypy.tree.mount(PipWeb(self), "/repo/pop", {'/': {'tools.trailing_slash.on': False,
-                                                              'tools.db.on': True}})
+        cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
+                                                               'tools.db.on': True}})
 
         # ensure bucket exists
         #TODO bucket creation should happen in server.py
@@ -206,6 +210,7 @@ class PypiProvider(object):
             # add to db
             pkg = PipPackage(repo=repo,
                              dist=metadata["fields"]["dist"],
+                             dist_norm=normalize(metadata["fields"]["dist"]),  # index me ?
                              version=metadata["fields"]["version"],
                              build=metadata["fields"]["build"],
                              python=metadata["fields"]["python"],
@@ -230,13 +235,62 @@ class PypiProvider(object):
             yield json.dumps(metadata, indent=4)
 
 
-@cherrypy.popargs("reponame")
+@cherrypy.popargs("reponame", "distname", "filename")
 class PipWeb(object):
     def __init__(self, base):
         self.base = base
-        # self.dists = AptDists(base)
-        # self.packages = AptFiles(base)
+
+        template_dir = "templates" if os.path.exists("templates") else os.path.join(APPROOT, "templates")
+        self.tpl = Environment(loader=FileSystemLoader(template_dir),
+                               autoescape=select_autoescape(['html', 'xml']))
+        self.tpl.filters.update(normalize=normalize)
 
     @cherrypy.expose
-    def index(self, reponame=None):
-        yield "viewing repo {}".format(reponame)
+    def index(self, reponame=None, distname=None, filename=None):
+        if filename:
+            return self.handle_download(reponame, distname, filename)
+        else:
+            return self.handle_navigation(reponame, distname, filename)
+
+    def handle_navigation(self, reponame=None, distname=None, filename=None):
+        if reponame:
+            repo = get_repo(db(), reponame, create_ok=False)
+            if distname:
+                yield self.tpl.get_template("pypi/dist.html") \
+                    .render(repo=repo,
+                            pkgs=db().query(PipPackage).filter(PipPackage.repo == repo,
+                                                               PipPackage.dist_norm == distname).
+                            order_by(PipPackage.version).all(),
+                            distname=normalize(distname))
+                return
+
+            yield self.tpl.get_template("pypi/repo.html") \
+                .render(repo=repo,
+                        dists=db().query(PipPackage).filter(PipPackage.repo == repo).order_by(PipPackage.dist).all())
+            return
+
+        yield self.tpl.get_template("pypi/root.html") \
+            .render(repos=db().query(PipRepo).order_by(PipRepo.name).all())
+
+    def handle_download(self, reponame, distname, filename):
+        repo = get_repo(db(), reponame, create_ok=False)
+        pkg = db().query(PipPackage).filter(PipPackage.repo == repo, PipPackage.fname == filename).first()
+        if not pkg:
+            raise cherrypy.HTTPError(404)
+        dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0], pkg.fname)
+
+        response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)
+
+        cherrypy.response.headers["Content-Type"] = "binary/octet-stream"
+        cherrypy.response.headers["Content-Length"] = response["ContentLength"]
+
+        def stream():
+            while True:
+                data = response["Body"].read(65535)
+                if not data:
+                    return
+                yield data
+
+        return stream()
+
+    handle_download._cp_config = {'response.stream': True}

requirements.txt

@@ -9,7 +9,9 @@ CherryPy==18.1.1
 cryptography==2.6.1
 docutils==0.14
 jaraco.functools==2.0
+Jinja2==2.10.1
 jmespath==0.9.4
+MarkupSafe==1.1.1
 more-itertools==7.0.0
 PGPy==0.4.1
 portend==2.4
@@ -20,7 +22,6 @@ PyMySQL==0.9.3
 python-dateutil==2.8.0
 python-gnupg==0.4.4
 pytz==2019.1
--e git+ssh://git@git.davepedu.com:223/dave/docker-artifact.git@48de79c18776e77bbd9b956afc27a872efeb0e9f#egg=repobot
 s3transfer==0.2.0
 singledispatch==3.4.0.3
 six==1.12.0

templates/pypi/dist.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Simple index</title>
+    <style type="text/css">
+        a {
+          display: block;
+        }
+    </style>
+  </head>
+  <body>
+    {%- for pkg in pkgs %}
+    <a href="/repo/pypi/{{ repo.name }}/{{ distname }}/{{ pkg.fname }}">{{ pkg.fname }}</a>
+    {%- endfor %}
+  </body>
+</html>

templates/pypi/project.html → templates/pypi/repo.html

@@ -9,8 +9,8 @@
     </style>
   </head>
   <body>
-    {%- for file in files %}
-    <a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a>
+    {%- for dist in dists %}
+    <a href="/repo/pypi/{{ repo.name }}/{{ dist.dist_norm }}/">{{ dist.dist_norm }}</a>
     {%- endfor %}
   </body>
 </html>

templates/pypi/root.html

@@ -9,8 +9,8 @@
     </style>
   </head>
   <body>
-    {%- for package in packages %}
-    <a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a>
+    {%- for repo in repos %}
+    <a href="/repo/pypi/{{ repo.name }}/">{{ repo.name }}</a>
     {%- endfor %}
   </body>
 </html>