Browse Source

pip working

master
dave 1 year ago
parent
commit
70d6d0d458
6 changed files with 94 additions and 23 deletions
  1. +72
    -18
      repobot/pypiprovider.py
  2. +2
    -1
      requirements.txt
  3. +0
    -0
      templates/mkdir
  4. +16
    -0
      templates/pypi/dist.html
  5. +2
    -2
      templates/pypi/repo.html
  6. +2
    -2
      templates/pypi/root.html

+ 72
- 18
repobot/pypiprovider.py View File

@@ -2,9 +2,9 @@ import cherrypy
import hashlib
import json
import os
import queue
import re
from email import message_from_string
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import Column, ForeignKey, UniqueConstraint
from sqlalchemy.orm import relationship
from sqlalchemy.types import String, Integer, Text
@@ -13,6 +13,9 @@ from wheel import wheelfile
from repobot.tables import Base, db


APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))


def parse_wheel(path):
fsize = os.path.getsize(path)

@@ -80,6 +83,10 @@ def parse_wheel(path):
"size": fsize}


def normalize(name):
return re.sub(r"[-_.]+", "-", name).lower()


# https://stackoverflow.com/a/5967539
def sort_atoi(text):
return int(text) if text.isdigit() else text
@@ -112,12 +119,13 @@ class PipPackage(Base):

# see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
dist = Column(String(length=128), nullable=False) # 'requests'
version = Column(String(length=64), nullable=False) # '2.14.2'
build = Column(String(length=64), nullable=True) # '1234'
python = Column(String(length=64), nullable=False) # 'cp37'
api = Column(String(length=64), nullable=False) # 'cp37m'
platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64'
dist = Column(String(length=128), nullable=False) # 'requests'
dist_norm = Column(String(length=128), nullable=False) # 'requests'
version = Column(String(length=64), nullable=False) # '2.14.2'
build = Column(String(length=64), nullable=True) # '1234'
python = Column(String(length=64), nullable=False) # 'cp37'
api = Column(String(length=64), nullable=False) # 'cp37m'
platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64'

fname = Column(String(length=256), nullable=False)

@@ -168,13 +176,9 @@ class PypiProvider(object):
self.bucket = bucket
"""base path within the s3 bucket"""
self.basepath = "data/provider/pip"
"""queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for"""
self.queue = queue.Queue()

return

cherrypy.tree.mount(PipWeb(self), "/repo/pop", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}})
cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}})

# ensure bucket exists
#TODO bucket creation should happen in server.py
@@ -206,6 +210,7 @@ class PypiProvider(object):
# add to db
pkg = PipPackage(repo=repo,
dist=metadata["fields"]["dist"],
dist_norm=normalize(metadata["fields"]["dist"]), # index me ?
version=metadata["fields"]["version"],
build=metadata["fields"]["build"],
python=metadata["fields"]["python"],
@@ -230,13 +235,62 @@ class PypiProvider(object):
yield json.dumps(metadata, indent=4)


@cherrypy.popargs("reponame")
@cherrypy.popargs("reponame", "distname", "filename")
class PipWeb(object):
def __init__(self, base):
self.base = base
# self.dists = AptDists(base)
# self.packages = AptFiles(base)

template_dir = "templates" if os.path.exists("templates") else os.path.join(APPROOT, "templates")
self.tpl = Environment(loader=FileSystemLoader(template_dir),
autoescape=select_autoescape(['html', 'xml']))
self.tpl.filters.update(normalize=normalize)

@cherrypy.expose
def index(self, reponame=None):
yield "viewing repo {}".format(reponame)
def index(self, reponame=None, distname=None, filename=None):
if filename:
return self.handle_download(reponame, distname, filename)
else:
return self.handle_navigation(reponame, distname, filename)

def handle_navigation(self, reponame=None, distname=None, filename=None):
if reponame:
repo = get_repo(db(), reponame, create_ok=False)
if distname:
yield self.tpl.get_template("pypi/dist.html") \
.render(repo=repo,
pkgs=db().query(PipPackage).filter(PipPackage.repo == repo,
PipPackage.dist_norm == distname).
order_by(PipPackage.version).all(),
distname=normalize(distname))
return

yield self.tpl.get_template("pypi/repo.html") \
.render(repo=repo,
dists=db().query(PipPackage).filter(PipPackage.repo == repo).order_by(PipPackage.dist).all())
return

yield self.tpl.get_template("pypi/root.html") \
.render(repos=db().query(PipRepo).order_by(PipRepo.name).all())

def handle_download(self, reponame, distname, filename):
repo = get_repo(db(), reponame, create_ok=False)
pkg = db().query(PipPackage).filter(PipPackage.repo == repo, PipPackage.fname == filename).first()
if not pkg:
raise cherrypy.HTTPError(404)
dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0], pkg.fname)

response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)

cherrypy.response.headers["Content-Type"] = "binary/octet-stream"
cherrypy.response.headers["Content-Length"] = response["ContentLength"]

def stream():
while True:
data = response["Body"].read(65535)
if not data:
return
yield data

return stream()

handle_download._cp_config = {'response.stream': True}

+ 2
- 1
requirements.txt View File

@@ -9,7 +9,9 @@ CherryPy==18.1.1
cryptography==2.6.1
docutils==0.14
jaraco.functools==2.0
Jinja2==2.10.1
jmespath==0.9.4
MarkupSafe==1.1.1
more-itertools==7.0.0
PGPy==0.4.1
portend==2.4
@@ -20,7 +22,6 @@ PyMySQL==0.9.3
python-dateutil==2.8.0
python-gnupg==0.4.4
pytz==2019.1
-e git+ssh://git@git.davepedu.com:223/dave/docker-artifact.git@48de79c18776e77bbd9b956afc27a872efeb0e9f#egg=repobot
s3transfer==0.2.0
singledispatch==3.4.0.3
six==1.12.0


+ 0
- 0
templates/mkdir View File


+ 16
- 0
templates/pypi/dist.html View File

@@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<title>Simple index</title>
<style type="text/css">
a {
display: block;
}
</style>
</head>
<body>
{%- for pkg in pkgs %}
<a href="/repo/pypi/{{ repo.name }}/{{ distname }}/{{ pkg.fname }}">{{ pkg.fname }}</a>
{%- endfor %}
</body>
</html>

templates/pypi/project.html → templates/pypi/repo.html View File

@@ -9,8 +9,8 @@
</style>
</head>
<body>
{%- for file in files %}
<a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a>
{%- for dist in dists %}
<a href="/repo/pypi/{{ repo.name }}/{{ dist.dist_norm }}/">{{ dist.dist_norm }}</a>
{%- endfor %}
</body>
</html>

+ 2
- 2
templates/pypi/root.html View File

@@ -9,8 +9,8 @@
</style>
</head>
<body>
{%- for package in packages %}
<a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a>
{%- for repo in repos %}
<a href="/repo/pypi/{{ repo.name }}/">{{ repo.name }}</a>
{%- endfor %}
</body>
</html>

Loading…
Cancel
Save