pip working

This commit is contained in:
dave 2019-05-04 19:29:49 -07:00
parent f759f47616
commit 70d6d0d458
6 changed files with 94 additions and 23 deletions

View File

@ -2,9 +2,9 @@ import cherrypy
import hashlib import hashlib
import json import json
import os import os
import queue
import re import re
from email import message_from_string from email import message_from_string
from jinja2 import Environment, FileSystemLoader, select_autoescape
from sqlalchemy import Column, ForeignKey, UniqueConstraint from sqlalchemy import Column, ForeignKey, UniqueConstraint
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from sqlalchemy.types import String, Integer, Text from sqlalchemy.types import String, Integer, Text
@ -13,6 +13,9 @@ from wheel import wheelfile
from repobot.tables import Base, db from repobot.tables import Base, db
APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
def parse_wheel(path): def parse_wheel(path):
fsize = os.path.getsize(path) fsize = os.path.getsize(path)
@ -80,6 +83,10 @@ def parse_wheel(path):
"size": fsize} "size": fsize}
def normalize(name):
return re.sub(r"[-_.]+", "-", name).lower()
# https://stackoverflow.com/a/5967539 # https://stackoverflow.com/a/5967539
def sort_atoi(text): def sort_atoi(text):
return int(text) if text.isdigit() else text return int(text) if text.isdigit() else text
@ -112,12 +119,13 @@ class PipPackage(Base):
# see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py # see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
dist = Column(String(length=128), nullable=False) # 'requests' dist = Column(String(length=128), nullable=False) # 'requests'
version = Column(String(length=64), nullable=False) # '2.14.2' dist_norm = Column(String(length=128), nullable=False) # 'requests'
build = Column(String(length=64), nullable=True) # '1234' version = Column(String(length=64), nullable=False) # '2.14.2'
python = Column(String(length=64), nullable=False) # 'cp37' build = Column(String(length=64), nullable=True) # '1234'
api = Column(String(length=64), nullable=False) # 'cp37m' python = Column(String(length=64), nullable=False) # 'cp37'
platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64' api = Column(String(length=64), nullable=False) # 'cp37m'
platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64'
fname = Column(String(length=256), nullable=False) fname = Column(String(length=256), nullable=False)
@ -168,13 +176,9 @@ class PypiProvider(object):
self.bucket = bucket self.bucket = bucket
"""base path within the s3 bucket""" """base path within the s3 bucket"""
self.basepath = "data/provider/pip" self.basepath = "data/provider/pip"
"""queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for"""
self.queue = queue.Queue()
return cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}})
cherrypy.tree.mount(PipWeb(self), "/repo/pop", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}})
# ensure bucket exists # ensure bucket exists
#TODO bucket creation should happen in server.py #TODO bucket creation should happen in server.py
@ -206,6 +210,7 @@ class PypiProvider(object):
# add to db # add to db
pkg = PipPackage(repo=repo, pkg = PipPackage(repo=repo,
dist=metadata["fields"]["dist"], dist=metadata["fields"]["dist"],
dist_norm=normalize(metadata["fields"]["dist"]), # index me ?
version=metadata["fields"]["version"], version=metadata["fields"]["version"],
build=metadata["fields"]["build"], build=metadata["fields"]["build"],
python=metadata["fields"]["python"], python=metadata["fields"]["python"],
@ -230,13 +235,62 @@ class PypiProvider(object):
yield json.dumps(metadata, indent=4) yield json.dumps(metadata, indent=4)
@cherrypy.popargs("reponame") @cherrypy.popargs("reponame", "distname", "filename")
class PipWeb(object): class PipWeb(object):
def __init__(self, base): def __init__(self, base):
self.base = base self.base = base
# self.dists = AptDists(base)
# self.packages = AptFiles(base) template_dir = "templates" if os.path.exists("templates") else os.path.join(APPROOT, "templates")
self.tpl = Environment(loader=FileSystemLoader(template_dir),
autoescape=select_autoescape(['html', 'xml']))
self.tpl.filters.update(normalize=normalize)
@cherrypy.expose @cherrypy.expose
def index(self, reponame=None): def index(self, reponame=None, distname=None, filename=None):
yield "viewing repo {}".format(reponame) if filename:
return self.handle_download(reponame, distname, filename)
else:
return self.handle_navigation(reponame, distname, filename)
def handle_navigation(self, reponame=None, distname=None, filename=None):
if reponame:
repo = get_repo(db(), reponame, create_ok=False)
if distname:
yield self.tpl.get_template("pypi/dist.html") \
.render(repo=repo,
pkgs=db().query(PipPackage).filter(PipPackage.repo == repo,
PipPackage.dist_norm == distname).
order_by(PipPackage.version).all(),
distname=normalize(distname))
return
yield self.tpl.get_template("pypi/repo.html") \
.render(repo=repo,
dists=db().query(PipPackage).filter(PipPackage.repo == repo).order_by(PipPackage.dist).all())
return
yield self.tpl.get_template("pypi/root.html") \
.render(repos=db().query(PipRepo).order_by(PipRepo.name).all())
def handle_download(self, reponame, distname, filename):
repo = get_repo(db(), reponame, create_ok=False)
pkg = db().query(PipPackage).filter(PipPackage.repo == repo, PipPackage.fname == filename).first()
if not pkg:
raise cherrypy.HTTPError(404)
dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0], pkg.fname)
response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)
cherrypy.response.headers["Content-Type"] = "binary/octet-stream"
cherrypy.response.headers["Content-Length"] = response["ContentLength"]
def stream():
while True:
data = response["Body"].read(65535)
if not data:
return
yield data
return stream()
handle_download._cp_config = {'response.stream': True}

View File

@ -9,7 +9,9 @@ CherryPy==18.1.1
cryptography==2.6.1 cryptography==2.6.1
docutils==0.14 docutils==0.14
jaraco.functools==2.0 jaraco.functools==2.0
Jinja2==2.10.1
jmespath==0.9.4 jmespath==0.9.4
MarkupSafe==1.1.1
more-itertools==7.0.0 more-itertools==7.0.0
PGPy==0.4.1 PGPy==0.4.1
portend==2.4 portend==2.4
@ -20,7 +22,6 @@ PyMySQL==0.9.3
python-dateutil==2.8.0 python-dateutil==2.8.0
python-gnupg==0.4.4 python-gnupg==0.4.4
pytz==2019.1 pytz==2019.1
-e git+ssh://git@git.davepedu.com:223/dave/docker-artifact.git@48de79c18776e77bbd9b956afc27a872efeb0e9f#egg=repobot
s3transfer==0.2.0 s3transfer==0.2.0
singledispatch==3.4.0.3 singledispatch==3.4.0.3
six==1.12.0 six==1.12.0

View File

16
templates/pypi/dist.html Normal file
View File

@ -0,0 +1,16 @@
<!DOCTYPE html>
<html>
<head>
<title>Simple index</title>
<style type="text/css">
a {
display: block;
}
</style>
</head>
<body>
{%- for pkg in pkgs %}
<a href="/repo/pypi/{{ repo.name }}/{{ distname }}/{{ pkg.fname }}">{{ pkg.fname }}</a>
{%- endfor %}
</body>
</html>

View File

@ -9,8 +9,8 @@
</style> </style>
</head> </head>
<body> <body>
{%- for file in files %} {%- for dist in dists %}
<a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a> <a href="/repo/pypi/{{ repo.name }}/{{ dist.dist_norm }}/">{{ dist.dist_norm }}</a>
{%- endfor %} {%- endfor %}
</body> </body>
</html> </html>

View File

@ -9,8 +9,8 @@
</style> </style>
</head> </head>
<body> <body>
{%- for package in packages %} {%- for repo in repos %}
<a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a> <a href="/repo/pypi/{{ repo.name }}/">{{ repo.name }}</a>
{%- endfor %} {%- endfor %}
</body> </body>
</html> </html>