pip working
This commit is contained in:
parent
f759f47616
commit
70d6d0d458
|
@ -2,9 +2,9 @@ import cherrypy
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import queue
|
|
||||||
import re
|
import re
|
||||||
from email import message_from_string
|
from email import message_from_string
|
||||||
|
from jinja2 import Environment, FileSystemLoader, select_autoescape
|
||||||
from sqlalchemy import Column, ForeignKey, UniqueConstraint
|
from sqlalchemy import Column, ForeignKey, UniqueConstraint
|
||||||
from sqlalchemy.orm import relationship
|
from sqlalchemy.orm import relationship
|
||||||
from sqlalchemy.types import String, Integer, Text
|
from sqlalchemy.types import String, Integer, Text
|
||||||
|
@ -13,6 +13,9 @@ from wheel import wheelfile
|
||||||
from repobot.tables import Base, db
|
from repobot.tables import Base, db
|
||||||
|
|
||||||
|
|
||||||
|
APPROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
|
||||||
|
|
||||||
|
|
||||||
def parse_wheel(path):
|
def parse_wheel(path):
|
||||||
fsize = os.path.getsize(path)
|
fsize = os.path.getsize(path)
|
||||||
|
|
||||||
|
@ -80,6 +83,10 @@ def parse_wheel(path):
|
||||||
"size": fsize}
|
"size": fsize}
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(name):
|
||||||
|
return re.sub(r"[-_.]+", "-", name).lower()
|
||||||
|
|
||||||
|
|
||||||
# https://stackoverflow.com/a/5967539
|
# https://stackoverflow.com/a/5967539
|
||||||
def sort_atoi(text):
|
def sort_atoi(text):
|
||||||
return int(text) if text.isdigit() else text
|
return int(text) if text.isdigit() else text
|
||||||
|
@ -112,12 +119,13 @@ class PipPackage(Base):
|
||||||
|
|
||||||
# see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py
|
# see https://github.com/pypa/wheel/blob/master/wheel/wheelfile.py
|
||||||
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
|
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
|
||||||
dist = Column(String(length=128), nullable=False) # 'requests'
|
dist = Column(String(length=128), nullable=False) # 'requests'
|
||||||
version = Column(String(length=64), nullable=False) # '2.14.2'
|
dist_norm = Column(String(length=128), nullable=False) # 'requests'
|
||||||
build = Column(String(length=64), nullable=True) # '1234'
|
version = Column(String(length=64), nullable=False) # '2.14.2'
|
||||||
python = Column(String(length=64), nullable=False) # 'cp37'
|
build = Column(String(length=64), nullable=True) # '1234'
|
||||||
api = Column(String(length=64), nullable=False) # 'cp37m'
|
python = Column(String(length=64), nullable=False) # 'cp37'
|
||||||
platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64'
|
api = Column(String(length=64), nullable=False) # 'cp37m'
|
||||||
|
platform = Column(String(length=256), nullable=False) # 'manylinux1_x86_64'
|
||||||
|
|
||||||
fname = Column(String(length=256), nullable=False)
|
fname = Column(String(length=256), nullable=False)
|
||||||
|
|
||||||
|
@ -168,13 +176,9 @@ class PypiProvider(object):
|
||||||
self.bucket = bucket
|
self.bucket = bucket
|
||||||
"""base path within the s3 bucket"""
|
"""base path within the s3 bucket"""
|
||||||
self.basepath = "data/provider/pip"
|
self.basepath = "data/provider/pip"
|
||||||
"""queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for"""
|
|
||||||
self.queue = queue.Queue()
|
|
||||||
|
|
||||||
return
|
cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
|
||||||
|
'tools.db.on': True}})
|
||||||
cherrypy.tree.mount(PipWeb(self), "/repo/pop", {'/': {'tools.trailing_slash.on': False,
|
|
||||||
'tools.db.on': True}})
|
|
||||||
|
|
||||||
# ensure bucket exists
|
# ensure bucket exists
|
||||||
#TODO bucket creation should happen in server.py
|
#TODO bucket creation should happen in server.py
|
||||||
|
@ -206,6 +210,7 @@ class PypiProvider(object):
|
||||||
# add to db
|
# add to db
|
||||||
pkg = PipPackage(repo=repo,
|
pkg = PipPackage(repo=repo,
|
||||||
dist=metadata["fields"]["dist"],
|
dist=metadata["fields"]["dist"],
|
||||||
|
dist_norm=normalize(metadata["fields"]["dist"]), # index me ?
|
||||||
version=metadata["fields"]["version"],
|
version=metadata["fields"]["version"],
|
||||||
build=metadata["fields"]["build"],
|
build=metadata["fields"]["build"],
|
||||||
python=metadata["fields"]["python"],
|
python=metadata["fields"]["python"],
|
||||||
|
@ -230,13 +235,62 @@ class PypiProvider(object):
|
||||||
yield json.dumps(metadata, indent=4)
|
yield json.dumps(metadata, indent=4)
|
||||||
|
|
||||||
|
|
||||||
@cherrypy.popargs("reponame")
|
@cherrypy.popargs("reponame", "distname", "filename")
|
||||||
class PipWeb(object):
|
class PipWeb(object):
|
||||||
def __init__(self, base):
|
def __init__(self, base):
|
||||||
self.base = base
|
self.base = base
|
||||||
# self.dists = AptDists(base)
|
|
||||||
# self.packages = AptFiles(base)
|
template_dir = "templates" if os.path.exists("templates") else os.path.join(APPROOT, "templates")
|
||||||
|
self.tpl = Environment(loader=FileSystemLoader(template_dir),
|
||||||
|
autoescape=select_autoescape(['html', 'xml']))
|
||||||
|
self.tpl.filters.update(normalize=normalize)
|
||||||
|
|
||||||
@cherrypy.expose
|
@cherrypy.expose
|
||||||
def index(self, reponame=None):
|
def index(self, reponame=None, distname=None, filename=None):
|
||||||
yield "viewing repo {}".format(reponame)
|
if filename:
|
||||||
|
return self.handle_download(reponame, distname, filename)
|
||||||
|
else:
|
||||||
|
return self.handle_navigation(reponame, distname, filename)
|
||||||
|
|
||||||
|
def handle_navigation(self, reponame=None, distname=None, filename=None):
|
||||||
|
if reponame:
|
||||||
|
repo = get_repo(db(), reponame, create_ok=False)
|
||||||
|
if distname:
|
||||||
|
yield self.tpl.get_template("pypi/dist.html") \
|
||||||
|
.render(repo=repo,
|
||||||
|
pkgs=db().query(PipPackage).filter(PipPackage.repo == repo,
|
||||||
|
PipPackage.dist_norm == distname).
|
||||||
|
order_by(PipPackage.version).all(),
|
||||||
|
distname=normalize(distname))
|
||||||
|
return
|
||||||
|
|
||||||
|
yield self.tpl.get_template("pypi/repo.html") \
|
||||||
|
.render(repo=repo,
|
||||||
|
dists=db().query(PipPackage).filter(PipPackage.repo == repo).order_by(PipPackage.dist).all())
|
||||||
|
return
|
||||||
|
|
||||||
|
yield self.tpl.get_template("pypi/root.html") \
|
||||||
|
.render(repos=db().query(PipRepo).order_by(PipRepo.name).all())
|
||||||
|
|
||||||
|
def handle_download(self, reponame, distname, filename):
|
||||||
|
repo = get_repo(db(), reponame, create_ok=False)
|
||||||
|
pkg = db().query(PipPackage).filter(PipPackage.repo == repo, PipPackage.fname == filename).first()
|
||||||
|
if not pkg:
|
||||||
|
raise cherrypy.HTTPError(404)
|
||||||
|
dpath = os.path.join(self.base.basepath, "repos", repo.name, "wheels", pkg.fname[0], pkg.fname)
|
||||||
|
|
||||||
|
response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)
|
||||||
|
|
||||||
|
cherrypy.response.headers["Content-Type"] = "binary/octet-stream"
|
||||||
|
cherrypy.response.headers["Content-Length"] = response["ContentLength"]
|
||||||
|
|
||||||
|
def stream():
|
||||||
|
while True:
|
||||||
|
data = response["Body"].read(65535)
|
||||||
|
if not data:
|
||||||
|
return
|
||||||
|
yield data
|
||||||
|
|
||||||
|
return stream()
|
||||||
|
|
||||||
|
handle_download._cp_config = {'response.stream': True}
|
||||||
|
|
|
@ -9,7 +9,9 @@ CherryPy==18.1.1
|
||||||
cryptography==2.6.1
|
cryptography==2.6.1
|
||||||
docutils==0.14
|
docutils==0.14
|
||||||
jaraco.functools==2.0
|
jaraco.functools==2.0
|
||||||
|
Jinja2==2.10.1
|
||||||
jmespath==0.9.4
|
jmespath==0.9.4
|
||||||
|
MarkupSafe==1.1.1
|
||||||
more-itertools==7.0.0
|
more-itertools==7.0.0
|
||||||
PGPy==0.4.1
|
PGPy==0.4.1
|
||||||
portend==2.4
|
portend==2.4
|
||||||
|
@ -20,7 +22,6 @@ PyMySQL==0.9.3
|
||||||
python-dateutil==2.8.0
|
python-dateutil==2.8.0
|
||||||
python-gnupg==0.4.4
|
python-gnupg==0.4.4
|
||||||
pytz==2019.1
|
pytz==2019.1
|
||||||
-e git+ssh://git@git.davepedu.com:223/dave/docker-artifact.git@48de79c18776e77bbd9b956afc27a872efeb0e9f#egg=repobot
|
|
||||||
s3transfer==0.2.0
|
s3transfer==0.2.0
|
||||||
singledispatch==3.4.0.3
|
singledispatch==3.4.0.3
|
||||||
six==1.12.0
|
six==1.12.0
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Simple index</title>
|
||||||
|
<style type="text/css">
|
||||||
|
a {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
{%- for pkg in pkgs %}
|
||||||
|
<a href="/repo/pypi/{{ repo.name }}/{{ distname }}/{{ pkg.fname }}">{{ pkg.fname }}</a>
|
||||||
|
{%- endfor %}
|
||||||
|
</body>
|
||||||
|
</html>
|
|
@ -9,8 +9,8 @@
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
{%- for file in files %}
|
{%- for dist in dists %}
|
||||||
<a href="/repo/pypi/{{ reponame }}/{{ modulename }}/{{ file }}">{{ file }}</a>
|
<a href="/repo/pypi/{{ repo.name }}/{{ dist.dist_norm }}/">{{ dist.dist_norm }}</a>
|
||||||
{%- endfor %}
|
{%- endfor %}
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
|
@ -9,8 +9,8 @@
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
{%- for package in packages %}
|
{%- for repo in repos %}
|
||||||
<a href="/repo/pypi/{{ reponame }}/{{ package }}/">{{ package }}</a>
|
<a href="/repo/pypi/{{ repo.name }}/">{{ repo.name }}</a>
|
||||||
{%- endfor %}
|
{%- endfor %}
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|
Loading…
Reference in New Issue