browser navigatable

This commit is contained in:
dave 2019-04-30 21:40:48 -07:00
parent caebad0a16
commit 51ba72f042
1 changed files with 57 additions and 41 deletions

View File

@ -13,6 +13,7 @@ import hashlib
import os
from time import sleep
import gnupg
from datetime import datetime
import traceback
import json
@ -25,6 +26,8 @@ class AptRepo(Base):
gpgkeyprint = Column(Text(), nullable=True)
gpgpubkey = Column(Text(), nullable=True)
dists = relationship("AptDist")
class AptDist(Base):
__tablename__ = 'aptdist'
@ -74,11 +77,13 @@ class AptPackage(Base):
@property
def blobpath(self):
return "{}/{}/packages/{}/{}_{}.deb".format(self.repo.name, self.dist.name,
self.name[0], self.name, self.sha256[0:8])
return os.path.join("repos", self.repo.name, "packages", self.name[0], self.fname)
def get_repo(_db, repo_name, create_ok=True):
"""
Fetch a repo from the database by name
"""
repo = _db.query(AptRepo).filter(AptRepo.name == repo_name).first()
if not repo and create_ok:
repo = AptRepo(name=repo_name)
@ -88,6 +93,9 @@ def get_repo(_db, repo_name, create_ok=True):
def get_dist(_db, repo, dist_name, create_ok=True):
"""
Fetch a repo's dist from the database by name
"""
dist = _db.query(AptDist).filter(AptDist.name == dist_name, AptDist.repo_id == repo.id).first()
if not dist and create_ok:
dist = AptDist(name=dist_name, repo_id=repo.id)
@ -123,7 +131,7 @@ def copyhash(fin, fout):
def hashmany(data):
"""
Copy a file and calculate hashes while doing so
Hash the input data using several algos
"""
hashes = {}
for algo in algos.keys():
@ -140,11 +148,9 @@ class AptProvider(object):
self.db = dbcon
self.s3 = s3client
self.bucket = bucket
"""base path within the s3 bucket"""
self.basepath = "data/provider/apt"
"""
bucket path (after basedir)
repos/{reponame}/packages/f/foo.deb
"""
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}})
@ -157,24 +163,24 @@ class AptProvider(object):
self.updater.start()
def sign_packages(self):
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
Session.configure(bind=get_engine())
while True:
sleep(2)
session = Session()
try:
self._sign_packages()
self._sign_packages(session)
except:
traceback.print_exc()
# sleep(10)
break
def _sign_packages(self):
print("signing packages")
session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False))
session.configure(bind=get_engine())
finally:
session.close()
sleep(10)
def _sign_packages(self, session):
dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all()
for dist in dirtydists:
print("Signing dist {}/{}".format(dist.repo.name, dist.name))
print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name))
str_packages = ""
@ -196,17 +202,16 @@ class AptProvider(object):
dist.packages_cache = str_packages.encode("utf-8")
release_hashes = hashmany(dist.packages_cache)
print(release_hashes)
str_release = """Origin: . {dist}
Label: . {dist}
Suite: {dist}
Codename: {dist}
Date: Fri, 2 Nov 2018 04:58:59 UTC
Date: {time}
Architectures: amd64
Components: main
Description: Generated by yolo
""".format(dist=dist.name)
""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
for algo, algoname in algos.items():
str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname,
release_hashes[algo],
@ -249,8 +254,8 @@ Description: Generated by yolo
dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret',
detach=True, clearsign=False).data
session.commit()
dist.dirty = False
session.commit()
def web_addpkg(self, reponame, name, version, fobj, dist):
repo = get_repo(db(), reponame)
@ -312,15 +317,22 @@ class AptWeb(object):
@cherrypy.expose
def index(self, reponame=None):
if reponame:
#TODO
yield "about apt repo '{}'".format(reponame)
repo = get_repo(db(), reponame, create_ok=False)
yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a><hr/>".format(reponame=repo.name)
for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all():
yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name)
# yield "about apt repo '{}'".format(reponame)
else:
#TODO
yield "about all apt repos"
for repo in db().query(AptRepo).order_by(AptRepo.name).all():
yield "<a href='/repo/apt/{name}'>{name}</a><br/>".format(name=repo.name)
@cherrypy.expose
def pubkey(self, reponame=None):
yield get_repo(db(), reponame, create_ok=False).gpgpubkey
cherrypy.response.headers['Content-Type'] = 'text/plain'
return get_repo(db(), reponame, create_ok=False).gpgpubkey
@cherrypy.expose
@ -331,33 +343,38 @@ class AptDists(object):
self.base = base
def __call__(self, *segments, reponame=None):
repo = get_repo(db(), reponame, create_ok=False)
if len(segments) == 4 and segments[3] == "Packages":
distname, componentname, indexname, pkgs = segments
repo = get_repo(db(), reponame, create_ok=False)
dist = get_dist(db(), repo, distname, create_ok=False)
if not repo or not dist:
raise cherrypy.HTTPError(404)
yield dist.packages_cache
return
cherrypy.response.headers['Content-Type'] = 'text/plain'
return dist.packages_cache
elif len(segments) == 2:
distname, target = segments
repo = get_repo(db(), reponame, create_ok=False)
dist = get_dist(db(), repo, distname, create_ok=False)
cherrypy.response.headers['Content-Type'] = 'text/plain'
if target == "Release":
# yield "Release for repo={} dist={}".format(reponame, distname)
yield dist.release_cache
return
return dist.release_cache
elif target == "Release.gpg":
yield dist.sig_cache
return
return dist.sig_cache
else:
raise cherrypy.HTTPError(404)
elif len(segments) == 1:
distname = segments[0]
dist = get_dist(db(), repo, distname, create_ok=False)
body = ""
for package in db().query(AptPackage).filter(AptPackage.repo == repo,
AptPackage.dist == dist).order_by(AptPackage.fname).all():
body += "<a href='/repo/apt/{reponame}/packages/{fname[0]}/{fname}'>{fname}</a><br />".format(reponame=repo.name, fname=package.fname)
return body
raise cherrypy.HTTPError(404)
@ -378,8 +395,7 @@ class AptFiles(object):
if not package:
raise cherrypy.HTTPError(404)
dpath = os.path.join(self.base.basepath, "repos", repo.name, "packages", package.fname[0], package.fname)
dpath = os.path.join(self.base.basepath, package.blobpath)
response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)
cherrypy.response.headers["Content-Type"] = "application/x-debian-package"