browser navigatable
This commit is contained in:
parent
caebad0a16
commit
51ba72f042
|
@ -13,6 +13,7 @@ import hashlib
|
|||
import os
|
||||
from time import sleep
|
||||
import gnupg
|
||||
from datetime import datetime
|
||||
import traceback
|
||||
import json
|
||||
|
||||
|
@ -25,6 +26,8 @@ class AptRepo(Base):
|
|||
gpgkeyprint = Column(Text(), nullable=True)
|
||||
gpgpubkey = Column(Text(), nullable=True)
|
||||
|
||||
dists = relationship("AptDist")
|
||||
|
||||
|
||||
class AptDist(Base):
|
||||
__tablename__ = 'aptdist'
|
||||
|
@ -74,11 +77,13 @@ class AptPackage(Base):
|
|||
|
||||
@property
|
||||
def blobpath(self):
|
||||
return "{}/{}/packages/{}/{}_{}.deb".format(self.repo.name, self.dist.name,
|
||||
self.name[0], self.name, self.sha256[0:8])
|
||||
return os.path.join("repos", self.repo.name, "packages", self.name[0], self.fname)
|
||||
|
||||
|
||||
def get_repo(_db, repo_name, create_ok=True):
|
||||
"""
|
||||
Fetch a repo from the database by name
|
||||
"""
|
||||
repo = _db.query(AptRepo).filter(AptRepo.name == repo_name).first()
|
||||
if not repo and create_ok:
|
||||
repo = AptRepo(name=repo_name)
|
||||
|
@ -88,6 +93,9 @@ def get_repo(_db, repo_name, create_ok=True):
|
|||
|
||||
|
||||
def get_dist(_db, repo, dist_name, create_ok=True):
|
||||
"""
|
||||
Fetch a repo's dist from the database by name
|
||||
"""
|
||||
dist = _db.query(AptDist).filter(AptDist.name == dist_name, AptDist.repo_id == repo.id).first()
|
||||
if not dist and create_ok:
|
||||
dist = AptDist(name=dist_name, repo_id=repo.id)
|
||||
|
@ -123,7 +131,7 @@ def copyhash(fin, fout):
|
|||
|
||||
def hashmany(data):
|
||||
"""
|
||||
Copy a file and calculate hashes while doing so
|
||||
Hash the input data using several algos
|
||||
"""
|
||||
hashes = {}
|
||||
for algo in algos.keys():
|
||||
|
@ -140,11 +148,9 @@ class AptProvider(object):
|
|||
self.db = dbcon
|
||||
self.s3 = s3client
|
||||
self.bucket = bucket
|
||||
"""base path within the s3 bucket"""
|
||||
self.basepath = "data/provider/apt"
|
||||
"""
|
||||
bucket path (after basedir)
|
||||
repos/{reponame}/packages/f/foo.deb
|
||||
"""
|
||||
|
||||
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
|
||||
'tools.db.on': True}})
|
||||
|
||||
|
@ -157,24 +163,24 @@ class AptProvider(object):
|
|||
self.updater.start()
|
||||
|
||||
def sign_packages(self):
|
||||
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
|
||||
Session.configure(bind=get_engine())
|
||||
while True:
|
||||
sleep(2)
|
||||
session = Session()
|
||||
try:
|
||||
self._sign_packages()
|
||||
self._sign_packages(session)
|
||||
except:
|
||||
traceback.print_exc()
|
||||
# sleep(10)
|
||||
break
|
||||
|
||||
def _sign_packages(self):
|
||||
print("signing packages")
|
||||
session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False))
|
||||
session.configure(bind=get_engine())
|
||||
finally:
|
||||
session.close()
|
||||
sleep(10)
|
||||
|
||||
def _sign_packages(self, session):
|
||||
dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all()
|
||||
|
||||
for dist in dirtydists:
|
||||
print("Signing dist {}/{}".format(dist.repo.name, dist.name))
|
||||
print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name))
|
||||
|
||||
str_packages = ""
|
||||
|
||||
|
@ -196,17 +202,16 @@ class AptProvider(object):
|
|||
dist.packages_cache = str_packages.encode("utf-8")
|
||||
|
||||
release_hashes = hashmany(dist.packages_cache)
|
||||
print(release_hashes)
|
||||
|
||||
str_release = """Origin: . {dist}
|
||||
Label: . {dist}
|
||||
Suite: {dist}
|
||||
Codename: {dist}
|
||||
Date: Fri, 2 Nov 2018 04:58:59 UTC
|
||||
Date: {time}
|
||||
Architectures: amd64
|
||||
Components: main
|
||||
Description: Generated by yolo
|
||||
""".format(dist=dist.name)
|
||||
""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
|
||||
for algo, algoname in algos.items():
|
||||
str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname,
|
||||
release_hashes[algo],
|
||||
|
@ -249,7 +254,7 @@ Description: Generated by yolo
|
|||
|
||||
dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret',
|
||||
detach=True, clearsign=False).data
|
||||
|
||||
dist.dirty = False
|
||||
session.commit()
|
||||
|
||||
def web_addpkg(self, reponame, name, version, fobj, dist):
|
||||
|
@ -312,15 +317,22 @@ class AptWeb(object):
|
|||
@cherrypy.expose
|
||||
def index(self, reponame=None):
|
||||
if reponame:
|
||||
#TODO
|
||||
yield "about apt repo '{}'".format(reponame)
|
||||
repo = get_repo(db(), reponame, create_ok=False)
|
||||
|
||||
yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a><hr/>".format(reponame=repo.name)
|
||||
|
||||
for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all():
|
||||
yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name)
|
||||
|
||||
# yield "about apt repo '{}'".format(reponame)
|
||||
else:
|
||||
#TODO
|
||||
yield "about all apt repos"
|
||||
for repo in db().query(AptRepo).order_by(AptRepo.name).all():
|
||||
yield "<a href='/repo/apt/{name}'>{name}</a><br/>".format(name=repo.name)
|
||||
|
||||
@cherrypy.expose
|
||||
def pubkey(self, reponame=None):
|
||||
yield get_repo(db(), reponame, create_ok=False).gpgpubkey
|
||||
cherrypy.response.headers['Content-Type'] = 'text/plain'
|
||||
return get_repo(db(), reponame, create_ok=False).gpgpubkey
|
||||
|
||||
|
||||
@cherrypy.expose
|
||||
|
@ -331,33 +343,38 @@ class AptDists(object):
|
|||
self.base = base
|
||||
|
||||
def __call__(self, *segments, reponame=None):
|
||||
repo = get_repo(db(), reponame, create_ok=False)
|
||||
|
||||
if len(segments) == 4 and segments[3] == "Packages":
|
||||
distname, componentname, indexname, pkgs = segments
|
||||
|
||||
repo = get_repo(db(), reponame, create_ok=False)
|
||||
dist = get_dist(db(), repo, distname, create_ok=False)
|
||||
|
||||
if not repo or not dist:
|
||||
raise cherrypy.HTTPError(404)
|
||||
|
||||
yield dist.packages_cache
|
||||
|
||||
return
|
||||
cherrypy.response.headers['Content-Type'] = 'text/plain'
|
||||
return dist.packages_cache
|
||||
|
||||
elif len(segments) == 2:
|
||||
distname, target = segments
|
||||
|
||||
repo = get_repo(db(), reponame, create_ok=False)
|
||||
dist = get_dist(db(), repo, distname, create_ok=False)
|
||||
|
||||
cherrypy.response.headers['Content-Type'] = 'text/plain'
|
||||
if target == "Release":
|
||||
# yield "Release for repo={} dist={}".format(reponame, distname)
|
||||
yield dist.release_cache
|
||||
return
|
||||
|
||||
return dist.release_cache
|
||||
elif target == "Release.gpg":
|
||||
yield dist.sig_cache
|
||||
return
|
||||
return dist.sig_cache
|
||||
else:
|
||||
raise cherrypy.HTTPError(404)
|
||||
|
||||
elif len(segments) == 1:
|
||||
distname = segments[0]
|
||||
dist = get_dist(db(), repo, distname, create_ok=False)
|
||||
body = ""
|
||||
for package in db().query(AptPackage).filter(AptPackage.repo == repo,
|
||||
AptPackage.dist == dist).order_by(AptPackage.fname).all():
|
||||
body += "<a href='/repo/apt/{reponame}/packages/{fname[0]}/{fname}'>{fname}</a><br />".format(reponame=repo.name, fname=package.fname)
|
||||
return body
|
||||
|
||||
raise cherrypy.HTTPError(404)
|
||||
|
||||
|
@ -378,8 +395,7 @@ class AptFiles(object):
|
|||
if not package:
|
||||
raise cherrypy.HTTPError(404)
|
||||
|
||||
dpath = os.path.join(self.base.basepath, "repos", repo.name, "packages", package.fname[0], package.fname)
|
||||
|
||||
dpath = os.path.join(self.base.basepath, package.blobpath)
|
||||
response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)
|
||||
|
||||
cherrypy.response.headers["Content-Type"] = "application/x-debian-package"
|
||||
|
|
Loading…
Reference in New Issue