diff --git a/repobot/aptprovider.py b/repobot/aptprovider.py index 0aff3c4..a1c0483 100644 --- a/repobot/aptprovider.py +++ b/repobot/aptprovider.py @@ -13,6 +13,7 @@ import hashlib import os from time import sleep import gnupg +from datetime import datetime import traceback import json @@ -25,6 +26,8 @@ class AptRepo(Base): gpgkeyprint = Column(Text(), nullable=True) gpgpubkey = Column(Text(), nullable=True) + dists = relationship("AptDist") + class AptDist(Base): __tablename__ = 'aptdist' @@ -74,11 +77,13 @@ class AptPackage(Base): @property def blobpath(self): - return "{}/{}/packages/{}/{}_{}.deb".format(self.repo.name, self.dist.name, - self.name[0], self.name, self.sha256[0:8]) + return os.path.join("repos", self.repo.name, "packages", self.name[0], self.fname) def get_repo(_db, repo_name, create_ok=True): + """ + Fetch a repo from the database by name + """ repo = _db.query(AptRepo).filter(AptRepo.name == repo_name).first() if not repo and create_ok: repo = AptRepo(name=repo_name) @@ -88,6 +93,9 @@ def get_repo(_db, repo_name, create_ok=True): def get_dist(_db, repo, dist_name, create_ok=True): + """ + Fetch a repo's dist from the database by name + """ dist = _db.query(AptDist).filter(AptDist.name == dist_name, AptDist.repo_id == repo.id).first() if not dist and create_ok: dist = AptDist(name=dist_name, repo_id=repo.id) @@ -123,7 +131,7 @@ def copyhash(fin, fout): def hashmany(data): """ - Copy a file and calculate hashes while doing so + Hash the input data using several algos """ hashes = {} for algo in algos.keys(): @@ -140,11 +148,9 @@ class AptProvider(object): self.db = dbcon self.s3 = s3client self.bucket = bucket + """base path within the s3 bucket""" self.basepath = "data/provider/apt" - """ - bucket path (after basedir) - repos/{reponame}/packages/f/foo.deb - """ + cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False, 'tools.db.on': True}}) @@ -157,24 +163,24 @@ class AptProvider(object): self.updater.start() def sign_packages(self): + Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) + Session.configure(bind=get_engine()) while True: sleep(2) + session = Session() try: - self._sign_packages() + self._sign_packages(session) except: traceback.print_exc() - # sleep(10) - break - - def _sign_packages(self): - print("signing packages") - session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)) - session.configure(bind=get_engine()) + finally: + session.close() + sleep(10) + def _sign_packages(self, session): dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all() for dist in dirtydists: - print("Signing dist {}/{}".format(dist.repo.name, dist.name)) + print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name)) str_packages = "" @@ -196,17 +202,16 @@ class AptProvider(object): dist.packages_cache = str_packages.encode("utf-8") release_hashes = hashmany(dist.packages_cache) - print(release_hashes) str_release = """Origin: . {dist} Label: . {dist} Suite: {dist} Codename: {dist} -Date: Fri, 2 Nov 2018 04:58:59 UTC +Date: {time} Architectures: amd64 Components: main Description: Generated by yolo -""".format(dist=dist.name) +""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) for algo, algoname in algos.items(): str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname, release_hashes[algo], @@ -249,8 +254,8 @@ Description: Generated by yolo dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret', detach=True, clearsign=False).data - - session.commit() + dist.dirty = False + session.commit() def web_addpkg(self, reponame, name, version, fobj, dist): repo = get_repo(db(), reponame) @@ -312,15 +317,22 @@ class AptWeb(object): @cherrypy.expose def index(self, reponame=None): if reponame: - #TODO - yield "about apt repo '{}'".format(reponame) + repo = get_repo(db(), reponame, create_ok=False) + + yield "pubkey
".format(reponame=repo.name) + + for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all(): + yield "{name}: Packages Release Release.gpg
".format(reponame=repo.name, name=dist.name) + + # yield "about apt repo '{}'".format(reponame) else: - #TODO - yield "about all apt repos" + for repo in db().query(AptRepo).order_by(AptRepo.name).all(): + yield "{name}
".format(name=repo.name) @cherrypy.expose def pubkey(self, reponame=None): - yield get_repo(db(), reponame, create_ok=False).gpgpubkey + cherrypy.response.headers['Content-Type'] = 'text/plain' + return get_repo(db(), reponame, create_ok=False).gpgpubkey @cherrypy.expose @@ -331,33 +343,38 @@ class AptDists(object): self.base = base def __call__(self, *segments, reponame=None): + repo = get_repo(db(), reponame, create_ok=False) + if len(segments) == 4 and segments[3] == "Packages": distname, componentname, indexname, pkgs = segments - - repo = get_repo(db(), reponame, create_ok=False) dist = get_dist(db(), repo, distname, create_ok=False) if not repo or not dist: raise cherrypy.HTTPError(404) - yield dist.packages_cache - - return + cherrypy.response.headers['Content-Type'] = 'text/plain' + return dist.packages_cache elif len(segments) == 2: distname, target = segments - - repo = get_repo(db(), reponame, create_ok=False) dist = get_dist(db(), repo, distname, create_ok=False) + cherrypy.response.headers['Content-Type'] = 'text/plain' if target == "Release": - # yield "Release for repo={} dist={}".format(reponame, distname) - yield dist.release_cache - return - + return dist.release_cache elif target == "Release.gpg": - yield dist.sig_cache - return + return dist.sig_cache + else: + raise cherrypy.HTTPError(404) + + elif len(segments) == 1: + distname = segments[0] + dist = get_dist(db(), repo, distname, create_ok=False) + body = "" + for package in db().query(AptPackage).filter(AptPackage.repo == repo, + AptPackage.dist == dist).order_by(AptPackage.fname).all(): + body += "{fname}
".format(reponame=repo.name, fname=package.fname) + return body raise cherrypy.HTTPError(404) @@ -378,8 +395,7 @@ class AptFiles(object): if not package: raise cherrypy.HTTPError(404) - dpath = os.path.join(self.base.basepath, "repos", repo.name, "packages", package.fname[0], package.fname) - + dpath = os.path.join(self.base.basepath, package.blobpath) response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath) cherrypy.response.headers["Content-Type"] = "application/x-debian-package"