diff --git a/repobot/aptprovider.py b/repobot/aptprovider.py index 0aff3c4..a1c0483 100644 --- a/repobot/aptprovider.py +++ b/repobot/aptprovider.py @@ -13,6 +13,7 @@ import hashlib import os from time import sleep import gnupg +from datetime import datetime import traceback import json @@ -25,6 +26,8 @@ class AptRepo(Base): gpgkeyprint = Column(Text(), nullable=True) gpgpubkey = Column(Text(), nullable=True) + dists = relationship("AptDist") + class AptDist(Base): __tablename__ = 'aptdist' @@ -74,11 +77,13 @@ class AptPackage(Base): @property def blobpath(self): - return "{}/{}/packages/{}/{}_{}.deb".format(self.repo.name, self.dist.name, - self.name[0], self.name, self.sha256[0:8]) + return os.path.join("repos", self.repo.name, "packages", self.name[0], self.fname) def get_repo(_db, repo_name, create_ok=True): + """ + Fetch a repo from the database by name + """ repo = _db.query(AptRepo).filter(AptRepo.name == repo_name).first() if not repo and create_ok: repo = AptRepo(name=repo_name) @@ -88,6 +93,9 @@ def get_repo(_db, repo_name, create_ok=True): def get_dist(_db, repo, dist_name, create_ok=True): + """ + Fetch a repo's dist from the database by name + """ dist = _db.query(AptDist).filter(AptDist.name == dist_name, AptDist.repo_id == repo.id).first() if not dist and create_ok: dist = AptDist(name=dist_name, repo_id=repo.id) @@ -123,7 +131,7 @@ def copyhash(fin, fout): def hashmany(data): """ - Copy a file and calculate hashes while doing so + Hash the input data using several algos """ hashes = {} for algo in algos.keys(): @@ -140,11 +148,9 @@ class AptProvider(object): self.db = dbcon self.s3 = s3client self.bucket = bucket + """base path within the s3 bucket""" self.basepath = "data/provider/apt" - """ - bucket path (after basedir) - repos/{reponame}/packages/f/foo.deb - """ + cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False, 'tools.db.on': True}}) @@ -157,24 +163,24 @@ class AptProvider(object): self.updater.start() def sign_packages(self): + Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) + Session.configure(bind=get_engine()) while True: sleep(2) + session = Session() try: - self._sign_packages() + self._sign_packages(session) except: traceback.print_exc() - # sleep(10) - break - - def _sign_packages(self): - print("signing packages") - session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)) - session.configure(bind=get_engine()) + finally: + session.close() + sleep(10) + def _sign_packages(self, session): dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all() for dist in dirtydists: - print("Signing dist {}/{}".format(dist.repo.name, dist.name)) + print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name)) str_packages = "" @@ -196,17 +202,16 @@ class AptProvider(object): dist.packages_cache = str_packages.encode("utf-8") release_hashes = hashmany(dist.packages_cache) - print(release_hashes) str_release = """Origin: . {dist} Label: . {dist} Suite: {dist} Codename: {dist} -Date: Fri, 2 Nov 2018 04:58:59 UTC +Date: {time} Architectures: amd64 Components: main Description: Generated by yolo -""".format(dist=dist.name) +""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) for algo, algoname in algos.items(): str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname, release_hashes[algo], @@ -249,8 +254,8 @@ Description: Generated by yolo dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret', detach=True, clearsign=False).data - - session.commit() + dist.dirty = False + session.commit() def web_addpkg(self, reponame, name, version, fobj, dist): repo = get_repo(db(), reponame) @@ -312,15 +317,22 @@ class AptWeb(object): @cherrypy.expose def index(self, reponame=None): if reponame: - #TODO - yield "about apt repo '{}'".format(reponame) + repo = get_repo(db(), reponame, create_ok=False) + + yield "pubkey