From 4f5966b415d30016515c68fdfc2e104f3d2d2854 Mon Sep 17 00:00:00 2001 From: dave Date: Wed, 1 May 2019 18:45:52 -0700 Subject: [PATCH] regen queue instead of annoying poller --- repobot/aptprovider.py | 141 ++++++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 64 deletions(-) diff --git a/repobot/aptprovider.py b/repobot/aptprovider.py index a1c0483..77d07a6 100644 --- a/repobot/aptprovider.py +++ b/repobot/aptprovider.py @@ -11,11 +11,11 @@ from tempfile import TemporaryDirectory from threading import Thread import hashlib import os -from time import sleep import gnupg from datetime import datetime import traceback import json +import queue class AptRepo(Base): @@ -150,6 +150,8 @@ class AptProvider(object): self.bucket = bucket """base path within the s3 bucket""" self.basepath = "data/provider/apt" + """queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for""" + self.queue = queue.Queue() cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False, 'tools.db.on': True}}) @@ -166,96 +168,99 @@ class AptProvider(object): Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) Session.configure(bind=get_engine()) while True: - sleep(2) + try: + work = self.queue.get(block=True, timeout=5) + except queue.Empty: + continue + session = Session() try: - self._sign_packages(session) + self._sign_packages(session, work) except: traceback.print_exc() finally: session.close() - sleep(10) - def _sign_packages(self, session): - dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all() + def _sign_packages(self, session, work): + dist_id = work[0] + dist = session.query(AptDist).filter(AptDist.id == dist_id).first() + print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name)) - for dist in dirtydists: - print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name)) + str_packages = "" - str_packages = "" + for package in session.query(AptPackage) \ + .filter(AptPackage.repo == dist.repo, + AptPackage.dist == dist) \ + .order_by(AptPackage.id).all(): + fields = json.loads(package.fields) + for k, v in fields.items(): + str_packages += "{}: {}\n".format(k, v) + for algo, algoname in algos.items(): + str_packages += "{}: {}\n".format(algoname, getattr(package, algo)) - for package in session.query(AptPackage) \ - .filter(AptPackage.repo == dist.repo, - AptPackage.dist == dist) \ - .order_by(AptPackage.id).all(): - fields = json.loads(package.fields) - for k, v in fields.items(): - str_packages += "{}: {}\n".format(k, v) - for algo, algoname in algos.items(): - str_packages += "{}: {}\n".format(algoname, getattr(package, algo)) + str_packages += "Filename: packages/{}/{}\n".format(package.fname[0], package.fname) + str_packages += "Size: {}\n".format(package.size) - str_packages += "Filename: packages/{}/{}\n".format(package.fname[0], package.fname) - str_packages += "Size: {}\n".format(package.size) + str_packages += "\n" - str_packages += "\n" + dist.packages_cache = str_packages.encode("utf-8") - dist.packages_cache = str_packages.encode("utf-8") + release_hashes = hashmany(dist.packages_cache) - release_hashes = hashmany(dist.packages_cache) - - str_release = """Origin: . {dist} + str_release = """Origin: . {dist} Label: . {dist} Suite: {dist} Codename: {dist} Date: {time} Architectures: amd64 Components: main -Description: Generated by yolo +Description: Generated by Repobot """.format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) - for algo, algoname in algos.items(): - str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname, - release_hashes[algo], - len(dist.packages_cache), - "main", #TODO component - "binary-amd64", #TODO whatever this was - "Packages") + for algo, algoname in algos.items(): + str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname, + release_hashes[algo], + len(dist.packages_cache), + "main", #TODO component + "binary-amd64", #TODO whatever this was + "Packages") - dist.release_cache = str_release.encode("utf-8") + dist.release_cache = str_release.encode("utf-8") - keyemail = 'debian_signing@localhost' + keyemail = 'debian_signing@localhost' - with TemporaryDirectory() as tdir: - gpg = gnupg.GPG(gnupghome=tdir) + with TemporaryDirectory() as tdir: + gpg = gnupg.GPG(gnupghome=tdir) - def getkey(): - keys = [i for i in gpg.list_keys(secret=True) if any([keyemail in k for k in i["uids"]])] - if keys: - return keys[0] + def getkey(): + keys = [i for i in gpg.list_keys(secret=True) if any([keyemail in k for k in i["uids"]])] + if keys: + return keys[0] - fingerprint = None + fingerprint = None - if not dist.repo.gpgkey: - print("Generating key for", dist.repo.name) - key = gpg.gen_key(gpg.gen_key_input(name_email=keyemail, - expire_date='2029-04-28', - key_type='RSA', - key_length=4096, - key_usage='encrypt,sign,auth', - passphrase="secret")) - fingerprint = key.fingerprint - dist.repo.gpgkey = gpg.export_keys(fingerprint, secret=True, passphrase="secret") - dist.repo.gpgkeyprint = fingerprint - dist.repo.gpgpubkey = gpg.export_keys(fingerprint) + if not dist.repo.gpgkey: + print("Generating key for", dist.repo.name) + key = gpg.gen_key(gpg.gen_key_input(name_email=keyemail, + expire_date='2029-04-28', + key_type='RSA', + key_length=4096, + key_usage='encrypt,sign,auth', + passphrase="secret")) + fingerprint = key.fingerprint + dist.repo.gpgkey = gpg.export_keys(fingerprint, secret=True, passphrase="secret") + dist.repo.gpgkeyprint = fingerprint + dist.repo.gpgpubkey = gpg.export_keys(fingerprint) - else: - import_result = gpg.import_keys(dist.repo.gpgkey) - fingerprint = import_result.results[0]['fingerprint'] # errors here suggests some gpg import issue - assert(fingerprint == getkey()['fingerprint']) + else: + import_result = gpg.import_keys(dist.repo.gpgkey) + fingerprint = import_result.results[0]['fingerprint'] # errors here suggests some gpg import issue + assert(fingerprint == getkey()['fingerprint']) - dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret', - detach=True, clearsign=False).data - dist.dirty = False - session.commit() + dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret', + detach=True, clearsign=False).data + dist.dirty = False + session.commit() + print("Metadata generation complete") def web_addpkg(self, reponame, name, version, fobj, dist): repo = get_repo(db(), reponame) @@ -300,6 +305,11 @@ Description: Generated by yolo db().add(pkg) db().commit() + self.regen_dist(dist.id) + + def regen_dist(self, dist_id): + self.queue.put((dist_id, )) + #TODO # - verify dpkg name & version match params # - copy to persistent storage @@ -315,14 +325,17 @@ class AptWeb(object): self.packages = AptFiles(base) @cherrypy.expose - def index(self, reponame=None): + def index(self, reponame=None, regen=False): if reponame: repo = get_repo(db(), reponame, create_ok=False) - yield "pubkey
".format(reponame=repo.name) + yield "pubkey " \ + "regen
".format(reponame=repo.name) for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all(): yield "{name}: Packages Release Release.gpg
".format(reponame=repo.name, name=dist.name) + if regen: + self.base.regen_dist(dist.id) # yield "about apt repo '{}'".format(reponame) else: