regen queue instead of annoying poller

repobot/aptprovider.py

@@ -11,11 +11,11 @@ from tempfile import TemporaryDirectory
 from threading import Thread
 import hashlib
 import os
-from time import sleep
 import gnupg
 from datetime import datetime
 import traceback
 import json
+import queue
 
 
 class AptRepo(Base):
@@ -150,6 +150,8 @@ class AptProvider(object):
         self.bucket = bucket
         """base path within the s3 bucket"""
         self.basepath = "data/provider/apt"
+        """queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for"""
+        self.queue = queue.Queue()
 
         cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
                                                               'tools.db.on': True}})
@@ -166,20 +168,22 @@ class AptProvider(object):
         Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
         Session.configure(bind=get_engine())
         while True:
-            sleep(2)
+            try:
+                work = self.queue.get(block=True, timeout=5)
+            except queue.Empty:
+                continue
+
             session = Session()
             try:
-                self._sign_packages(session)
+                self._sign_packages(session, work)
             except:
                 traceback.print_exc()
             finally:
                 session.close()
-            sleep(10)
 
-    def _sign_packages(self, session):
-        dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all()
-
-        for dist in dirtydists:
+    def _sign_packages(self, session, work):
+        dist_id = work[0]
+        dist = session.query(AptDist).filter(AptDist.id == dist_id).first()
         print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name))
 
         str_packages = ""
@@ -210,7 +214,7 @@ Codename: {dist}
 Date: {time}
 Architectures: amd64
 Components: main
-Description: Generated by yolo
+Description: Generated by Repobot
 """.format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
         for algo, algoname in algos.items():
             str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname,
@@ -256,6 +260,7 @@ Description: Generated by yolo
                                       detach=True, clearsign=False).data
             dist.dirty = False
             session.commit()
+        print("Metadata generation complete")
 
     def web_addpkg(self, reponame, name, version, fobj, dist):
         repo = get_repo(db(), reponame)
@@ -300,6 +305,11 @@ Description: Generated by yolo
             db().add(pkg)
             db().commit()
 
+        self.regen_dist(dist.id)
+
+    def regen_dist(self, dist_id):
+        self.queue.put((dist_id, ))
+
         #TODO
         # - verify dpkg name & version match params
         # - copy to persistent storage
@@ -315,14 +325,17 @@ class AptWeb(object):
         self.packages = AptFiles(base)
 
     @cherrypy.expose
-    def index(self, reponame=None):
+    def index(self, reponame=None, regen=False):
         if reponame:
             repo = get_repo(db(), reponame, create_ok=False)
 
-            yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a><hr/>".format(reponame=repo.name)
+            yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a> " \
+                  "<a href='/repo/apt/{reponame}?regen=1'>regen</a><hr/>".format(reponame=repo.name)
 
             for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all():
                 yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name)
+                if regen:
+                    self.base.regen_dist(dist.id)
 
             # yield "about apt repo '{}'".format(reponame)
         else: