dave
/
docker-artifact
1
0
Fork 0
Code Issues 4 Pull Requests Releases Wiki Activity

regen queue instead of annoying poller

This commit is contained in:
dave 2019-05-01 18:45:52 -07:00
parent 51ba72f042
commit 4f5966b415
1 changed files with 77 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
repobot/aptprovider.py
View File

@ -11,11 +11,11 @@ from tempfile import TemporaryDirectory
from threading import Thread from threading import Thread
import hashlib import hashlib
import os import os
from time import sleep
import gnupg import gnupg
from datetime import datetime from datetime import datetime
import traceback import traceback
import json import json
import queue
class AptRepo(Base): class AptRepo(Base):
@ -150,6 +150,8 @@ class AptProvider(object):
self.bucket = bucket self.bucket = bucket
"""base path within the s3 bucket""" """base path within the s3 bucket"""
self.basepath = "data/provider/apt" self.basepath = "data/provider/apt"
"""queue entries are tuples containing the database id of the dist to regenerate indexes and signatures for"""
self.queue = queue.Queue()
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False, cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}}) 'tools.db.on': True}})
@ -166,20 +168,22 @@ class AptProvider(object):
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
Session.configure(bind=get_engine()) Session.configure(bind=get_engine())
while True: while True:
sleep(2) try:
work = self.queue.get(block=True, timeout=5)
except queue.Empty:
continue
session = Session() session = Session()
try: try:
self._sign_packages(session) self._sign_packages(session, work)
except: except:
traceback.print_exc() traceback.print_exc()
finally: finally:
session.close() session.close()
sleep(10)
def _sign_packages(self, session): def _sign_packages(self, session, work):
dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all() dist_id = work[0]
dist = session.query(AptDist).filter(AptDist.id == dist_id).first()
for dist in dirtydists:
print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name)) print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name))
str_packages = "" str_packages = ""
@ -210,7 +214,7 @@ Codename: {dist}
Date: {time} Date: {time}
Architectures: amd64 Architectures: amd64
Components: main Components: main
Description: Generated by yolo Description: Generated by Repobot
""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) """.format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
for algo, algoname in algos.items(): for algo, algoname in algos.items():
str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname, str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname,
@ -256,6 +260,7 @@ Description: Generated by yolo
detach=True, clearsign=False).data detach=True, clearsign=False).data
dist.dirty = False dist.dirty = False
session.commit() session.commit()
print("Metadata generation complete")
def web_addpkg(self, reponame, name, version, fobj, dist): def web_addpkg(self, reponame, name, version, fobj, dist):
repo = get_repo(db(), reponame) repo = get_repo(db(), reponame)
@ -300,6 +305,11 @@ Description: Generated by yolo
db().add(pkg) db().add(pkg)
db().commit() db().commit()
self.regen_dist(dist.id)
def regen_dist(self, dist_id):
self.queue.put((dist_id, ))
#TODO #TODO
# - verify dpkg name & version match params # - verify dpkg name & version match params
# - copy to persistent storage # - copy to persistent storage
@ -315,14 +325,17 @@ class AptWeb(object):
self.packages = AptFiles(base) self.packages = AptFiles(base)
@cherrypy.expose @cherrypy.expose
def index(self, reponame=None): def index(self, reponame=None, regen=False):
if reponame: if reponame:
repo = get_repo(db(), reponame, create_ok=False) repo = get_repo(db(), reponame, create_ok=False)
yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a><hr/>".format(reponame=repo.name) yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a> " \
"<a href='/repo/apt/{reponame}?regen=1'>regen</a><hr/>".format(reponame=repo.name)
for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all(): for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all():
yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name) yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name)
if regen:
self.base.regen_dist(dist.id)
# yield "about apt repo '{}'".format(reponame) # yield "about apt repo '{}'".format(reponame)
else: else: