browser navigatable

repobot/aptprovider.py

@@ -13,6 +13,7 @@ import hashlib
 import os
 from time import sleep
 import gnupg
+from datetime import datetime
 import traceback
 import json
 
@@ -25,6 +26,8 @@ class AptRepo(Base):
     gpgkeyprint = Column(Text(), nullable=True)
     gpgpubkey = Column(Text(), nullable=True)
 
+    dists = relationship("AptDist")
+
 
 class AptDist(Base):
     __tablename__ = 'aptdist'
@@ -74,11 +77,13 @@ class AptPackage(Base):
 
     @property
     def blobpath(self):
-        return "{}/{}/packages/{}/{}_{}.deb".format(self.repo.name, self.dist.name,
+        return os.path.join("repos", self.repo.name, "packages", self.name[0], self.fname)
-                                                    self.name[0], self.name, self.sha256[0:8])
 
 
 def get_repo(_db, repo_name, create_ok=True):
+    """
+    Fetch a repo from the database by name
+    """
     repo = _db.query(AptRepo).filter(AptRepo.name == repo_name).first()
     if not repo and create_ok:
         repo = AptRepo(name=repo_name)
@@ -88,6 +93,9 @@ def get_repo(_db, repo_name, create_ok=True):
 
 
 def get_dist(_db, repo, dist_name, create_ok=True):
+    """
+    Fetch a repo's dist from the database by name
+    """
     dist = _db.query(AptDist).filter(AptDist.name == dist_name, AptDist.repo_id == repo.id).first()
     if not dist and create_ok:
         dist = AptDist(name=dist_name, repo_id=repo.id)
@@ -123,7 +131,7 @@ def copyhash(fin, fout):
 
 def hashmany(data):
     """
-    Copy a file and calculate hashes while doing so
+    Hash the input data using several algos
     """
     hashes = {}
     for algo in algos.keys():
@@ -140,11 +148,9 @@ class AptProvider(object):
         self.db = dbcon
         self.s3 = s3client
         self.bucket = bucket
+        """base path within the s3 bucket"""
         self.basepath = "data/provider/apt"
-        """
+
-        bucket path (after basedir)
-        repos/{reponame}/packages/f/foo.deb
-        """
         cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
                                                               'tools.db.on': True}})
 
@@ -157,24 +163,24 @@ class AptProvider(object):
         self.updater.start()
 
     def sign_packages(self):
+        Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
+        Session.configure(bind=get_engine())
         while True:
             sleep(2)
+            session = Session()
             try:
-                self._sign_packages()
+                self._sign_packages(session)
             except:
                 traceback.print_exc()
-            # sleep(10)
+            finally:
-            break
+                session.close()
-
+            sleep(10)
-    def _sign_packages(self):
-        print("signing packages")
-        session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False))
-        session.configure(bind=get_engine())
 
+    def _sign_packages(self, session):
         dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all()
 
         for dist in dirtydists:
-            print("Signing dist {}/{}".format(dist.repo.name, dist.name))
+            print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name))
 
             str_packages = ""
 
@@ -196,17 +202,16 @@ class AptProvider(object):
             dist.packages_cache = str_packages.encode("utf-8")
 
             release_hashes = hashmany(dist.packages_cache)
-            print(release_hashes)
 
             str_release = """Origin: . {dist}
 Label: . {dist}
 Suite: {dist}
 Codename: {dist}
-Date: Fri, 2 Nov 2018 04:58:59 UTC
+Date: {time}
 Architectures: amd64
 Components: main
 Description: Generated by yolo
-""".format(dist=dist.name)
+""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC"))
             for algo, algoname in algos.items():
                 str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname,
                                                                release_hashes[algo],
@@ -249,8 +254,8 @@ Description: Generated by yolo
 
                 dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret',
                                           detach=True, clearsign=False).data
-
+                dist.dirty = False
-        session.commit()
+                session.commit()
 
     def web_addpkg(self, reponame, name, version, fobj, dist):
         repo = get_repo(db(), reponame)
@@ -312,15 +317,22 @@ class AptWeb(object):
     @cherrypy.expose
     def index(self, reponame=None):
         if reponame:
-            #TODO
+            repo = get_repo(db(), reponame, create_ok=False)
-            yield "about apt repo '{}'".format(reponame)
+
+            yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a><hr/>".format(reponame=repo.name)
+
+            for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all():
+                yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name)
+
+            # yield "about apt repo '{}'".format(reponame)
         else:
-            #TODO
+            for repo in db().query(AptRepo).order_by(AptRepo.name).all():
-            yield "about all apt repos"
+                yield "<a href='/repo/apt/{name}'>{name}</a><br/>".format(name=repo.name)
 
     @cherrypy.expose
     def pubkey(self, reponame=None):
-        yield get_repo(db(), reponame, create_ok=False).gpgpubkey
+        cherrypy.response.headers['Content-Type'] = 'text/plain'
+        return get_repo(db(), reponame, create_ok=False).gpgpubkey
 
 
 @cherrypy.expose
@@ -331,33 +343,38 @@ class AptDists(object):
         self.base = base
 
     def __call__(self, *segments, reponame=None):
+        repo = get_repo(db(), reponame, create_ok=False)
+
         if len(segments) == 4 and segments[3] == "Packages":
             distname, componentname, indexname, pkgs = segments
-
-            repo = get_repo(db(), reponame, create_ok=False)
             dist = get_dist(db(), repo, distname, create_ok=False)
 
             if not repo or not dist:
                 raise cherrypy.HTTPError(404)
 
-            yield dist.packages_cache
+            cherrypy.response.headers['Content-Type'] = 'text/plain'
-
+            return dist.packages_cache
-            return
 
         elif len(segments) == 2:
             distname, target = segments
-
-            repo = get_repo(db(), reponame, create_ok=False)
             dist = get_dist(db(), repo, distname, create_ok=False)
 
+            cherrypy.response.headers['Content-Type'] = 'text/plain'
             if target == "Release":
-                # yield "Release for repo={} dist={}".format(reponame, distname)
+                return dist.release_cache
-                yield dist.release_cache
-                return
-
             elif target == "Release.gpg":
-                yield dist.sig_cache
+                return dist.sig_cache
-                return
+            else:
+                raise cherrypy.HTTPError(404)
+
+        elif len(segments) == 1:
+            distname = segments[0]
+            dist = get_dist(db(), repo, distname, create_ok=False)
+            body = ""
+            for package in db().query(AptPackage).filter(AptPackage.repo == repo,
+                                                         AptPackage.dist == dist).order_by(AptPackage.fname).all():
+                body += "<a href='/repo/apt/{reponame}/packages/{fname[0]}/{fname}'>{fname}</a><br />".format(reponame=repo.name, fname=package.fname)
+            return body
 
         raise cherrypy.HTTPError(404)
 
@@ -378,8 +395,7 @@ class AptFiles(object):
         if not package:
             raise cherrypy.HTTPError(404)
 
-        dpath = os.path.join(self.base.basepath, "repos", repo.name, "packages", package.fname[0], package.fname)
+        dpath = os.path.join(self.base.basepath, package.blobpath)
-
         response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath)
 
         cherrypy.response.headers["Content-Type"] = "application/x-debian-package"