|
|
@ -13,6 +13,7 @@ import hashlib |
|
|
|
import os |
|
|
|
from time import sleep |
|
|
|
import gnupg |
|
|
|
from datetime import datetime |
|
|
|
import traceback |
|
|
|
import json |
|
|
|
|
|
|
@ -25,6 +26,8 @@ class AptRepo(Base): |
|
|
|
gpgkeyprint = Column(Text(), nullable=True) |
|
|
|
gpgpubkey = Column(Text(), nullable=True) |
|
|
|
|
|
|
|
dists = relationship("AptDist") |
|
|
|
|
|
|
|
|
|
|
|
class AptDist(Base): |
|
|
|
__tablename__ = 'aptdist' |
|
|
@ -74,11 +77,13 @@ class AptPackage(Base): |
|
|
|
|
|
|
|
@property |
|
|
|
def blobpath(self): |
|
|
|
return "{}/{}/packages/{}/{}_{}.deb".format(self.repo.name, self.dist.name, |
|
|
|
self.name[0], self.name, self.sha256[0:8]) |
|
|
|
return os.path.join("repos", self.repo.name, "packages", self.name[0], self.fname) |
|
|
|
|
|
|
|
|
|
|
|
def get_repo(_db, repo_name, create_ok=True): |
|
|
|
""" |
|
|
|
Fetch a repo from the database by name |
|
|
|
""" |
|
|
|
repo = _db.query(AptRepo).filter(AptRepo.name == repo_name).first() |
|
|
|
if not repo and create_ok: |
|
|
|
repo = AptRepo(name=repo_name) |
|
|
@ -88,6 +93,9 @@ def get_repo(_db, repo_name, create_ok=True): |
|
|
|
|
|
|
|
|
|
|
|
def get_dist(_db, repo, dist_name, create_ok=True): |
|
|
|
""" |
|
|
|
Fetch a repo's dist from the database by name |
|
|
|
""" |
|
|
|
dist = _db.query(AptDist).filter(AptDist.name == dist_name, AptDist.repo_id == repo.id).first() |
|
|
|
if not dist and create_ok: |
|
|
|
dist = AptDist(name=dist_name, repo_id=repo.id) |
|
|
@ -123,7 +131,7 @@ def copyhash(fin, fout): |
|
|
|
|
|
|
|
def hashmany(data): |
|
|
|
""" |
|
|
|
Copy a file and calculate hashes while doing so |
|
|
|
Hash the input data using several algos |
|
|
|
""" |
|
|
|
hashes = {} |
|
|
|
for algo in algos.keys(): |
|
|
@ -140,11 +148,9 @@ class AptProvider(object): |
|
|
|
self.db = dbcon |
|
|
|
self.s3 = s3client |
|
|
|
self.bucket = bucket |
|
|
|
"""base path within the s3 bucket""" |
|
|
|
self.basepath = "data/provider/apt" |
|
|
|
""" |
|
|
|
bucket path (after basedir) |
|
|
|
repos/{reponame}/packages/f/foo.deb |
|
|
|
""" |
|
|
|
|
|
|
|
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False, |
|
|
|
'tools.db.on': True}}) |
|
|
|
|
|
|
@ -157,24 +163,24 @@ class AptProvider(object): |
|
|
|
self.updater.start() |
|
|
|
|
|
|
|
def sign_packages(self): |
|
|
|
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) |
|
|
|
Session.configure(bind=get_engine()) |
|
|
|
while True: |
|
|
|
sleep(2) |
|
|
|
session = Session() |
|
|
|
try: |
|
|
|
self._sign_packages() |
|
|
|
self._sign_packages(session) |
|
|
|
except: |
|
|
|
traceback.print_exc() |
|
|
|
# sleep(10) |
|
|
|
break |
|
|
|
|
|
|
|
def _sign_packages(self): |
|
|
|
print("signing packages") |
|
|
|
session = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)) |
|
|
|
session.configure(bind=get_engine()) |
|
|
|
finally: |
|
|
|
session.close() |
|
|
|
sleep(10) |
|
|
|
|
|
|
|
def _sign_packages(self, session): |
|
|
|
dirtydists = session.query(AptDist).filter(AptDist.dirty == True).all() |
|
|
|
|
|
|
|
for dist in dirtydists: |
|
|
|
print("Signing dist {}/{}".format(dist.repo.name, dist.name)) |
|
|
|
print("Generating metadata for repo:{} dist:{}".format(dist.repo.name, dist.name)) |
|
|
|
|
|
|
|
str_packages = "" |
|
|
|
|
|
|
@ -196,17 +202,16 @@ class AptProvider(object): |
|
|
|
dist.packages_cache = str_packages.encode("utf-8") |
|
|
|
|
|
|
|
release_hashes = hashmany(dist.packages_cache) |
|
|
|
print(release_hashes) |
|
|
|
|
|
|
|
str_release = """Origin: . {dist} |
|
|
|
Label: . {dist} |
|
|
|
Suite: {dist} |
|
|
|
Codename: {dist} |
|
|
|
Date: Fri, 2 Nov 2018 04:58:59 UTC |
|
|
|
Date: {time} |
|
|
|
Architectures: amd64 |
|
|
|
Components: main |
|
|
|
Description: Generated by yolo |
|
|
|
""".format(dist=dist.name) |
|
|
|
""".format(dist=dist.name, time=datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S UTC")) |
|
|
|
for algo, algoname in algos.items(): |
|
|
|
str_release += "{}:\n {} {} {}/{}/{}\n".format(algoname, |
|
|
|
release_hashes[algo], |
|
|
@ -249,8 +254,8 @@ Description: Generated by yolo |
|
|
|
|
|
|
|
dist.sig_cache = gpg.sign(dist.release_cache, keyid=fingerprint, passphrase='secret', |
|
|
|
detach=True, clearsign=False).data |
|
|
|
|
|
|
|
session.commit() |
|
|
|
dist.dirty = False |
|
|
|
session.commit() |
|
|
|
|
|
|
|
def web_addpkg(self, reponame, name, version, fobj, dist): |
|
|
|
repo = get_repo(db(), reponame) |
|
|
@ -312,15 +317,22 @@ class AptWeb(object): |
|
|
|
@cherrypy.expose |
|
|
|
def index(self, reponame=None): |
|
|
|
if reponame: |
|
|
|
#TODO |
|
|
|
yield "about apt repo '{}'".format(reponame) |
|
|
|
repo = get_repo(db(), reponame, create_ok=False) |
|
|
|
|
|
|
|
yield "<a href='/repo/apt/{reponame}/pubkey'>pubkey</a><hr/>".format(reponame=repo.name) |
|
|
|
|
|
|
|
for dist in db().query(AptDist).filter(AptDist.repo == repo).order_by(AptDist.name).all(): |
|
|
|
yield "<a href='/repo/apt/{reponame}/dists/{name}'>{name}</a>: <a href='/repo/apt/{reponame}/dists/{name}/main/indexname/Packages'>Packages</a> <a href='/repo/apt/{reponame}/dists/{name}/Release'>Release</a> <a href='/repo/apt/{reponame}/dists/{name}/Release.gpg'>Release.gpg</a><br />".format(reponame=repo.name, name=dist.name) |
|
|
|
|
|
|
|
# yield "about apt repo '{}'".format(reponame) |
|
|
|
else: |
|
|
|
#TODO |
|
|
|
yield "about all apt repos" |
|
|
|
for repo in db().query(AptRepo).order_by(AptRepo.name).all(): |
|
|
|
yield "<a href='/repo/apt/{name}'>{name}</a><br/>".format(name=repo.name) |
|
|
|
|
|
|
|
@cherrypy.expose |
|
|
|
def pubkey(self, reponame=None): |
|
|
|
yield get_repo(db(), reponame, create_ok=False).gpgpubkey |
|
|
|
cherrypy.response.headers['Content-Type'] = 'text/plain' |
|
|
|
return get_repo(db(), reponame, create_ok=False).gpgpubkey |
|
|
|
|
|
|
|
|
|
|
|
@cherrypy.expose |
|
|
@ -331,33 +343,38 @@ class AptDists(object): |
|
|
|
self.base = base |
|
|
|
|
|
|
|
def __call__(self, *segments, reponame=None): |
|
|
|
repo = get_repo(db(), reponame, create_ok=False) |
|
|
|
|
|
|
|
if len(segments) == 4 and segments[3] == "Packages": |
|
|
|
distname, componentname, indexname, pkgs = segments |
|
|
|
|
|
|
|
repo = get_repo(db(), reponame, create_ok=False) |
|
|
|
dist = get_dist(db(), repo, distname, create_ok=False) |
|
|
|
|
|
|
|
if not repo or not dist: |
|
|
|
raise cherrypy.HTTPError(404) |
|
|
|
|
|
|
|
yield dist.packages_cache |
|
|
|
|
|
|
|
return |
|
|
|
cherrypy.response.headers['Content-Type'] = 'text/plain' |
|
|
|
return dist.packages_cache |
|
|
|
|
|
|
|
elif len(segments) == 2: |
|
|
|
distname, target = segments |
|
|
|
|
|
|
|
repo = get_repo(db(), reponame, create_ok=False) |
|
|
|
dist = get_dist(db(), repo, distname, create_ok=False) |
|
|
|
|
|
|
|
cherrypy.response.headers['Content-Type'] = 'text/plain' |
|
|
|
if target == "Release": |
|
|
|
# yield "Release for repo={} dist={}".format(reponame, distname) |
|
|
|
yield dist.release_cache |
|
|
|
return |
|
|
|
|
|
|
|
return dist.release_cache |
|
|
|
elif target == "Release.gpg": |
|
|
|
yield dist.sig_cache |
|
|
|
return |
|
|
|
return dist.sig_cache |
|
|
|
else: |
|
|
|
raise cherrypy.HTTPError(404) |
|
|
|
|
|
|
|
elif len(segments) == 1: |
|
|
|
distname = segments[0] |
|
|
|
dist = get_dist(db(), repo, distname, create_ok=False) |
|
|
|
body = "" |
|
|
|
for package in db().query(AptPackage).filter(AptPackage.repo == repo, |
|
|
|
AptPackage.dist == dist).order_by(AptPackage.fname).all(): |
|
|
|
body += "<a href='/repo/apt/{reponame}/packages/{fname[0]}/{fname}'>{fname}</a><br />".format(reponame=repo.name, fname=package.fname) |
|
|
|
return body |
|
|
|
|
|
|
|
raise cherrypy.HTTPError(404) |
|
|
|
|
|
|
@ -378,8 +395,7 @@ class AptFiles(object): |
|
|
|
if not package: |
|
|
|
raise cherrypy.HTTPError(404) |
|
|
|
|
|
|
|
dpath = os.path.join(self.base.basepath, "repos", repo.name, "packages", package.fname[0], package.fname) |
|
|
|
|
|
|
|
dpath = os.path.join(self.base.basepath, package.blobpath) |
|
|
|
response = self.base.s3.get_object(Bucket=self.base.bucket, Key=dpath) |
|
|
|
|
|
|
|
cherrypy.response.headers["Content-Type"] = "application/x-debian-package" |
|
|
|