import dist metadata from source repo

This commit is contained in:
dave 2022-11-30 23:47:48 -08:00
parent b57676d972
commit ea5a42daf6
1 changed files with 22 additions and 5 deletions

View File

@ -9,6 +9,10 @@ from email import message_from_string
from dataclasses import dataclass
def message_to_string(message):
return message.as_string()[0:-2]
def dict_factory(c, row):
d = {}
for idx, col in enumerate(c.description):
@ -42,6 +46,10 @@ def get_db(db_file):
'name' TEXT,
'version' TEXT,
UNIQUE(dist, component, arch, name, version)
"""CREATE TABLE IF NOT EXISTS 'dist_meta' (
'dist' TEXT,
'metadata' TEXT,
UNIQUE(dist)
)""",
]
@ -105,7 +113,11 @@ class Repoline:
dist_path = "{}dists/{}/".format(self.base_url, self.dist)
r = requests.get("{}Release".format(dist_path))
r.raise_for_status()
# release = r.text
release = message_from_string(r.text)
ignore_keys = ('MD5Sum', 'SHA1', 'SHA256', 'Acquire-By-Hash', )
for key in ignore_keys:
del release[key]
# parse out each component's Packages/.gz/.xz file
# files = parse_dist_release(release)
@ -133,7 +145,7 @@ class Repoline:
for p in fetch_packages_file("{}{}".format(dist_path, component_prefix)).split("\n\n")[0:-1]
]
return packages
return release, packages
@staticmethod
def parse(line) -> "Repoline":
@ -183,18 +195,19 @@ class Repo(object):
return dist
def import_source_metadata(self, line):
packages = line.get_packages()
release, packages = line.get_packages()
dist = self.get_dist(line.dist)
dirty = False
with closing(self.db.cursor()) as c:
dist.update_metadata(c, release)
for component_name in line.components:
component = dist.get_component(component_name)
arch = component.get_arch(line.arch)
for package in packages[component_name]:
dirty = arch.add_package(c, package) or dirty
if dirty:
c.execute("COMMIT")
# if dirty:
c.execute("COMMIT")
def import_source_packages(self, line):
#TODO parallelize downloads
@ -252,6 +265,10 @@ class Dist(object):
self.components[name] = component
return component
def update_metadata(self, c, release):
c.execute("REPLACE INTO dist_meta (dist, metadata) VALUES (?, ?);",
(self.name, message_to_string(release), ))
class Component(object):
def __init__(self, dist, name):