diff --git a/pydebmirror/cli2.py b/pydebmirror/cli2.py index 858d61c..5cdc875 100644 --- a/pydebmirror/cli2.py +++ b/pydebmirror/cli2.py @@ -9,6 +9,10 @@ from email import message_from_string from dataclasses import dataclass +def message_to_string(message): + return message.as_string()[0:-2] + + def dict_factory(c, row): d = {} for idx, col in enumerate(c.description): @@ -42,6 +46,10 @@ def get_db(db_file): 'name' TEXT, 'version' TEXT, UNIQUE(dist, component, arch, name, version) + """CREATE TABLE IF NOT EXISTS 'dist_meta' ( + 'dist' TEXT, + 'metadata' TEXT, + UNIQUE(dist) )""", ] @@ -105,7 +113,11 @@ class Repoline: dist_path = "{}dists/{}/".format(self.base_url, self.dist) r = requests.get("{}Release".format(dist_path)) r.raise_for_status() - # release = r.text + release = message_from_string(r.text) + + ignore_keys = ('MD5Sum', 'SHA1', 'SHA256', 'Acquire-By-Hash', ) + for key in ignore_keys: + del release[key] # parse out each component's Packages/.gz/.xz file # files = parse_dist_release(release) @@ -133,7 +145,7 @@ class Repoline: for p in fetch_packages_file("{}{}".format(dist_path, component_prefix)).split("\n\n")[0:-1] ] - return packages + return release, packages @staticmethod def parse(line) -> "Repoline": @@ -183,18 +195,19 @@ class Repo(object): return dist def import_source_metadata(self, line): - packages = line.get_packages() + release, packages = line.get_packages() dist = self.get_dist(line.dist) dirty = False with closing(self.db.cursor()) as c: + dist.update_metadata(c, release) for component_name in line.components: component = dist.get_component(component_name) arch = component.get_arch(line.arch) for package in packages[component_name]: dirty = arch.add_package(c, package) or dirty - if dirty: - c.execute("COMMIT") + # if dirty: + c.execute("COMMIT") def import_source_packages(self, line): #TODO parallelize downloads @@ -252,6 +265,10 @@ class Dist(object): self.components[name] = component return component + def update_metadata(self, c, release): + c.execute("REPLACE INTO dist_meta (dist, metadata) VALUES (?, ?);", + (self.name, message_to_string(release), )) + class Component(object): def __init__(self, dist, name):