import dist metadata from source repo

pydebmirror/cli2.py

@@ -9,6 +9,10 @@ from email import message_from_string
 from dataclasses import dataclass
 
 
+def message_to_string(message):
+    return message.as_string()[0:-2]
+
+
 def dict_factory(c, row):
     d = {}
     for idx, col in enumerate(c.description):
@@ -42,6 +46,10 @@ def get_db(db_file):
             'name'          TEXT,
             'version'       TEXT,
             UNIQUE(dist, component, arch, name, version)
+        """CREATE TABLE IF NOT EXISTS 'dist_meta' (
+            'dist'          TEXT,
+            'metadata'      TEXT,
+            UNIQUE(dist)
         )""",
     ]
 
@@ -105,7 +113,11 @@ class Repoline:
         dist_path = "{}dists/{}/".format(self.base_url, self.dist)
         r = requests.get("{}Release".format(dist_path))
         r.raise_for_status()
-        # release = r.text
+        release = message_from_string(r.text)
+
+        ignore_keys = ('MD5Sum', 'SHA1', 'SHA256', 'Acquire-By-Hash', )
+        for key in ignore_keys:
+            del release[key]
 
         # parse out each component's Packages/.gz/.xz file
         # files = parse_dist_release(release)
@@ -133,7 +145,7 @@ class Repoline:
                 for p in fetch_packages_file("{}{}".format(dist_path, component_prefix)).split("\n\n")[0:-1]
             ]
 
-        return packages
+        return release, packages
 
     @staticmethod
     def parse(line) -> "Repoline":
@@ -183,18 +195,19 @@ class Repo(object):
         return dist
 
     def import_source_metadata(self, line):
-        packages = line.get_packages()
+        release, packages = line.get_packages()
         dist = self.get_dist(line.dist)
         dirty = False
         with closing(self.db.cursor()) as c:
+            dist.update_metadata(c, release)
             for component_name in line.components:
                 component = dist.get_component(component_name)
                 arch = component.get_arch(line.arch)
                 for package in packages[component_name]:
                     dirty = arch.add_package(c, package) or dirty
 
-            if dirty:
-                c.execute("COMMIT")
+            # if dirty:
+            c.execute("COMMIT")
 
     def import_source_packages(self, line):
         #TODO parallelize downloads
@@ -252,6 +265,10 @@ class Dist(object):
         self.components[name] = component
         return component
 
+    def update_metadata(self, c, release):
+        c.execute("REPLACE INTO dist_meta (dist, metadata) VALUES (?, ?);",
+                  (self.name, message_to_string(release), ))
+
 
 class Component(object):
     def __init__(self, dist, name):