|
|
|
@ -9,6 +9,10 @@ from email import message_from_string
|
|
|
|
|
from dataclasses import dataclass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def message_to_string(message):
|
|
|
|
|
return message.as_string()[0:-2]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def dict_factory(c, row):
|
|
|
|
|
d = {}
|
|
|
|
|
for idx, col in enumerate(c.description):
|
|
|
|
@ -42,6 +46,10 @@ def get_db(db_file):
|
|
|
|
|
'name' TEXT,
|
|
|
|
|
'version' TEXT,
|
|
|
|
|
UNIQUE(dist, component, arch, name, version)
|
|
|
|
|
"""CREATE TABLE IF NOT EXISTS 'dist_meta' (
|
|
|
|
|
'dist' TEXT,
|
|
|
|
|
'metadata' TEXT,
|
|
|
|
|
UNIQUE(dist)
|
|
|
|
|
)""",
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
@ -105,7 +113,11 @@ class Repoline:
|
|
|
|
|
dist_path = "{}dists/{}/".format(self.base_url, self.dist)
|
|
|
|
|
r = requests.get("{}Release".format(dist_path))
|
|
|
|
|
r.raise_for_status()
|
|
|
|
|
# release = r.text
|
|
|
|
|
release = message_from_string(r.text)
|
|
|
|
|
|
|
|
|
|
ignore_keys = ('MD5Sum', 'SHA1', 'SHA256', 'Acquire-By-Hash', )
|
|
|
|
|
for key in ignore_keys:
|
|
|
|
|
del release[key]
|
|
|
|
|
|
|
|
|
|
# parse out each component's Packages/.gz/.xz file
|
|
|
|
|
# files = parse_dist_release(release)
|
|
|
|
@ -133,7 +145,7 @@ class Repoline:
|
|
|
|
|
for p in fetch_packages_file("{}{}".format(dist_path, component_prefix)).split("\n\n")[0:-1]
|
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
return packages
|
|
|
|
|
return release, packages
|
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
|
def parse(line) -> "Repoline":
|
|
|
|
@ -183,18 +195,19 @@ class Repo(object):
|
|
|
|
|
return dist
|
|
|
|
|
|
|
|
|
|
def import_source_metadata(self, line):
|
|
|
|
|
packages = line.get_packages()
|
|
|
|
|
release, packages = line.get_packages()
|
|
|
|
|
dist = self.get_dist(line.dist)
|
|
|
|
|
dirty = False
|
|
|
|
|
with closing(self.db.cursor()) as c:
|
|
|
|
|
dist.update_metadata(c, release)
|
|
|
|
|
for component_name in line.components:
|
|
|
|
|
component = dist.get_component(component_name)
|
|
|
|
|
arch = component.get_arch(line.arch)
|
|
|
|
|
for package in packages[component_name]:
|
|
|
|
|
dirty = arch.add_package(c, package) or dirty
|
|
|
|
|
|
|
|
|
|
if dirty:
|
|
|
|
|
c.execute("COMMIT")
|
|
|
|
|
# if dirty:
|
|
|
|
|
c.execute("COMMIT")
|
|
|
|
|
|
|
|
|
|
def import_source_packages(self, line):
|
|
|
|
|
#TODO parallelize downloads
|
|
|
|
@ -252,6 +265,10 @@ class Dist(object):
|
|
|
|
|
self.components[name] = component
|
|
|
|
|
return component
|
|
|
|
|
|
|
|
|
|
def update_metadata(self, c, release):
|
|
|
|
|
c.execute("REPLACE INTO dist_meta (dist, metadata) VALUES (?, ?);",
|
|
|
|
|
(self.name, message_to_string(release), ))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Component(object):
|
|
|
|
|
def __init__(self, dist, name):
|
|
|
|
|