Software repository API
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

200 lines
7.0 KiB

  1. import os
  2. import shutil
  3. from repobot.common import plist, pmap
  4. from jinja2 import Environment, FileSystemLoader, select_autoescape
  5. import cherrypy
  6. class PkgProvider(object):
  7. def __init__(self, db, repo, datadir):
  8. """
  9. Base package provider class
  10. """
  11. self.db = db
  12. self.repo = repo
  13. self.dir = datadir
  14. def render(self):
  15. """
  16. Respond to requests to browse the repo
  17. """
  18. raise NotImplementedError()
  19. def add_package(self, pkobj, fname, fobj, params):
  20. """
  21. Add a package to the repo
  22. """
  23. raise NotImplementedError()
  24. class PyPiProvider(PkgProvider):
  25. def add_package(self, pkgobj, fname, fobj, params):
  26. if "files" not in pkgobj.data:
  27. pkgobj.data["files"] = plist()
  28. if fname in pkgobj.data["files"]:
  29. raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version))
  30. pkgdir = os.path.join(self.dir, pkgobj.name)
  31. os.makedirs(pkgdir, exist_ok=True)
  32. # TODO handle duplicate files better
  33. pkgfilepath = os.path.join(pkgdir, fname)
  34. with open(pkgfilepath, "wb") as fdest:
  35. shutil.copyfileobj(fobj, fdest)
  36. pkgobj.data["files"].append(fname)
  37. def browse(self, args):
  38. tpl = Environment(loader=FileSystemLoader("templates"), autoescape=select_autoescape(['html', 'xml']))
  39. if len(args) == 0: # repo root
  40. return tpl.get_template("pypi/root.html"). \
  41. render(reponame=self.repo.name,
  42. packages=self.repo.packages.keys())
  43. elif len(args) == 1: # single module dir
  44. files = []
  45. if args[0] not in self.repo.packages:
  46. raise cherrypy.HTTPError(404, 'Invalid package')
  47. for _, version in self.repo.packages[args[0]].items():
  48. files += version.data["files"]
  49. return tpl.get_template("pypi/project.html"). \
  50. render(reponame=self.repo.name,
  51. modulename=args[0],
  52. files=files)
  53. elif len(args) == 2: # fetch file
  54. fpath = os.path.join(self.dir, args[0], args[1])
  55. return cherrypy.lib.static.serve_file(os.path.abspath(fpath), "application/octet-stream")
  56. from subprocess import check_call, check_output, Popen, PIPE
  57. from tempfile import NamedTemporaryFile, TemporaryDirectory
  58. import json
  59. class AptlyConfig(object):
  60. """
  61. Context manager providing an aptly config file
  62. """
  63. def __init__(self, rootdir):
  64. self.conf = {"rootDir": rootdir} # , "gpgDisableSign": True, "gpgDisableVerify": True}
  65. self.file = None
  66. def __enter__(self):
  67. self.file = NamedTemporaryFile()
  68. with open(self.file.name, "w") as f:
  69. f.write(json.dumps(self.conf))
  70. return self.file.name
  71. def __exit__(self, *args):
  72. self.file.close()
  73. class AptProvider(PkgProvider):
  74. def add_package(self, pkgobj, fname, fobj, params):
  75. # first package added sets the Distribution of the repo
  76. # subsequent package add MUST specify the same dist
  77. if "dist" not in self.repo.data:
  78. self.repo.data["dist"] = params["dist"]
  79. assert self.repo.data["dist"] == params["dist"]
  80. # Generate a GPG key to sign packages in this repo
  81. # TODO support passing keypath=... param to import existing keys and maybe other key generation options
  82. if not os.path.exists(self._gpg_dir):
  83. self._generate_gpg_key()
  84. if "files" not in pkgobj.data:
  85. pkgobj.data["files"] = plist()
  86. if fname in pkgobj.data["files"]:
  87. # raise Exception("File {} already in package {}-{}".format(fname, pkgobj.name, pkgobj.version))
  88. pass
  89. with AptlyConfig(self.dir) as conf:
  90. if not os.path.exists(os.path.join(self.dir, "db")):
  91. os.makedirs(self.dir, exist_ok=True)
  92. check_call(["aptly", "-config", conf, "repo", "create",
  93. "-distribution", self.repo.data["dist"], "main"]) # TODO dist param
  94. # put the file somewhere for now
  95. with TemporaryDirectory() as tdir:
  96. tmppkgpath = os.path.join(tdir, fname)
  97. with open(tmppkgpath, "wb") as fdest:
  98. shutil.copyfileobj(fobj, fdest)
  99. check_call(["aptly", "-config", conf, "repo", "add", "main", tmppkgpath])
  100. if not os.path.exists(os.path.join(self.dir, "public")):
  101. check_call(["aptly", "-config", conf, "publish", "repo", "main"],
  102. env=self._env)
  103. else:
  104. check_call(["aptly", "-config", conf, "publish", "update",
  105. "-force-overwrite", self.repo.data["dist"]],
  106. env=self._env)
  107. # Make the public key available for clients
  108. self._export_pubkey()
  109. pkgobj.data["files"].append(fname)
  110. # TODO validate deb file name version against user passed version
  111. def browse(self, args):
  112. fpath = os.path.abspath(os.path.join(self.dir, "public", *args))
  113. if not os.path.exists(fpath):
  114. raise cherrypy.HTTPError(404)
  115. return cherrypy.lib.static.serve_file(fpath)
  116. def _generate_gpg_key(self):
  117. """
  118. Generate a GPG key for signing packages in this repo. Because only gpg2 supports unattended generation of
  119. passwordless keys we generate the key with gpg2 then export/import it into gpg1.
  120. """
  121. # Generate the key
  122. os.makedirs(self._gpg_dir)
  123. proc = Popen(["gpg", "--batch", "--gen-key"], stdin=PIPE, env=self._env)
  124. proc.stdin.write("""%no-protection
  125. Key-Type: rsa
  126. Key-Length: 1024
  127. Subkey-Type: default
  128. Subkey-Length: 1024
  129. Name-Real: Apt Master
  130. Name-Comment: Apt signing key
  131. Name-Email: aptmaster@localhost
  132. Expire-Date: 0
  133. %commit""".encode("ascii"))
  134. proc.stdin.close()
  135. proc.wait()
  136. assert proc.returncode == 0
  137. # Export the private key
  138. keydata = check_output(["gpg", "--export-secret-key", "--armor", "aptmaster@localhost"], env=self._env)
  139. shutil.rmtree(self._gpg_dir)
  140. os.makedirs(self._gpg_dir)
  141. # Import the private key
  142. proc = Popen(["gpg1", "--import"], stdin=PIPE, env=self._env)
  143. proc.stdin.write(keydata)
  144. proc.stdin.close()
  145. proc.wait()
  146. assert proc.returncode == 0
  147. def _export_pubkey(self):
  148. keypath = os.path.join(self.dir, "public", "repo.key")
  149. if not os.path.exists(keypath):
  150. keydata = check_output(["gpg", "--export", "--armor", "aptmaster@localhost"], env=self._env)
  151. with open(keypath, "wb") as f:
  152. f.write(keydata)
  153. @property
  154. def _env(self):
  155. """
  156. Return env vars to be used for subprocesses of this module
  157. """
  158. print(os.environ["PATH"])
  159. return {"GNUPGHOME": self._gpg_dir,
  160. "PATH": os.environ["PATH"]}
  161. @property
  162. def _gpg_dir(self):
  163. return os.path.join(self.dir, "gpg")
  164. providers = {"pypi": PyPiProvider,
  165. "apt": AptProvider}