un-hardcode s3 and db connections

This commit is contained in:
dave 2019-05-04 21:14:49 -07:00
parent 64f51bdf7c
commit 958805a587
4 changed files with 48 additions and 39 deletions

View File

@ -14,7 +14,7 @@ from sqlalchemy.orm import relationship
from sqlalchemy.types import String, Integer, Text, BOOLEAN from sqlalchemy.types import String, Integer, Text, BOOLEAN
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from threading import Thread from threading import Thread
from repobot.tables import Base, db, get_engine from repobot.tables import Base, db
class AptRepo(Base): class AptRepo(Base):
@ -143,7 +143,7 @@ def hashmany(data):
class AptProvider(object): class AptProvider(object):
def __init__(self, dbcon, s3client, bucket="aptprovider"): def __init__(self, dbcon, s3client, bucket):
self.db = dbcon self.db = dbcon
self.s3 = s3client self.s3 = s3client
self.bucket = bucket self.bucket = bucket
@ -155,17 +155,12 @@ class AptProvider(object):
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False, cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}}) 'tools.db.on': True}})
# ensure bucket exists
if bucket not in [b['Name'] for b in self.s3.list_buckets()['Buckets']]:
print("Creating bucket")
self.s3.create_bucket(Bucket=bucket)
self.updater = Thread(target=self.sign_packages, daemon=True) self.updater = Thread(target=self.sign_packages, daemon=True)
self.updater.start() self.updater.start()
def sign_packages(self): def sign_packages(self):
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False) Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
Session.configure(bind=get_engine()) Session.configure(bind=self.db)
while True: while True:
try: try:
work = self.queue.get(block=True, timeout=5) work = self.queue.get(block=True, timeout=5)

View File

@ -170,7 +170,7 @@ def copysha256(fin, fout):
class PypiProvider(object): class PypiProvider(object):
def __init__(self, dbcon, s3client, bucket="aptprovider"): def __init__(self, dbcon, s3client, bucket):
self.db = dbcon self.db = dbcon
self.s3 = s3client self.s3 = s3client
self.bucket = bucket self.bucket = bucket
@ -180,12 +180,6 @@ class PypiProvider(object):
cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False, cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}}) 'tools.db.on': True}})
# ensure bucket exists
#TODO bucket creation should happen in server.py
if bucket not in [b['Name'] for b in self.s3.list_buckets()['Buckets']]:
print("Creating bucket")
self.s3.create_bucket(Bucket=bucket)
def web_addpkg(self, reponame, name, version, fobj): def web_addpkg(self, reponame, name, version, fobj):
repo = get_repo(db(), reponame) repo = get_repo(db(), reponame)

View File

@ -1,12 +1,13 @@
import boto3
import cherrypy import cherrypy
import logging import logging
from repobot.tables import get_engine, SAEnginePlugin, SATool import os
import sqlalchemy
from botocore.client import Config as BotoConfig
from repobot.aptprovider import AptProvider from repobot.aptprovider import AptProvider
from repobot.pypiprovider import PypiProvider from repobot.pypiprovider import PypiProvider
from repobot.tables import SAEnginePlugin, SATool
import boto3 from urllib.parse import urlparse
from botocore.client import Config as BotoConfig
class AppWeb(object): class AppWeb(object):
@ -32,33 +33,56 @@ def main():
import argparse import argparse
import signal import signal
parser = argparse.ArgumentParser(description="irc web client server") parser = argparse.ArgumentParser(description="package storage database")
parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on") parser.add_argument('-p', '--port', default=8080, type=int, help="http port to listen on")
parser.add_argument('-s', '--database', help="mysql connection string") parser.add_argument('-d', '--database', help="mysql+pymysql:// connection string",
default=os.environ.get("DATABASE_URL"))
parser.add_argument('-s', '--s3', help="http:// or https:// connection string",
default=os.environ.get("S3_URL"))
parser.add_argument('--debug', action="store_true", help="enable development options") parser.add_argument('--debug', action="store_true", help="enable development options")
args = parser.parse_args() args = parser.parse_args()
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING, logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s") format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
dbcon = get_engine() if not args.database:
parser.error("--database or DATABASE_URL required")
if not args.s3:
parser.error("--s3 or S3_URL required")
# set up database client
dbcon = sqlalchemy.create_engine(args.database, echo=args.debug, encoding="utf8")
SAEnginePlugin(cherrypy.engine, dbcon).subscribe() SAEnginePlugin(cherrypy.engine, dbcon).subscribe()
cherrypy.tools.db = SATool() cherrypy.tools.db = SATool()
s3 = boto3.client('s3', config=BotoConfig(signature_version='s3v4'), region_name='us-east-1', # set up s3 client
endpoint_url='', s3url = urlparse(args.s3)
aws_access_key_id='', s3args = {"config": BotoConfig(signature_version='s3v4')}
aws_secret_access_key='')
providers = {"apt": AptProvider(dbcon, s3), endpoint_url = f"{s3url.scheme}://{s3url.hostname}"
"pypi": PypiProvider(dbcon, s3)} if s3url.port:
endpoint_url += f":{s3url.port}"
s3args["endpoint_url"] = endpoint_url
if s3url.username and s3url.password:
s3args["aws_access_key_id"] = s3url.username
s3args["aws_secret_access_key"] = s3url.password
s3 = boto3.client('s3', **s3args)
bucket = s3url.path[1:]
# ensure bucket exists
if bucket not in [b['Name'] for b in s3.list_buckets()['Buckets']]:
print("Creating bucket")
s3.create_bucket(Bucket=bucket)
# set up providers
providers = {"apt": AptProvider(dbcon, s3, bucket),
"pypi": PypiProvider(dbcon, s3, bucket)}
# set up main web screen
web = AppWeb(providers) web = AppWeb(providers)
def validate_password(realm, username, password):
return True
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False, cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
'tools.db.on': True}}) 'tools.db.on': True}})

View File

@ -11,10 +11,6 @@ def db():
return cherrypy.request.db return cherrypy.request.db
def get_engine(echo=False):
return sqlalchemy.create_engine('mysql+pymysql://root:root@localhost/repobot', echo=echo, encoding="utf8")
class SAEnginePlugin(plugins.SimplePlugin): class SAEnginePlugin(plugins.SimplePlugin):
def __init__(self, bus, dbcon): def __init__(self, bus, dbcon):
plugins.SimplePlugin.__init__(self, bus) plugins.SimplePlugin.__init__(self, bus)
@ -60,8 +56,8 @@ class SATool(cherrypy.Tool):
def commit_transaction(self): def commit_transaction(self):
cherrypy.request.db = None cherrypy.request.db = None
try: try:
self.session.commit() #TODO commit is issued even on endpoints with no queries self.session.commit()
except: except Exception:
self.session.rollback() self.session.rollback()
raise raise
finally: finally: