un-hardcode s3 and db connections
This commit is contained in:
parent
64f51bdf7c
commit
958805a587
|
@ -14,7 +14,7 @@ from sqlalchemy.orm import relationship
|
||||||
from sqlalchemy.types import String, Integer, Text, BOOLEAN
|
from sqlalchemy.types import String, Integer, Text, BOOLEAN
|
||||||
from tempfile import TemporaryDirectory
|
from tempfile import TemporaryDirectory
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
from repobot.tables import Base, db, get_engine
|
from repobot.tables import Base, db
|
||||||
|
|
||||||
|
|
||||||
class AptRepo(Base):
|
class AptRepo(Base):
|
||||||
|
@ -143,7 +143,7 @@ def hashmany(data):
|
||||||
|
|
||||||
|
|
||||||
class AptProvider(object):
|
class AptProvider(object):
|
||||||
def __init__(self, dbcon, s3client, bucket="aptprovider"):
|
def __init__(self, dbcon, s3client, bucket):
|
||||||
self.db = dbcon
|
self.db = dbcon
|
||||||
self.s3 = s3client
|
self.s3 = s3client
|
||||||
self.bucket = bucket
|
self.bucket = bucket
|
||||||
|
@ -155,17 +155,12 @@ class AptProvider(object):
|
||||||
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
|
cherrypy.tree.mount(AptWeb(self), "/repo/apt", {'/': {'tools.trailing_slash.on': False,
|
||||||
'tools.db.on': True}})
|
'tools.db.on': True}})
|
||||||
|
|
||||||
# ensure bucket exists
|
|
||||||
if bucket not in [b['Name'] for b in self.s3.list_buckets()['Buckets']]:
|
|
||||||
print("Creating bucket")
|
|
||||||
self.s3.create_bucket(Bucket=bucket)
|
|
||||||
|
|
||||||
self.updater = Thread(target=self.sign_packages, daemon=True)
|
self.updater = Thread(target=self.sign_packages, daemon=True)
|
||||||
self.updater.start()
|
self.updater.start()
|
||||||
|
|
||||||
def sign_packages(self):
|
def sign_packages(self):
|
||||||
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
|
Session = sqlalchemy.orm.sessionmaker(autoflush=True, autocommit=False)
|
||||||
Session.configure(bind=get_engine())
|
Session.configure(bind=self.db)
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
work = self.queue.get(block=True, timeout=5)
|
work = self.queue.get(block=True, timeout=5)
|
||||||
|
|
|
@ -170,7 +170,7 @@ def copysha256(fin, fout):
|
||||||
|
|
||||||
|
|
||||||
class PypiProvider(object):
|
class PypiProvider(object):
|
||||||
def __init__(self, dbcon, s3client, bucket="aptprovider"):
|
def __init__(self, dbcon, s3client, bucket):
|
||||||
self.db = dbcon
|
self.db = dbcon
|
||||||
self.s3 = s3client
|
self.s3 = s3client
|
||||||
self.bucket = bucket
|
self.bucket = bucket
|
||||||
|
@ -180,12 +180,6 @@ class PypiProvider(object):
|
||||||
cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
|
cherrypy.tree.mount(PipWeb(self), "/repo/pypi", {'/': {'tools.trailing_slash.on': False,
|
||||||
'tools.db.on': True}})
|
'tools.db.on': True}})
|
||||||
|
|
||||||
# ensure bucket exists
|
|
||||||
#TODO bucket creation should happen in server.py
|
|
||||||
if bucket not in [b['Name'] for b in self.s3.list_buckets()['Buckets']]:
|
|
||||||
print("Creating bucket")
|
|
||||||
self.s3.create_bucket(Bucket=bucket)
|
|
||||||
|
|
||||||
def web_addpkg(self, reponame, name, version, fobj):
|
def web_addpkg(self, reponame, name, version, fobj):
|
||||||
repo = get_repo(db(), reponame)
|
repo = get_repo(db(), reponame)
|
||||||
|
|
||||||
|
|
|
@ -1,12 +1,13 @@
|
||||||
|
import boto3
|
||||||
import cherrypy
|
import cherrypy
|
||||||
import logging
|
import logging
|
||||||
from repobot.tables import get_engine, SAEnginePlugin, SATool
|
import os
|
||||||
|
import sqlalchemy
|
||||||
|
from botocore.client import Config as BotoConfig
|
||||||
from repobot.aptprovider import AptProvider
|
from repobot.aptprovider import AptProvider
|
||||||
from repobot.pypiprovider import PypiProvider
|
from repobot.pypiprovider import PypiProvider
|
||||||
|
from repobot.tables import SAEnginePlugin, SATool
|
||||||
import boto3
|
from urllib.parse import urlparse
|
||||||
from botocore.client import Config as BotoConfig
|
|
||||||
|
|
||||||
|
|
||||||
class AppWeb(object):
|
class AppWeb(object):
|
||||||
|
@ -32,33 +33,56 @@ def main():
|
||||||
import argparse
|
import argparse
|
||||||
import signal
|
import signal
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="irc web client server")
|
parser = argparse.ArgumentParser(description="package storage database")
|
||||||
parser.add_argument('-p', '--port', default=8080, type=int, help="tcp port to listen on")
|
parser.add_argument('-p', '--port', default=8080, type=int, help="http port to listen on")
|
||||||
parser.add_argument('-s', '--database', help="mysql connection string")
|
parser.add_argument('-d', '--database', help="mysql+pymysql:// connection string",
|
||||||
|
default=os.environ.get("DATABASE_URL"))
|
||||||
|
parser.add_argument('-s', '--s3', help="http:// or https:// connection string",
|
||||||
|
default=os.environ.get("S3_URL"))
|
||||||
parser.add_argument('--debug', action="store_true", help="enable development options")
|
parser.add_argument('--debug', action="store_true", help="enable development options")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
|
logging.basicConfig(level=logging.INFO if args.debug else logging.WARNING,
|
||||||
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
|
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s")
|
||||||
|
|
||||||
dbcon = get_engine()
|
if not args.database:
|
||||||
|
parser.error("--database or DATABASE_URL required")
|
||||||
|
if not args.s3:
|
||||||
|
parser.error("--s3 or S3_URL required")
|
||||||
|
|
||||||
|
# set up database client
|
||||||
|
dbcon = sqlalchemy.create_engine(args.database, echo=args.debug, encoding="utf8")
|
||||||
SAEnginePlugin(cherrypy.engine, dbcon).subscribe()
|
SAEnginePlugin(cherrypy.engine, dbcon).subscribe()
|
||||||
cherrypy.tools.db = SATool()
|
cherrypy.tools.db = SATool()
|
||||||
|
|
||||||
s3 = boto3.client('s3', config=BotoConfig(signature_version='s3v4'), region_name='us-east-1',
|
# set up s3 client
|
||||||
endpoint_url='',
|
s3url = urlparse(args.s3)
|
||||||
aws_access_key_id='',
|
s3args = {"config": BotoConfig(signature_version='s3v4')}
|
||||||
aws_secret_access_key='')
|
|
||||||
|
|
||||||
providers = {"apt": AptProvider(dbcon, s3),
|
endpoint_url = f"{s3url.scheme}://{s3url.hostname}"
|
||||||
"pypi": PypiProvider(dbcon, s3)}
|
if s3url.port:
|
||||||
|
endpoint_url += f":{s3url.port}"
|
||||||
|
s3args["endpoint_url"] = endpoint_url
|
||||||
|
|
||||||
|
if s3url.username and s3url.password:
|
||||||
|
s3args["aws_access_key_id"] = s3url.username
|
||||||
|
s3args["aws_secret_access_key"] = s3url.password
|
||||||
|
|
||||||
|
s3 = boto3.client('s3', **s3args)
|
||||||
|
bucket = s3url.path[1:]
|
||||||
|
|
||||||
|
# ensure bucket exists
|
||||||
|
if bucket not in [b['Name'] for b in s3.list_buckets()['Buckets']]:
|
||||||
|
print("Creating bucket")
|
||||||
|
s3.create_bucket(Bucket=bucket)
|
||||||
|
|
||||||
|
# set up providers
|
||||||
|
providers = {"apt": AptProvider(dbcon, s3, bucket),
|
||||||
|
"pypi": PypiProvider(dbcon, s3, bucket)}
|
||||||
|
|
||||||
|
# set up main web screen
|
||||||
web = AppWeb(providers)
|
web = AppWeb(providers)
|
||||||
|
|
||||||
def validate_password(realm, username, password):
|
|
||||||
return True
|
|
||||||
|
|
||||||
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
|
cherrypy.tree.mount(web, '/', {'/': {'tools.trailing_slash.on': False,
|
||||||
'tools.db.on': True}})
|
'tools.db.on': True}})
|
||||||
|
|
||||||
|
|
|
@ -11,10 +11,6 @@ def db():
|
||||||
return cherrypy.request.db
|
return cherrypy.request.db
|
||||||
|
|
||||||
|
|
||||||
def get_engine(echo=False):
|
|
||||||
return sqlalchemy.create_engine('mysql+pymysql://root:root@localhost/repobot', echo=echo, encoding="utf8")
|
|
||||||
|
|
||||||
|
|
||||||
class SAEnginePlugin(plugins.SimplePlugin):
|
class SAEnginePlugin(plugins.SimplePlugin):
|
||||||
def __init__(self, bus, dbcon):
|
def __init__(self, bus, dbcon):
|
||||||
plugins.SimplePlugin.__init__(self, bus)
|
plugins.SimplePlugin.__init__(self, bus)
|
||||||
|
@ -60,8 +56,8 @@ class SATool(cherrypy.Tool):
|
||||||
def commit_transaction(self):
|
def commit_transaction(self):
|
||||||
cherrypy.request.db = None
|
cherrypy.request.db = None
|
||||||
try:
|
try:
|
||||||
self.session.commit() #TODO commit is issued even on endpoints with no queries
|
self.session.commit()
|
||||||
except:
|
except Exception:
|
||||||
self.session.rollback()
|
self.session.rollback()
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
|
|
Loading…
Reference in New Issue