211 lines
7.1 KiB
Python
211 lines
7.1 KiB
Python
import json
|
||
import argparse
|
||
import cherrypy
|
||
import logging
|
||
import signal
|
||
from datetime import datetime
|
||
from urllib.parse import urlparse
|
||
|
||
from backupdb2.upload import stream_to_s3, S3UploadError
|
||
from backupdb2.boto import get_s3
|
||
from backupdb2.common import BackupManager
|
||
|
||
|
||
class WebBase(object):
|
||
"""
|
||
Base class for web components that provides common tools
|
||
"""
|
||
def __init__(self, bucket, s3conn):
|
||
self.bucket = bucket
|
||
self.s3 = s3conn
|
||
self.mgr = BackupManager(bucket, s3conn)
|
||
|
||
|
||
class BackupdbHttp(WebBase):
|
||
"""
|
||
Absolute bare minimum html browser web interface
|
||
"""
|
||
def __init__(self, bucket, s3conn):
|
||
super().__init__(bucket, s3conn)
|
||
self.api = BackupdbApiV1(bucket, s3conn)
|
||
|
||
@cherrypy.expose
|
||
def index(self):
|
||
yield "<h1>Namespaces</h1><hr>"
|
||
for ns in self.mgr.list_namespaces():
|
||
yield f'<a href="backups?namespace={ns}">{ns}</a><br />'# TODO lol injection
|
||
|
||
@cherrypy.expose
|
||
def backups(self, namespace="default"):
|
||
yield f'<h1>Backups for namespace: <em>{namespace}</em></h1><hr>'# TODO lol injection
|
||
for backup in self.mgr.list_backups(namespace=namespace):
|
||
yield f'<a href="dates?namespace={namespace}&name={backup}">{backup}</a><br />'
|
||
|
||
@cherrypy.expose
|
||
def dates(self, name, namespace="default"):
|
||
yield f'<h1>Dates for backup: <em>{name}</em> in namespace: <em>{namespace}</em></h1><hr>'# TODO lol injection
|
||
for date in self.mgr.list_dates(name, namespace=namespace):
|
||
yield f'<a href="api/v1/download?namespace={namespace}&name={name}&date={date}">{date}</a> (<a href="api/v1/download?namespace={namespace}&name={name}&date={date}&meta=1">meta</a>)<br />'
|
||
|
||
|
||
class BackupdbApiV1(WebBase):
|
||
"""
|
||
V1 json api
|
||
|
||
/api/v1/namespaces -> list of namespaces -> ["default"]
|
||
/api/v1/backups -> list of backup names -> ["testbackup"]
|
||
/api/v1/dates -> list of backup dates ?backup=testbackup -> ["2021-05-27T20:41:32.833886"]
|
||
/api/v1/download -> stream of tar.gz data ?backup=testbackup&date=<date> -> (data)
|
||
/api/v1/download -> json metadata above + &meta=1
|
||
|
||
Param `namespace` is optional on all endpoints except /namespaces
|
||
"""
|
||
def __init__(self, bucket, s3conn):
|
||
super().__init__(bucket, s3conn)
|
||
self.v1 = self
|
||
|
||
@cherrypy.expose
|
||
def index(self, name=None, namespace="default"):
|
||
yield '<a href="namespaces">namespaces</a><br />'
|
||
yield '<a href="backups">backups</a><br />'
|
||
yield '<a href="dates">dates</a><br />'
|
||
yield '<a href="download">download</a><br />'
|
||
|
||
@cherrypy.expose
|
||
@cherrypy.tools.json_out()
|
||
def namespaces(self):
|
||
return self.mgr.list_namespaces()
|
||
|
||
@cherrypy.expose
|
||
@cherrypy.tools.json_out()
|
||
def backups(self, namespace="default"):
|
||
return self.mgr.list_backups(namespace)
|
||
|
||
@cherrypy.expose
|
||
@cherrypy.tools.json_out()
|
||
def dates(self, backup, namespace="default"):
|
||
return self.mgr.list_dates(backup, namespace=namespace)
|
||
|
||
@cherrypy.expose
|
||
def download(self, name, date=None, meta=False, namespace="default"):
|
||
if not date:
|
||
date = sorted(self.mgr.list_dates(name, namespace))[-1]
|
||
|
||
metadata = self.mgr.get_metadata(name, date, namespace=namespace)
|
||
if meta:
|
||
cherrypy.response.headers["Content-Type"] = "application/json"
|
||
else:
|
||
cherrypy.response.headers["Content-Type"] = "application/gzip"
|
||
cherrypy.response.headers["Content-Length"] = metadata["size"]
|
||
cherrypy.response.headers["Content-Disposition"] = f'filename="{namespace}-{name}-{date}.tar.gz"'#TODO lol injection
|
||
|
||
cherrypy.response.headers["X-Backupdb-Date"] = date
|
||
|
||
def download():
|
||
if meta:
|
||
yield json.dumps(metadata).encode('utf-8')
|
||
else:
|
||
yield from self.mgr.get_stream(name, date, namespace=namespace)
|
||
|
||
return download()
|
||
download._cp_config = {'response.stream': True}
|
||
|
||
@cherrypy.expose
|
||
@cherrypy.tools.json_out()
|
||
def upload(self, name, namespace="default"):
|
||
#TODO validate name & namespace
|
||
# cherrypy.response.timeout = 3600
|
||
|
||
now = datetime.now()
|
||
|
||
try:
|
||
metadata = stream_to_s3(
|
||
cherrypy.request.body,
|
||
self.s3,
|
||
self.bucket,
|
||
f"{namespace}/{name}/backups/{now.isoformat()}/backup.tar.gz.{{sequence:08d}}"
|
||
)
|
||
except S3UploadError as ue:
|
||
cherrypy.response.status = 500
|
||
logging.error(f"uploader failed: {ue.errors}")
|
||
return {"errors": ue.errors}
|
||
|
||
metadata["date"] = now.isoformat()
|
||
|
||
logging.debug("upload complete, writing metadata")
|
||
meta_response = self.s3.put_object(
|
||
Bucket=self.bucket,
|
||
Key=f"{namespace}/{name}/backups/{now.isoformat()}/meta.json",
|
||
Body=json.dumps(metadata, indent=4, sort_keys=True)
|
||
)
|
||
if meta_response["ResponseMetadata"]["HTTPStatusCode"] != 200:
|
||
cherrypy.response.status = 500
|
||
return {"errors": "backend upload failed: " + str(meta_response["ResponseMetadata"])}
|
||
|
||
logging.debug("upload success")
|
||
return metadata
|
||
|
||
@cherrypy.expose
|
||
@cherrypy.tools.json_out()
|
||
def get_latest(self, name, namespace="default"):
|
||
pass
|
||
|
||
|
||
def run_http(args):
|
||
s3url = urlparse(args.s3_url)
|
||
bucket = s3url.path[1:]
|
||
s3 = get_s3(s3url)
|
||
|
||
# ensure bucket exists
|
||
if bucket not in [b['Name'] for b in s3.list_buckets()['Buckets']]:
|
||
logging.warning("Creating bucket")
|
||
s3.create_bucket(Bucket=bucket)
|
||
|
||
web = BackupdbHttp(bucket, s3)
|
||
|
||
cherrypy.tree.mount(web, '/', {'/': {}})
|
||
|
||
# General config options
|
||
cherrypy.config.update({
|
||
'request.show_tracebacks': True,
|
||
'server.thread_pool': 5,
|
||
'server.socket_host': "0.0.0.0",
|
||
'server.socket_port': args.port,
|
||
'server.show_tracebacks': True,
|
||
'log.screen': False,
|
||
'engine.autoreload.on': args.debug,
|
||
'server.max_request_body_size': 0,
|
||
})
|
||
|
||
def signal_handler(signum, stack):
|
||
logging.critical('Got sig {}, exiting...'.format(signum))
|
||
cherrypy.engine.exit()
|
||
|
||
signal.signal(signal.SIGINT, signal_handler)
|
||
signal.signal(signal.SIGTERM, signal_handler)
|
||
|
||
try:
|
||
cherrypy.engine.start()
|
||
cherrypy.engine.block()
|
||
finally:
|
||
cherrypy.engine.exit()
|
||
|
||
|
||
def get_args():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("-p", "--port", default=8080, type=int, help="listen port for http server")
|
||
p.add_argument("-s", "--s3-url", required=True, help="minio server address")
|
||
p.add_argument("--debug", action="store_true", help="debug mode")
|
||
return p.parse_args()
|
||
|
||
|
||
def main():
|
||
args = get_args()
|
||
logging.basicConfig(
|
||
level=logging.DEBUG if args.debug else logging.INFO,
|
||
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s"
|
||
)
|
||
logging.getLogger("botocore").setLevel(logging.ERROR)
|
||
logging.getLogger("urllib3").setLevel(logging.ERROR)
|
||
run_http(args)
|