309 lines
11 KiB
Python
309 lines
11 KiB
Python
import os
|
|
import sys
|
|
import logging
|
|
import argparse
|
|
import hashlib
|
|
import subprocess
|
|
import requests
|
|
from threading import Thread
|
|
from backupdb2.misc import load_cli_config, tabulate, tabulate_dict, has_binary, get_tarcmd, \
|
|
tar_scan_errors, WrappedStdout
|
|
from backupdb2.common import LOCKFILE
|
|
|
|
|
|
class BackupdbClient(object):
|
|
def __init__(self, base_url, passwd=None):
|
|
self.session = requests.Session()
|
|
if passwd:
|
|
self.session.auth = passwd # user, pass tuple
|
|
self.base_url = base_url
|
|
if not self.base_url.endswith("/"):
|
|
self.base_url += "/"
|
|
|
|
def get(self, url, **params):
|
|
return self.do("get", url, **params)
|
|
|
|
def post(self, url, **params):
|
|
return self.do("post", url, **params)
|
|
|
|
def put(self, url, **params):
|
|
return self.do("put", url, **params)
|
|
|
|
def delete(self, url, **params):
|
|
return self.do("delete", url, **params)
|
|
|
|
def do(self, method, url, **kwargs):
|
|
resp = getattr(self.session, method)(self.base_url + "api/v1/" + url, **kwargs)
|
|
resp.raise_for_status()
|
|
return resp
|
|
|
|
def list_namespaces(self):
|
|
return self.get("namespaces").json()
|
|
|
|
def list_backups(self, namespace="default"):
|
|
return self.get("backups", params=dict(namespace=namespace)).json()
|
|
|
|
def list_dates(self, backup, namespace="default"):
|
|
return self.get("dates", params=dict(namespace=namespace, backup=backup)).json()
|
|
|
|
def upload(self, stream, backup, namespace="default"):
|
|
hasher = WrappedStdout(stream)
|
|
return self.post("upload", data=hasher, params=dict(namespace=namespace, name=backup)), hasher.sha256()
|
|
|
|
def get_meta(self, backup, namespace="default", date=None):
|
|
return self.get("download", params=dict(namespace=namespace, name=backup, date=date, meta=True)).json()
|
|
|
|
def download(self, backup, namespace="default", date=None):
|
|
"""
|
|
Download a backup by date, or the latest if date is not supplied. Returns the request's Response object
|
|
"""
|
|
return self.get("download", stream=True, params=dict(namespace=namespace, name=backup, date=date))
|
|
|
|
|
|
def cmd_list_configured(args, parser, config, client):
|
|
"""
|
|
If there is a config file, list backups configured in it
|
|
"""
|
|
if not config["backups"]:
|
|
print("No backups configured")
|
|
return
|
|
|
|
tabulate([
|
|
[
|
|
name,
|
|
c["dir"],
|
|
c["method"],
|
|
] for name, c in config["backups"].items()],
|
|
headers=["name", "path", "method"]
|
|
)
|
|
|
|
|
|
def cmd_list_namespace(args, parser, config, client):
|
|
"""
|
|
List namespaces
|
|
"""
|
|
tabulate([
|
|
[i] for i in client.list_namespaces()],
|
|
headers=["namespaces"]
|
|
)
|
|
|
|
|
|
def cmd_list_backup(args, parser, config, client):
|
|
"""
|
|
List backups in a namespace
|
|
"""
|
|
tabulate([
|
|
[i] for i in client.list_backups(namespace=args.namespace)],
|
|
headers=["backups"]
|
|
)
|
|
|
|
|
|
def cmd_list_dates(args, parser, config, client):
|
|
"""
|
|
List available dates for a backup
|
|
"""
|
|
tabulate([
|
|
[i] for i in client.list_dates(backup=args.backup, namespace=args.namespace)],
|
|
headers=[args.backup]
|
|
)
|
|
|
|
|
|
def cmd_download(args, parser, config, client):
|
|
"""
|
|
Download a backup
|
|
"""
|
|
pass
|
|
|
|
|
|
def cmd_backup(args, parser, config, client):
|
|
"""
|
|
Create a new backup - requires it be defined in the config file
|
|
"""
|
|
backup_config = config["backups"][args.backup]
|
|
|
|
# Refuse to backup if a lockfile created by this cli is not present
|
|
if not os.path.exists(os.path.join(backup_config["dir"], LOCKFILE)) and not args.force:
|
|
print("Error: data is missing (Use --force?)")
|
|
return 1
|
|
|
|
# stream tar/gz to backup server
|
|
|
|
args_tar = []
|
|
|
|
if has_binary("ionice"):
|
|
args_tar += ['ionice', '-c', '3']
|
|
|
|
args_tar += ['nice', '-n', '19']
|
|
args_tar += [get_tarcmd(),
|
|
'--sort=name',
|
|
f'--exclude={LOCKFILE}',
|
|
'--warning=no-file-changed',
|
|
'--warning=no-file-removed',
|
|
'--warning=no-file-ignored',
|
|
'--warning=no-file-shrank']
|
|
|
|
# Use pigz if available (Parallel gzip - http://zlib.net/pigz/)
|
|
if has_binary("pigz"):
|
|
args_tar += ["--use-compress-program", "pigz -n"]
|
|
else:
|
|
args_tar += ["--use-compress-program", "gzip -n"] # pass -n to gzip to produce reproducible archives
|
|
|
|
# Excluded paths
|
|
if backup_config["exclude"]:
|
|
for exclude_path in backup_config["exclude"].split(","):
|
|
if exclude_path:
|
|
args_tar.append("--exclude")
|
|
args_tar.append(exclude_path)
|
|
|
|
args_tar += ['-cv', './']
|
|
tar_dir = os.path.normpath(backup_config["dir"]) + '/'
|
|
tar = subprocess.Popen(args_tar, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=tar_dir)
|
|
|
|
tar_errors = []
|
|
error_scanner = Thread(target=tar_scan_errors, args=(tar.stderr, tar_errors), daemon=True)
|
|
error_scanner.start()
|
|
|
|
response, local_sha = client.upload(tar.stdout, args.backup, args.namespace)
|
|
logging.info("local sha256: %s", local_sha)
|
|
|
|
tar.wait()
|
|
error_scanner.join()
|
|
|
|
if response.status_code != 200:
|
|
print("Error: upload failed with code: {}".format(response.status_code))
|
|
print(response.text)
|
|
return 1
|
|
|
|
if tar.returncode != 0 and tar_errors > 0:
|
|
print("Error: tar process exited with nonzero code: {}.".format(tar.returncode))
|
|
print("Tar errors: \n {}".format("\n ".join(tar_errors)))
|
|
return 1
|
|
#TODO call delete api if we detected tar errors
|
|
#TODO calculate sha256 on the fly and verify in response
|
|
#TODO also call delete api on ctrl-c too early
|
|
|
|
print("\nbackup complete!\n")
|
|
tabulate_dict(response.json())
|
|
|
|
|
|
def cmd_restore(args, parser, config, client):
|
|
"""
|
|
Restore a backup
|
|
"""
|
|
backup_config = config["backups"][args.backup]
|
|
|
|
dest = os.path.normpath(args.output_dir or backup_config["dir"])
|
|
os.makedirs(dest, exist_ok=True)
|
|
original_perms = os.stat(dest)
|
|
|
|
# Refuse to restore if a lockfile created by this cli is already present
|
|
if os.path.exists(os.path.join(backup_config["dir"], LOCKFILE)) and not args.force:
|
|
print("Error: data is missing (Use --force?)")
|
|
return 1
|
|
|
|
meta = client.get_meta(args.backup, args.namespace, args.date)
|
|
#TODO could catch 404 here and clean exit like the old tool used to do
|
|
|
|
response = client.download(args.backup, args.namespace, args.date)
|
|
|
|
# r.raise_for_status()
|
|
# with open(local_filename, 'wb') as f:
|
|
# for chunk in r.iter_content(chunk_size=8192):
|
|
|
|
args_tar = [get_tarcmd(), 'zxv', '-C', dest + '/']
|
|
|
|
print("Tar restore call: {}".format(args_tar))
|
|
|
|
extract = subprocess.Popen(args_tar, stdin=subprocess.PIPE)
|
|
|
|
hasher = hashlib.sha256()
|
|
for chunk in response.iter_content(WrappedStdout.BUFFSIZE):
|
|
extract.stdin.write(chunk)
|
|
hasher.update(chunk)
|
|
#TODO also sha256 it
|
|
extract.stdin.close()
|
|
|
|
extract.wait()
|
|
# TODO: convert to pure python?
|
|
|
|
sha256 = hasher.hexdigest()
|
|
if sha256 != meta["sha256"]:
|
|
raise Exception("Downloaded archive (%s) didn't match expected hash (%s)", sha256, meta["sha256"])
|
|
if extract.returncode != 0:
|
|
raise Exception("Could not extract archive")
|
|
|
|
# Restore original permissions on data dir
|
|
if not args.output_dir:
|
|
os.chmod(dest, original_perms.st_mode)
|
|
os.chown(dest, original_perms.st_uid, original_perms.st_gid)
|
|
|
|
|
|
def get_args():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("-c", "--config", help="config file path")
|
|
parser.add_argument("-s", "--server", help="server address")
|
|
|
|
sp_action = parser.add_subparsers(dest="action", required=True, help="action to take")
|
|
|
|
p_list = sp_action.add_parser("list", help="list action")
|
|
sp_list = p_list.add_subparsers(dest="list_action", required=True, help="item to list")
|
|
|
|
p_list_configured = sp_list.add_parser("configured", help="list locally configured backups")
|
|
p_list_configured.set_defaults(func=cmd_list_configured)
|
|
|
|
p_list_namespace = sp_list.add_parser("namespace", help="list namespaces")
|
|
p_list_namespace.set_defaults(func=cmd_list_namespace)
|
|
|
|
p_list_backup = sp_list.add_parser("backup", help="list backups")
|
|
p_list_backup.set_defaults(func=cmd_list_backup)
|
|
p_list_backup.add_argument("-n", "--namespace", default="default", help="parent namespace to list in")
|
|
|
|
p_list_dates = sp_list.add_parser("dates", help="list dates")
|
|
p_list_dates.set_defaults(func=cmd_list_dates)
|
|
p_list_dates.add_argument("-n", "--namespace", default="default", help="parent namespace to list in")
|
|
p_list_dates.add_argument("backup", help="backup to list dates for")
|
|
|
|
p_backup = sp_action.add_parser("backup", help="backup action")
|
|
p_backup.set_defaults(func=cmd_backup)
|
|
p_backup.add_argument("--force", action="store_true", help="force backup operation if lockfile not found")
|
|
p_backup.add_argument("-n", "--namespace", default="default", help="parent namespace backup to")
|
|
p_backup.add_argument("backup", help="backup to make")
|
|
|
|
p_download = sp_action.add_parser("download", help="download action")
|
|
p_download.set_defaults(func=cmd_download)
|
|
p_download.add_argument("-n", "--namespace", default="default", help="parent namespace to download from")
|
|
p_download.add_argument("backup", help="backup to download")
|
|
p_download.add_argument("date", help="date of backup to download")
|
|
|
|
p_restore = sp_action.add_parser("restore", help="restore action")
|
|
p_restore.set_defaults(func=cmd_restore)
|
|
p_restore.add_argument("--force", help="force restore operation if destination data already exists",
|
|
action="store_true", )
|
|
p_restore.add_argument("-n", "--namespace", default="default", help="parent namespace download from")
|
|
p_restore.add_argument("-o", "--output-dir", help="override restore path")
|
|
p_restore.add_argument("backup", help="backup to download")
|
|
p_restore.add_argument("date", nargs="?", help="date of backup to download")
|
|
|
|
return parser.parse_args(), parser
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(level=logging.INFO)
|
|
args, parser = get_args()
|
|
|
|
logging.basicConfig(
|
|
# level=logging.DEBUG if args.debug else logging.INFO,
|
|
format="%(asctime)-15s %(levelname)-8s %(filename)s:%(lineno)d %(message)s"
|
|
)
|
|
logging.getLogger("botocore").setLevel(logging.ERROR)
|
|
logging.getLogger("urllib3").setLevel(logging.ERROR)
|
|
|
|
config = load_cli_config(args.config)
|
|
client = BackupdbClient(args.server or config["options"].get("server"))
|
|
|
|
sys.exit(args.func(args, parser, config, client) or 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|