diff --git a/resticbackup/__init__.py b/resticbackup/__init__.py index e69de29..eac601c 100644 --- a/resticbackup/__init__.py +++ b/resticbackup/__init__.py @@ -0,0 +1,4 @@ +import os + +CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d") +RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic") diff --git a/resticbackup/cli.py b/resticbackup/cli.py index 3c2eff0..c825c95 100644 --- a/resticbackup/cli.py +++ b/resticbackup/cli.py @@ -1,156 +1,10 @@ import os -import sys -import json import argparse import subprocess from socket import getfqdn -from urllib.parse import urlparse -from dataclasses import dataclass, field -from typing import List - -CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d") -RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic") - - -def die(msg, rc=1): - print(msg) - sys.exit(rc) - - -def pdie(popen): - popen.wait() - sys.exit(popen.returncode) - - -def checkp(popen): - popen.wait() - if popen.returncode != 0: - die(rc=popen.returncode) - return popen - - -def list_configs(): - config_names = [] - for fname in os.listdir(CFG_DIR): - name = fname.split(".")[0] - if name == "main": - continue - config_names.append(name) - config_names.sort() - return config_names - - -def load_base_config(): - main_path = os.path.join(CFG_DIR, "main.json") - with open(main_path) as f: - return json.load(f) - - -def load_config(name): - # load configs from /etc/resticbackup.d/ - # we have special handling for /etc/resticbackup.d/main.json - # return (main_config, dict(config_name=>config)) - cfg_path = os.path.join(CFG_DIR, "{}.json".format(name)) - - with open(cfg_path) as f: - backup_config = json.load(f) - - return ClientConfig.load(load_base_config(), backup_config) - - -@dataclass -class BackupConfig: - path: str - repo: str - schedule: dict = field(default_factory=dict) - exclude: List[str] = field(default_factory=list) - backup_preexec: List[str] = field(default_factory=list) - backup_postexec: List[str] = field(default_factory=list) - restore_preexec: List[str] = field(default_factory=list) - restore_postexec: List[str] = field(default_factory=list) - - @staticmethod - def load(data: dict) -> 'BackupConfig': - return BackupConfig( - path=data['path'], - repo=data['repo'], - schedule=data.get('schedule', {}), - exclude=data.get('exclude', []), - backup_preexec=data.get('backup_preexec', []), - backup_postexec=data.get('backup_postexec', []), - restore_preexec=data.get('restore_preexec', []), - restore_postexec=data.get('restore_postexec', []), - ) - - -@dataclass -class ClientConfig: - server_type: str - uri: str - secret: str - backup: BackupConfig - - @property - def repo(self): # port ignored - return "{}:{}://{}/{}".format(self.server_type, self.uri.scheme, self.uri.hostname, self.backup.repo) - - @property - def env(self): - return { - "AWS_ACCESS_KEY_ID": self.uri.username, - "AWS_SECRET_ACCESS_KEY": self.uri.password, - "RESTIC_PASSWORD": self.secret, - "RESTIC_REPOSITORY": self.repo, - } - - @staticmethod - def load(main, backup) -> "ClientConfig": - backup = BackupConfig.load(backup) - server_type, server = main["server"].split(":", 1) - if server_type != "s3": - raise Exception("unsupported server type: {}".format(server_type)) - return ClientConfig(server_type=server_type, uri=urlparse(server), secret=main["secret"], backup=backup) - - def run(self, args, **kwargs): - env = dict(os.environ) - env.update(**self.env) - return subprocess.Popen([RESTIC_BIN] + args, env=env, **kwargs) - - -class ExecWrapper(object): - def __init__(self, pre=None, post=None): - self.pre = pre or [] - self.post = post or [] - - def __enter__(self): - for command in self.pre: - print("+", command) - subprocess.check_call(command, shell=True) - - def __exit__(self, exc_type, exc_value, exc_tb): - for command in self.post: - print("+", command) - subprocess.check_call(command, shell=True) - - -def init_ok(message): - """ - Restic doesn't have a way to check if a repo is initialized or otherwise cleanly initialize it. So, we try to - initialize it and read the error message. Either a sucess message or an error suggesting it is already initialized - will cause this function to return true. - - Success message: - created restic repository xxxxx at xxxxx\n\n - - Error message: - Fatal: create key in repository at xxxx failed: repository master key and config already initialized\n\n' - """ - message = message.strip() - - return message.startswith("created restic repository ") or ( - message.startswith("Fatal: create key in repository at ") and - message.endswith("repository master key and config already initialized")) +from resticbackup.types import ClientConfig, load_config, load_base_config, list_configs +from resticbackup.lib import ExecWrapper, init_ok, die, pdie, checkp, get_retention_args, get_newest_snapshot def cmd_backup(args, parser): @@ -205,48 +59,42 @@ def cmd_backup(args, parser): return pdie(config.run(["prune"])) -def get_retention_args(schedule): - """ - given a retention schedule, return restic command(s) needed to make it so - """ - mode = schedule.get("function") - if mode is None: # default is to just keep stuff lol - return None - if mode == "forever": - # do not perform deletions - return None - elif mode == "keep": - # just keep the last X snapshots - return ["--keep-last", str(mode["count"])] - elif mode == "cycle": - # keep the last $last backups. - # keep a daily backup after that for the past $daily days. - # keep a weekly backup after that for the past $weekly weeks. - # keep a monthly backup after that for the past $monthly months. - cmd = [] - last = schedule.get("last") - if last is not None: - cmd.extend(["--keep-last", str(last)]) - - daily = schedule.get("daily") - if daily is not None: - cmd.extend(["--keep-daily", str(daily)]) - - weekly = schedule.get("weekly") - if weekly is not None: - cmd.extend(["--keep-weekly", str(weekly)]) - - monthly = schedule.get("monthly") - if monthly is not None: - cmd.extend(["--keep-monthly", str(monthly)]) - - return cmd or None - else: - raise Exception("unknown retention function {}".format(mode)) - - def cmd_restore(args, parser): - pass # TODO restore backups + config = load_config(args.name) + + try: + if os.listdir(config.backup.path): + return die("output dir has files present - refusing to restore") + except FileNotFoundError: + return die("output dir '{}' does not exist".format(config.backup.path)) + + snapshot = get_newest_snapshot(args.name, config) + + if not snapshot: + return die("no snapshots found") + + os.chdir(config.backup.path) + with ExecWrapper(post=config.backup.restore_postexec): + with ExecWrapper(pre=config.backup.restore_preexec): + return pdie(config.run(["restore", snapshot["id"], "-t", "./"])) + + +def cmd_retrieve(args, parser): + # this is the same as cmd_restore except for how the config is formed and no execs + config = ClientConfig.load(load_base_config(), {"repo": args.repo, "path": args.outdir}) + + try: + if os.listdir(config.backup.path): + return die("output dir has files present - refusing to restore") + except FileNotFoundError: + return die("output dir does not exist") + + snapshot = get_newest_snapshot(args.name, config) + + if not snapshot: + return die("no snapshots found") + + return pdie(config.run(["restore", snapshot["id"], "-t", args.outdir])) def cmd_list(args, parser): @@ -267,7 +115,6 @@ def cmd_list(args, parser): print(entry) - def cmd_exec(args, parser): checkp(load_config(args.name).run(args.args)) @@ -275,16 +122,24 @@ def cmd_exec(args, parser): def main(): parser = argparse.ArgumentParser(description="restic wrapper") - parser.add_argument('-n', '--no-exec', action='store_true', help='do not run pre/post-exec commands') - parser.add_argument('-b', '--no-pre-exec', action='store_true', help='do not run pre-exec commands') - parser.add_argument('-m', '--no-post-exec', action='store_true', help='do not run post-exec commands') + # parser.add_argument('-n', '--no-exec', action='store_true', help='do not run pre/post-exec commands') # TODO + # parser.add_argument('-b', '--no-pre-exec', action='store_true', help='do not run pre-exec commands') # TODO + # parser.add_argument('-m', '--no-post-exec', action='store_true', help='do not run post-exec commands') # TODO sp_action = parser.add_subparsers(dest="action", help="action to take") p_backup = sp_action.add_parser("backup", help="take a backup") p_backup.set_defaults(func=cmd_backup) p_backup.add_argument("name", help="name of backup to execute") - # p_backup.add_argument("-p", "--print", action="store_true", help="just print the restic commands instead") # TODO + # p_backup.add_argument("-c", "--print", action="store_true", help="just print the restic commands instead") # TODO + # p_backup.add_argument("-e", "--env", action="store_true", help="just print the environment variables instead") # TODO + + p_restore = sp_action.add_parser("restore", help="restore a backup") + p_restore.set_defaults(func=cmd_restore) + p_restore.add_argument("name", help="name of backup to restore") + # p_restore.add_argument("snapshot", nargs="?", help="id of snapshot to restore") # TODO + p_restore.add_argument('--no-snapshots-ok', action='store_true', + help='don\'t return an error if no snapshots are available to be restored') p_exec = sp_action.add_parser("exec", help="execute a restic command") p_exec.set_defaults(func=cmd_exec) @@ -292,11 +147,16 @@ def main(): p_exec.add_argument("args", nargs="+", help="command arguments") p_list = sp_action.add_parser("list", help="list configs or snapshots") - p_list.add_argument("name", nargs="?", help="name of config to list snapshots for") p_list.set_defaults(func=cmd_list) + p_list.add_argument("name", nargs="?", help="name of config to list snapshots for") + + p_retrieve = sp_action.add_parser("retrieve", help="download arbitrary snapshots") + p_retrieve.set_defaults(func=cmd_retrieve) + p_retrieve.add_argument("repo", help="repo path") + p_retrieve.add_argument("name", help="backup name") + p_retrieve.add_argument("outdir", help="file path to write to") args = parser.parse_args() - args.func(args, parser) diff --git a/resticbackup/lib.py b/resticbackup/lib.py new file mode 100644 index 0000000..84eccbd --- /dev/null +++ b/resticbackup/lib.py @@ -0,0 +1,125 @@ +import sys +import json +import subprocess + + +def die(msg, rc=1): + print(msg) + sys.exit(rc) + + +def pdie(popen): + popen.wait() + sys.exit(popen.returncode) + + +def checkp(popen): + popen.wait() + if popen.returncode != 0: + die(rc=popen.returncode) + return popen + + +class ExecWrapper(object): + def __init__(self, pre=None, post=None): + self.pre = pre or [] + self.post = post or [] + + def __enter__(self): + for command in self.pre: + print("+", command) + subprocess.check_call(command, shell=True) + + def __exit__(self, exc_type, exc_value, exc_tb): + for command in self.post: + print("+", command) + subprocess.check_call(command, shell=True) + + +def init_ok(message): + """ + Restic doesn't have a way to check if a repo is initialized or otherwise cleanly initialize it. So, we try to + initialize it and read the error message. Either a sucess message or an error suggesting it is already initialized + will cause this function to return true. + + Success message: + created restic repository xxxxx at xxxxx\n\n + + Error message: + Fatal: create key in repository at xxxx failed: repository master key and config already initialized\n\n' + """ + message = message.strip() + + return message.startswith("created restic repository ") or ( + message.startswith("Fatal: create key in repository at ") and + message.endswith("repository master key and config already initialized")) + + +def get_newest_snapshot(name, config): + snapshout_groups = get_snapshot_groups(name, config) + if len(snapshout_groups) > 1: + return die("found {} groups, but only support 1".format(len(snapshout_groups))) + if not snapshout_groups: + return None + + snapshots = snapshout_groups[0]['snapshots'] + snapshots.sort(key=lambda x: x["time"], reverse=True) + + if not snapshots: + return None + + return snapshots[0] + + +def get_snapshot_groups(name, config): + cmd = ["snapshots", "--group-by", "tags", "--json"] + + retention_tags = { + "name": name, + } + for k, v in retention_tags.items(): + cmd.extend(["--tag", "{}={}".format(k, v)]) + + p = checkp(config.run(cmd, stdout=subprocess.PIPE)) + stdout, _ = p.communicate() + return json.loads(stdout.decode()) + + +def get_retention_args(schedule): + """ + given a retention schedule, return restic command(s) needed to make it so + """ + mode = schedule.get("function") + if mode is None: # default is to just keep stuff lol + return None + if mode == "forever": + # do not perform deletions + return None + elif mode == "keep": + # just keep the last X snapshots + return ["--keep-last", str(mode["count"])] + elif mode == "cycle": + # keep the last $last backups. + # keep a daily backup after that for the past $daily days. + # keep a weekly backup after that for the past $weekly weeks. + # keep a monthly backup after that for the past $monthly months. + cmd = [] + last = schedule.get("last") + if last is not None: + cmd.extend(["--keep-last", str(last)]) + + daily = schedule.get("daily") + if daily is not None: + cmd.extend(["--keep-daily", str(daily)]) + + weekly = schedule.get("weekly") + if weekly is not None: + cmd.extend(["--keep-weekly", str(weekly)]) + + monthly = schedule.get("monthly") + if monthly is not None: + cmd.extend(["--keep-monthly", str(monthly)]) + + return cmd or None + else: + raise Exception("unknown retention function {}".format(mode)) diff --git a/resticbackup/types.py b/resticbackup/types.py new file mode 100644 index 0000000..67b88da --- /dev/null +++ b/resticbackup/types.py @@ -0,0 +1,96 @@ +import os +import json +import subprocess +from urllib.parse import urlparse +from dataclasses import dataclass, field +from typing import List + +from resticbackup import CFG_DIR, RESTIC_BIN + + +def list_configs(): + config_names = [] + for fname in os.listdir(CFG_DIR): + name = fname.split(".")[0] + if name == "main": + continue + config_names.append(name) + config_names.sort() + return config_names + + +def load_base_config(): + main_path = os.path.join(CFG_DIR, "main.json") + with open(main_path) as f: + return json.load(f) + + +def load_config(name): + # load configs from /etc/resticbackup.d/ + # we have special handling for /etc/resticbackup.d/main.json + # return (main_config, dict(config_name=>config)) + cfg_path = os.path.join(CFG_DIR, "{}.json".format(name)) + + with open(cfg_path) as f: + backup_config = json.load(f) + + return ClientConfig.load(load_base_config(), backup_config) + + +@dataclass +class BackupConfig: + path: str + repo: str + schedule: dict = field(default_factory=dict) + exclude: List[str] = field(default_factory=list) + backup_preexec: List[str] = field(default_factory=list) + backup_postexec: List[str] = field(default_factory=list) + restore_preexec: List[str] = field(default_factory=list) + restore_postexec: List[str] = field(default_factory=list) + + @staticmethod + def load(data: dict) -> 'BackupConfig': + return BackupConfig( + path=data['path'], + repo=data['repo'], + schedule=data.get('schedule', {}), + exclude=data.get('exclude', []), + backup_preexec=data.get('backup_preexec', []), + backup_postexec=data.get('backup_postexec', []), + restore_preexec=data.get('restore_preexec', []), + restore_postexec=data.get('restore_postexec', []), + ) + + +@dataclass +class ClientConfig: + server_type: str + uri: str + secret: str + backup: BackupConfig + + @property + def repo(self): # port ignored + return "{}:{}://{}/{}".format(self.server_type, self.uri.scheme, self.uri.hostname, self.backup.repo) + + @property + def env(self): + return { + "AWS_ACCESS_KEY_ID": self.uri.username, + "AWS_SECRET_ACCESS_KEY": self.uri.password, + "RESTIC_PASSWORD": self.secret, + "RESTIC_REPOSITORY": self.repo, + } + + @staticmethod + def load(main, backup) -> "ClientConfig": + backup = BackupConfig.load(backup) + server_type, server = main["server"].split(":", 1) + if server_type != "s3": + raise Exception("unsupported server type: {}".format(server_type)) + return ClientConfig(server_type=server_type, uri=urlparse(server), secret=main["secret"], backup=backup) + + def run(self, args, **kwargs): + env = dict(os.environ) + env.update(**self.env) + return subprocess.Popen([RESTIC_BIN] + args, env=env, **kwargs)