add restore function and refactor into a few files
This commit is contained in:
parent
3671125f34
commit
bf764305d3
|
|
@ -0,0 +1,4 @@
|
|||
import os
|
||||
|
||||
CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d")
|
||||
RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic")
|
||||
|
|
@ -1,156 +1,10 @@
|
|||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import subprocess
|
||||
from socket import getfqdn
|
||||
from urllib.parse import urlparse
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
|
||||
CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d")
|
||||
RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic")
|
||||
|
||||
|
||||
def die(msg, rc=1):
|
||||
print(msg)
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
def pdie(popen):
|
||||
popen.wait()
|
||||
sys.exit(popen.returncode)
|
||||
|
||||
|
||||
def checkp(popen):
|
||||
popen.wait()
|
||||
if popen.returncode != 0:
|
||||
die(rc=popen.returncode)
|
||||
return popen
|
||||
|
||||
|
||||
def list_configs():
|
||||
config_names = []
|
||||
for fname in os.listdir(CFG_DIR):
|
||||
name = fname.split(".")[0]
|
||||
if name == "main":
|
||||
continue
|
||||
config_names.append(name)
|
||||
config_names.sort()
|
||||
return config_names
|
||||
|
||||
|
||||
def load_base_config():
|
||||
main_path = os.path.join(CFG_DIR, "main.json")
|
||||
with open(main_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def load_config(name):
|
||||
# load configs from /etc/resticbackup.d/
|
||||
# we have special handling for /etc/resticbackup.d/main.json
|
||||
# return (main_config, dict(config_name=>config))
|
||||
cfg_path = os.path.join(CFG_DIR, "{}.json".format(name))
|
||||
|
||||
with open(cfg_path) as f:
|
||||
backup_config = json.load(f)
|
||||
|
||||
return ClientConfig.load(load_base_config(), backup_config)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupConfig:
|
||||
path: str
|
||||
repo: str
|
||||
schedule: dict = field(default_factory=dict)
|
||||
exclude: List[str] = field(default_factory=list)
|
||||
backup_preexec: List[str] = field(default_factory=list)
|
||||
backup_postexec: List[str] = field(default_factory=list)
|
||||
restore_preexec: List[str] = field(default_factory=list)
|
||||
restore_postexec: List[str] = field(default_factory=list)
|
||||
|
||||
@staticmethod
|
||||
def load(data: dict) -> 'BackupConfig':
|
||||
return BackupConfig(
|
||||
path=data['path'],
|
||||
repo=data['repo'],
|
||||
schedule=data.get('schedule', {}),
|
||||
exclude=data.get('exclude', []),
|
||||
backup_preexec=data.get('backup_preexec', []),
|
||||
backup_postexec=data.get('backup_postexec', []),
|
||||
restore_preexec=data.get('restore_preexec', []),
|
||||
restore_postexec=data.get('restore_postexec', []),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClientConfig:
|
||||
server_type: str
|
||||
uri: str
|
||||
secret: str
|
||||
backup: BackupConfig
|
||||
|
||||
@property
|
||||
def repo(self): # port ignored
|
||||
return "{}:{}://{}/{}".format(self.server_type, self.uri.scheme, self.uri.hostname, self.backup.repo)
|
||||
|
||||
@property
|
||||
def env(self):
|
||||
return {
|
||||
"AWS_ACCESS_KEY_ID": self.uri.username,
|
||||
"AWS_SECRET_ACCESS_KEY": self.uri.password,
|
||||
"RESTIC_PASSWORD": self.secret,
|
||||
"RESTIC_REPOSITORY": self.repo,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def load(main, backup) -> "ClientConfig":
|
||||
backup = BackupConfig.load(backup)
|
||||
server_type, server = main["server"].split(":", 1)
|
||||
if server_type != "s3":
|
||||
raise Exception("unsupported server type: {}".format(server_type))
|
||||
return ClientConfig(server_type=server_type, uri=urlparse(server), secret=main["secret"], backup=backup)
|
||||
|
||||
def run(self, args, **kwargs):
|
||||
env = dict(os.environ)
|
||||
env.update(**self.env)
|
||||
return subprocess.Popen([RESTIC_BIN] + args, env=env, **kwargs)
|
||||
|
||||
|
||||
class ExecWrapper(object):
|
||||
def __init__(self, pre=None, post=None):
|
||||
self.pre = pre or []
|
||||
self.post = post or []
|
||||
|
||||
def __enter__(self):
|
||||
for command in self.pre:
|
||||
print("+", command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
for command in self.post:
|
||||
print("+", command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
|
||||
|
||||
def init_ok(message):
|
||||
"""
|
||||
Restic doesn't have a way to check if a repo is initialized or otherwise cleanly initialize it. So, we try to
|
||||
initialize it and read the error message. Either a sucess message or an error suggesting it is already initialized
|
||||
will cause this function to return true.
|
||||
|
||||
Success message:
|
||||
created restic repository xxxxx at xxxxx\n\n
|
||||
|
||||
Error message:
|
||||
Fatal: create key in repository at xxxx failed: repository master key and config already initialized\n\n'
|
||||
"""
|
||||
message = message.strip()
|
||||
|
||||
return message.startswith("created restic repository ") or (
|
||||
message.startswith("Fatal: create key in repository at ") and
|
||||
message.endswith("repository master key and config already initialized"))
|
||||
from resticbackup.types import ClientConfig, load_config, load_base_config, list_configs
|
||||
from resticbackup.lib import ExecWrapper, init_ok, die, pdie, checkp, get_retention_args, get_newest_snapshot
|
||||
|
||||
|
||||
def cmd_backup(args, parser):
|
||||
|
|
@ -205,48 +59,42 @@ def cmd_backup(args, parser):
|
|||
return pdie(config.run(["prune"]))
|
||||
|
||||
|
||||
def get_retention_args(schedule):
|
||||
"""
|
||||
given a retention schedule, return restic command(s) needed to make it so
|
||||
"""
|
||||
mode = schedule.get("function")
|
||||
if mode is None: # default is to just keep stuff lol
|
||||
return None
|
||||
if mode == "forever":
|
||||
# do not perform deletions
|
||||
return None
|
||||
elif mode == "keep":
|
||||
# just keep the last X snapshots
|
||||
return ["--keep-last", str(mode["count"])]
|
||||
elif mode == "cycle":
|
||||
# keep the last $last backups.
|
||||
# keep a daily backup after that for the past $daily days.
|
||||
# keep a weekly backup after that for the past $weekly weeks.
|
||||
# keep a monthly backup after that for the past $monthly months.
|
||||
cmd = []
|
||||
last = schedule.get("last")
|
||||
if last is not None:
|
||||
cmd.extend(["--keep-last", str(last)])
|
||||
|
||||
daily = schedule.get("daily")
|
||||
if daily is not None:
|
||||
cmd.extend(["--keep-daily", str(daily)])
|
||||
|
||||
weekly = schedule.get("weekly")
|
||||
if weekly is not None:
|
||||
cmd.extend(["--keep-weekly", str(weekly)])
|
||||
|
||||
monthly = schedule.get("monthly")
|
||||
if monthly is not None:
|
||||
cmd.extend(["--keep-monthly", str(monthly)])
|
||||
|
||||
return cmd or None
|
||||
else:
|
||||
raise Exception("unknown retention function {}".format(mode))
|
||||
|
||||
|
||||
def cmd_restore(args, parser):
|
||||
pass # TODO restore backups
|
||||
config = load_config(args.name)
|
||||
|
||||
try:
|
||||
if os.listdir(config.backup.path):
|
||||
return die("output dir has files present - refusing to restore")
|
||||
except FileNotFoundError:
|
||||
return die("output dir '{}' does not exist".format(config.backup.path))
|
||||
|
||||
snapshot = get_newest_snapshot(args.name, config)
|
||||
|
||||
if not snapshot:
|
||||
return die("no snapshots found")
|
||||
|
||||
os.chdir(config.backup.path)
|
||||
with ExecWrapper(post=config.backup.restore_postexec):
|
||||
with ExecWrapper(pre=config.backup.restore_preexec):
|
||||
return pdie(config.run(["restore", snapshot["id"], "-t", "./"]))
|
||||
|
||||
|
||||
def cmd_retrieve(args, parser):
|
||||
# this is the same as cmd_restore except for how the config is formed and no execs
|
||||
config = ClientConfig.load(load_base_config(), {"repo": args.repo, "path": args.outdir})
|
||||
|
||||
try:
|
||||
if os.listdir(config.backup.path):
|
||||
return die("output dir has files present - refusing to restore")
|
||||
except FileNotFoundError:
|
||||
return die("output dir does not exist")
|
||||
|
||||
snapshot = get_newest_snapshot(args.name, config)
|
||||
|
||||
if not snapshot:
|
||||
return die("no snapshots found")
|
||||
|
||||
return pdie(config.run(["restore", snapshot["id"], "-t", args.outdir]))
|
||||
|
||||
|
||||
def cmd_list(args, parser):
|
||||
|
|
@ -267,7 +115,6 @@ def cmd_list(args, parser):
|
|||
print(entry)
|
||||
|
||||
|
||||
|
||||
def cmd_exec(args, parser):
|
||||
checkp(load_config(args.name).run(args.args))
|
||||
|
||||
|
|
@ -275,16 +122,24 @@ def cmd_exec(args, parser):
|
|||
def main():
|
||||
parser = argparse.ArgumentParser(description="restic wrapper")
|
||||
|
||||
parser.add_argument('-n', '--no-exec', action='store_true', help='do not run pre/post-exec commands')
|
||||
parser.add_argument('-b', '--no-pre-exec', action='store_true', help='do not run pre-exec commands')
|
||||
parser.add_argument('-m', '--no-post-exec', action='store_true', help='do not run post-exec commands')
|
||||
# parser.add_argument('-n', '--no-exec', action='store_true', help='do not run pre/post-exec commands') # TODO
|
||||
# parser.add_argument('-b', '--no-pre-exec', action='store_true', help='do not run pre-exec commands') # TODO
|
||||
# parser.add_argument('-m', '--no-post-exec', action='store_true', help='do not run post-exec commands') # TODO
|
||||
|
||||
sp_action = parser.add_subparsers(dest="action", help="action to take")
|
||||
|
||||
p_backup = sp_action.add_parser("backup", help="take a backup")
|
||||
p_backup.set_defaults(func=cmd_backup)
|
||||
p_backup.add_argument("name", help="name of backup to execute")
|
||||
# p_backup.add_argument("-p", "--print", action="store_true", help="just print the restic commands instead") # TODO
|
||||
# p_backup.add_argument("-c", "--print", action="store_true", help="just print the restic commands instead") # TODO
|
||||
# p_backup.add_argument("-e", "--env", action="store_true", help="just print the environment variables instead") # TODO
|
||||
|
||||
p_restore = sp_action.add_parser("restore", help="restore a backup")
|
||||
p_restore.set_defaults(func=cmd_restore)
|
||||
p_restore.add_argument("name", help="name of backup to restore")
|
||||
# p_restore.add_argument("snapshot", nargs="?", help="id of snapshot to restore") # TODO
|
||||
p_restore.add_argument('--no-snapshots-ok', action='store_true',
|
||||
help='don\'t return an error if no snapshots are available to be restored')
|
||||
|
||||
p_exec = sp_action.add_parser("exec", help="execute a restic command")
|
||||
p_exec.set_defaults(func=cmd_exec)
|
||||
|
|
@ -292,11 +147,16 @@ def main():
|
|||
p_exec.add_argument("args", nargs="+", help="command arguments")
|
||||
|
||||
p_list = sp_action.add_parser("list", help="list configs or snapshots")
|
||||
p_list.add_argument("name", nargs="?", help="name of config to list snapshots for")
|
||||
p_list.set_defaults(func=cmd_list)
|
||||
p_list.add_argument("name", nargs="?", help="name of config to list snapshots for")
|
||||
|
||||
p_retrieve = sp_action.add_parser("retrieve", help="download arbitrary snapshots")
|
||||
p_retrieve.set_defaults(func=cmd_retrieve)
|
||||
p_retrieve.add_argument("repo", help="repo path")
|
||||
p_retrieve.add_argument("name", help="backup name")
|
||||
p_retrieve.add_argument("outdir", help="file path to write to")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
args.func(args, parser)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,125 @@
|
|||
import sys
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
|
||||
def die(msg, rc=1):
|
||||
print(msg)
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
def pdie(popen):
|
||||
popen.wait()
|
||||
sys.exit(popen.returncode)
|
||||
|
||||
|
||||
def checkp(popen):
|
||||
popen.wait()
|
||||
if popen.returncode != 0:
|
||||
die(rc=popen.returncode)
|
||||
return popen
|
||||
|
||||
|
||||
class ExecWrapper(object):
|
||||
def __init__(self, pre=None, post=None):
|
||||
self.pre = pre or []
|
||||
self.post = post or []
|
||||
|
||||
def __enter__(self):
|
||||
for command in self.pre:
|
||||
print("+", command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
for command in self.post:
|
||||
print("+", command)
|
||||
subprocess.check_call(command, shell=True)
|
||||
|
||||
|
||||
def init_ok(message):
|
||||
"""
|
||||
Restic doesn't have a way to check if a repo is initialized or otherwise cleanly initialize it. So, we try to
|
||||
initialize it and read the error message. Either a sucess message or an error suggesting it is already initialized
|
||||
will cause this function to return true.
|
||||
|
||||
Success message:
|
||||
created restic repository xxxxx at xxxxx\n\n
|
||||
|
||||
Error message:
|
||||
Fatal: create key in repository at xxxx failed: repository master key and config already initialized\n\n'
|
||||
"""
|
||||
message = message.strip()
|
||||
|
||||
return message.startswith("created restic repository ") or (
|
||||
message.startswith("Fatal: create key in repository at ") and
|
||||
message.endswith("repository master key and config already initialized"))
|
||||
|
||||
|
||||
def get_newest_snapshot(name, config):
|
||||
snapshout_groups = get_snapshot_groups(name, config)
|
||||
if len(snapshout_groups) > 1:
|
||||
return die("found {} groups, but only support 1".format(len(snapshout_groups)))
|
||||
if not snapshout_groups:
|
||||
return None
|
||||
|
||||
snapshots = snapshout_groups[0]['snapshots']
|
||||
snapshots.sort(key=lambda x: x["time"], reverse=True)
|
||||
|
||||
if not snapshots:
|
||||
return None
|
||||
|
||||
return snapshots[0]
|
||||
|
||||
|
||||
def get_snapshot_groups(name, config):
|
||||
cmd = ["snapshots", "--group-by", "tags", "--json"]
|
||||
|
||||
retention_tags = {
|
||||
"name": name,
|
||||
}
|
||||
for k, v in retention_tags.items():
|
||||
cmd.extend(["--tag", "{}={}".format(k, v)])
|
||||
|
||||
p = checkp(config.run(cmd, stdout=subprocess.PIPE))
|
||||
stdout, _ = p.communicate()
|
||||
return json.loads(stdout.decode())
|
||||
|
||||
|
||||
def get_retention_args(schedule):
|
||||
"""
|
||||
given a retention schedule, return restic command(s) needed to make it so
|
||||
"""
|
||||
mode = schedule.get("function")
|
||||
if mode is None: # default is to just keep stuff lol
|
||||
return None
|
||||
if mode == "forever":
|
||||
# do not perform deletions
|
||||
return None
|
||||
elif mode == "keep":
|
||||
# just keep the last X snapshots
|
||||
return ["--keep-last", str(mode["count"])]
|
||||
elif mode == "cycle":
|
||||
# keep the last $last backups.
|
||||
# keep a daily backup after that for the past $daily days.
|
||||
# keep a weekly backup after that for the past $weekly weeks.
|
||||
# keep a monthly backup after that for the past $monthly months.
|
||||
cmd = []
|
||||
last = schedule.get("last")
|
||||
if last is not None:
|
||||
cmd.extend(["--keep-last", str(last)])
|
||||
|
||||
daily = schedule.get("daily")
|
||||
if daily is not None:
|
||||
cmd.extend(["--keep-daily", str(daily)])
|
||||
|
||||
weekly = schedule.get("weekly")
|
||||
if weekly is not None:
|
||||
cmd.extend(["--keep-weekly", str(weekly)])
|
||||
|
||||
monthly = schedule.get("monthly")
|
||||
if monthly is not None:
|
||||
cmd.extend(["--keep-monthly", str(monthly)])
|
||||
|
||||
return cmd or None
|
||||
else:
|
||||
raise Exception("unknown retention function {}".format(mode))
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
import os
|
||||
import json
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
|
||||
from resticbackup import CFG_DIR, RESTIC_BIN
|
||||
|
||||
|
||||
def list_configs():
|
||||
config_names = []
|
||||
for fname in os.listdir(CFG_DIR):
|
||||
name = fname.split(".")[0]
|
||||
if name == "main":
|
||||
continue
|
||||
config_names.append(name)
|
||||
config_names.sort()
|
||||
return config_names
|
||||
|
||||
|
||||
def load_base_config():
|
||||
main_path = os.path.join(CFG_DIR, "main.json")
|
||||
with open(main_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def load_config(name):
|
||||
# load configs from /etc/resticbackup.d/
|
||||
# we have special handling for /etc/resticbackup.d/main.json
|
||||
# return (main_config, dict(config_name=>config))
|
||||
cfg_path = os.path.join(CFG_DIR, "{}.json".format(name))
|
||||
|
||||
with open(cfg_path) as f:
|
||||
backup_config = json.load(f)
|
||||
|
||||
return ClientConfig.load(load_base_config(), backup_config)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BackupConfig:
|
||||
path: str
|
||||
repo: str
|
||||
schedule: dict = field(default_factory=dict)
|
||||
exclude: List[str] = field(default_factory=list)
|
||||
backup_preexec: List[str] = field(default_factory=list)
|
||||
backup_postexec: List[str] = field(default_factory=list)
|
||||
restore_preexec: List[str] = field(default_factory=list)
|
||||
restore_postexec: List[str] = field(default_factory=list)
|
||||
|
||||
@staticmethod
|
||||
def load(data: dict) -> 'BackupConfig':
|
||||
return BackupConfig(
|
||||
path=data['path'],
|
||||
repo=data['repo'],
|
||||
schedule=data.get('schedule', {}),
|
||||
exclude=data.get('exclude', []),
|
||||
backup_preexec=data.get('backup_preexec', []),
|
||||
backup_postexec=data.get('backup_postexec', []),
|
||||
restore_preexec=data.get('restore_preexec', []),
|
||||
restore_postexec=data.get('restore_postexec', []),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClientConfig:
|
||||
server_type: str
|
||||
uri: str
|
||||
secret: str
|
||||
backup: BackupConfig
|
||||
|
||||
@property
|
||||
def repo(self): # port ignored
|
||||
return "{}:{}://{}/{}".format(self.server_type, self.uri.scheme, self.uri.hostname, self.backup.repo)
|
||||
|
||||
@property
|
||||
def env(self):
|
||||
return {
|
||||
"AWS_ACCESS_KEY_ID": self.uri.username,
|
||||
"AWS_SECRET_ACCESS_KEY": self.uri.password,
|
||||
"RESTIC_PASSWORD": self.secret,
|
||||
"RESTIC_REPOSITORY": self.repo,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def load(main, backup) -> "ClientConfig":
|
||||
backup = BackupConfig.load(backup)
|
||||
server_type, server = main["server"].split(":", 1)
|
||||
if server_type != "s3":
|
||||
raise Exception("unsupported server type: {}".format(server_type))
|
||||
return ClientConfig(server_type=server_type, uri=urlparse(server), secret=main["secret"], backup=backup)
|
||||
|
||||
def run(self, args, **kwargs):
|
||||
env = dict(os.environ)
|
||||
env.update(**self.env)
|
||||
return subprocess.Popen([RESTIC_BIN] + args, env=env, **kwargs)
|
||||
Loading…
Reference in New Issue