re-try retention and pruning with backoff

resticbackup/__init__.py

@@ -4,4 +4,4 @@ CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d")
 RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic")
 
 
-__version__ = "0.0.4"
+__version__ = "0.0.5"

resticbackup/cli.py

@@ -1,10 +1,11 @@
 import os
 import argparse
 import subprocess
-
+import traceback
+import time
 from resticbackup import __version__
 from resticbackup.types import ClientConfig, load_config, load_base_config, list_configs
-from resticbackup.lib import ExecWrapper, init_ok, die, pdie, checkp, get_retention_args, \
+from resticbackup.lib import ExecWrapper, init_ok, die, pdie, checkp, runp, get_retention_args, \
     get_newest_snapshot, update_statefile
 
 
@@ -13,7 +14,9 @@ def cmd_backup(args, parser):
 
     try:
         run_backup(args.name, config)
-        run_retention(args.name, config)
+        if backoff(run_retention, args.name, config):
+            # TODO don't run prune automatically as it is repo-wide and every other client runs it too. Unnecessary
+            backoff(run_prune, config)
         # update statefile if configured
         if config.statefile:
             path = config.statefile.format(name=args.name)
@@ -26,7 +29,8 @@ def cmd_backup(args, parser):
 
 
 def run_backup(name, config):
-    # these tags will be added to the backup. They need to match retention_tags in run_rentention, below
+    # these tags will be added to the backup.
+    # They need to match retention_tags in run_rentention, below
     backup_tags = {
         "name": name,
     }
@@ -55,8 +59,35 @@ def run_backup(name, config):
             checkp(config.run(backup_cmd))
 
 
+def backoff(func, *args, **kwargs):
+    """
+    run the retention function, but retry if it fails. Restic must lock the entire repo during this process, so it is
+    possible that it fails due to already being in use by another client
+    """
+
+    backoff = [1, 2, 3, 5, 10, 15, 30, 60]
+    backoffs = 0
+
+    while True:
+        try:
+            return func(*args, **kwargs)
+        except:
+            if backoffs > len(backoff):
+                raise
+            traceback.print_exc()
+            delay = backoff[backoffs]
+            backoffs += 1
+            print("...caught, and will be retried in {}m".format(delay))
+            time.sleep(delay)
+
+
 def run_retention(name, config):
-    # the tags in retention_tags will be used when pruning backups
+    """
+    run retention, which untags old backups. Returns true if retention is configured and was executed, false otherwise
+    """
+
+    # the tags in retention_tags will be used when pruning backups.
+    # They need to match backup_tags in run_backup, above
     retention_tags = {
         "name": name,
     }
@@ -67,10 +98,14 @@ def run_retention(name, config):
         forget_cmd = ["forget", "--group-by", "tags"] + retention_args
         for k, v in retention_tags.items():
             forget_cmd.extend(["--tag", "{}={}".format(k, v)])
-        checkp(config.run(forget_cmd))
+        runp(config.run(forget_cmd))
+        return True
+    return False
 
-        # perform junk deletion
-        checkp(config.run(["prune"]))
+
+def run_prune(config):
+    # perform junk deletion
+    runp(config.run(["prune"]))
 
 
 def cmd_restore(args, parser):

resticbackup/lib.py

@@ -22,6 +22,13 @@ def checkp(popen):
     return popen
 
 
+def runp(popen):
+    popen.wait()
+    if popen.returncode != 0:
+        raise Exception("command failed: rc={}".format(popen.returncode))  # TODO use subprocess.check_call
+    return popen
+
+
 class ExecWrapper(object):
     def __init__(self, pre=None, post=None):
         self.pre = pre or []