From 2d45f89d85592af205567cb5d0da164c17977a88 Mon Sep 17 00:00:00 2001
From: dave <dave@davepedu.com>
Date: Fri, 15 Sep 2023 00:36:53 -0700
Subject: [PATCH] re-try retention and pruning with backoff

---
 resticbackup/__init__.py |  2 +-
 resticbackup/cli.py      | 51 +++++++++++++++++++++++++++++++++-------
 resticbackup/lib.py      |  7 ++++++
 3 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/resticbackup/__init__.py b/resticbackup/__init__.py
index 744f088..07b6300 100644
--- a/resticbackup/__init__.py
+++ b/resticbackup/__init__.py
@@ -4,4 +4,4 @@ CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d")
 RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic")
 
 
-__version__ = "0.0.4"
+__version__ = "0.0.5"
diff --git a/resticbackup/cli.py b/resticbackup/cli.py
index a160915..411a764 100644
--- a/resticbackup/cli.py
+++ b/resticbackup/cli.py
@@ -1,10 +1,11 @@
 import os
 import argparse
 import subprocess
-
+import traceback
+import time
 from resticbackup import __version__
 from resticbackup.types import ClientConfig, load_config, load_base_config, list_configs
-from resticbackup.lib import ExecWrapper, init_ok, die, pdie, checkp, get_retention_args, \
+from resticbackup.lib import ExecWrapper, init_ok, die, pdie, checkp, runp, get_retention_args, \
     get_newest_snapshot, update_statefile
 
 
@@ -13,7 +14,9 @@ def cmd_backup(args, parser):
 
     try:
         run_backup(args.name, config)
-        run_retention(args.name, config)
+        if backoff(run_retention, args.name, config):
+            # TODO don't run prune automatically as it is repo-wide and every other client runs it too. Unnecessary
+            backoff(run_prune, config)
         # update statefile if configured
         if config.statefile:
             path = config.statefile.format(name=args.name)
@@ -26,7 +29,8 @@ def cmd_backup(args, parser):
 
 
 def run_backup(name, config):
-    # these tags will be added to the backup. They need to match retention_tags in run_rentention, below
+    # these tags will be added to the backup.
+    # They need to match retention_tags in run_rentention, below
     backup_tags = {
         "name": name,
     }
@@ -55,8 +59,35 @@ def run_backup(name, config):
             checkp(config.run(backup_cmd))
 
 
+def backoff(func, *args, **kwargs):
+    """
+    run the retention function, but retry if it fails. Restic must lock the entire repo during this process, so it is
+    possible that it fails due to already being in use by another client
+    """
+
+    backoff = [1, 2, 3, 5, 10, 15, 30, 60]
+    backoffs = 0
+
+    while True:
+        try:
+            return func(*args, **kwargs)
+        except:
+            if backoffs > len(backoff):
+                raise
+            traceback.print_exc()
+            delay = backoff[backoffs]
+            backoffs += 1
+            print("...caught, and will be retried in {}m".format(delay))
+            time.sleep(delay)
+
+
 def run_retention(name, config):
-    # the tags in retention_tags will be used when pruning backups
+    """
+    run retention, which untags old backups. Returns true if retention is configured and was executed, false otherwise
+    """
+
+    # the tags in retention_tags will be used when pruning backups.
+    # They need to match backup_tags in run_backup, above
     retention_tags = {
         "name": name,
     }
@@ -67,10 +98,14 @@ def run_retention(name, config):
         forget_cmd = ["forget", "--group-by", "tags"] + retention_args
         for k, v in retention_tags.items():
             forget_cmd.extend(["--tag", "{}={}".format(k, v)])
-        checkp(config.run(forget_cmd))
+        runp(config.run(forget_cmd))
+        return True
+    return False
 
-        # perform junk deletion
-        checkp(config.run(["prune"]))
+
+def run_prune(config):
+    # perform junk deletion
+    runp(config.run(["prune"]))
 
 
 def cmd_restore(args, parser):
diff --git a/resticbackup/lib.py b/resticbackup/lib.py
index eae8838..45a872b 100644
--- a/resticbackup/lib.py
+++ b/resticbackup/lib.py
@@ -22,6 +22,13 @@ def checkp(popen):
     return popen
 
 
+def runp(popen):
+    popen.wait()
+    if popen.returncode != 0:
+        raise Exception("command failed: rc={}".format(popen.returncode))  # TODO use subprocess.check_call
+    return popen
+
+
 class ExecWrapper(object):
     def __init__(self, pre=None, post=None):
         self.pre = pre or []