add timezone option

Dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:noble
+
+RUN apt-get update && \
+    apt-get install -y wget python3 python3-pip
+
+RUN wget -qO- https://github.com/restic/restic/releases/download/v0.15.1/restic_0.15.1_linux_amd64.bz2 | bunzip2 > /usr/local/bin/restic && \
+    chmod +x /usr/local/bin/restic
+
+RUN wget -O /tmp/resticbackup.tar.gz https://git.davepedu.com/dave/resticbackup/archive/0.0.12.tar.gz && \
+    mkdir /tmp/resticbackup/ && \
+    tar zxvf /tmp/resticbackup.tar.gz -C /tmp/resticbackup/ --strip-components=1 && \
+    cd /tmp/resticbackup/ && \
+    pip3 install --break-system-packages .[s3] && \
+    rm -rf /tmp/resticbackup.tar.gz /tmp/resticbackup
+
+USER nobody
+
+ENTRYPOINT ["resticbackup-s3-scanner"]

Jenkinsfile

@@ -0,0 +1,69 @@
+def image_name = "dpedu/resticbackup"
+
+pipeline {
+    agent {
+        kubernetes {
+            yaml """
+apiVersion: v1
+kind: Pod
+spec:
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:  # avoid nodes already running a jenkins job
+    - podAffinityTerm:
+        labelSelector:
+          matchExpressions:
+          - key: jenkins
+            operator: In
+            values:
+            - slave
+        topologyKey: node
+  containers:
+  - name: docker
+    image: docker:20-dind
+    args:
+      - "--insecure-registry"
+      - "dockermirror:5000"
+    securityContext:
+      privileged: true
+"""
+        }
+    }
+    stages {
+        stage("Build image") {
+            steps {
+                container("docker") {
+                    script {
+                        try {
+                            docker.withRegistry('http://dockermirror:5000') {
+                                docker.image("ubuntu:noble").pull()
+                                docker.image(image_name).pull() // Pull a recent version to share base layers with (?)
+                            }
+                        } catch (exc) {
+                            echo "couldn't pull image, assuming we're building it for the first time"
+                        }
+                        docker.build(image_name)
+                    }
+                }
+            }
+        }
+        stage("Push image") {
+            steps {
+                container("docker") {
+                    script {
+                        docker.withRegistry('http://dockermirror:5000') {
+                            docker.image(image_name).push("latest")
+                            docker.image(image_name).push("0.0.12")
+                        }
+                    }
+                }
+            }
+        }
+        stage("Show images") {
+            steps {
+                container("docker") {
+                    sh 'docker images'
+                }
+            }
+        }
+    }
+}

main.tf

@@ -0,0 +1,137 @@
+# this terraform module providers a kubernetes cron job that runs the pruner
+
+terraform {
+  backend "local" {}
+  required_version = ">= 1.5"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.25"
+    }
+  }
+}
+
+
+provider "kubernetes" {
+  config_path    = var.kubernetes_config_path
+  config_context = var.kubernetes_context
+}
+
+
+variable "kubernetes_config_path" {
+  type        = string
+  description = "path to kubernetes config to use"
+}
+
+variable "kubernetes_context" {
+  type        = string
+  description = "kubernetes context to use"
+}
+
+variable "namespace" {
+  type        = string
+  description = "kubernetes namespace to deploy in"
+}
+
+variable "image" {
+  type        = string
+  description = "docker image"
+  default     = "dockermirror:5000/dpedu/resticbackup:0.0.12"
+}
+
+variable "s3_uri" {
+  type        = string
+  description = "restic s3 uri"
+}
+
+variable "passwords" {
+  type        = map(string)
+  description = "restic encryption passwords"
+}
+
+variable "schedule" {
+  type        = string
+  description = "kubernetes cron expression"
+  default     = "15 0 * * *"
+}
+
+variable "schedule_timezone" {
+  type        = string
+  description = "timezone the schedule is interpreted as"
+  default     = "US/Pacific"
+}
+
+variable "suspend" {
+  type        = bool
+  description = "suspend the cronjob"
+  default     = false
+}
+
+variable "dry_run" {
+  type        = bool
+  description = "run in dry run mode"
+  default     = false
+}
+
+
+resource "kubernetes_secret_v1" "passwords" {
+  metadata {
+    name      = "resticbackup-pruner"
+    namespace = var.namespace
+  }
+
+  data = {
+    passwords = jsonencode(var.passwords)
+  }
+}
+
+resource "kubernetes_cron_job_v1" "scraper" {
+  metadata {
+    name      = "resticbackup-pruner"
+    namespace = var.namespace
+  }
+  spec {
+    schedule           = var.schedule
+    timezone           = var.schedule_timezone
+    suspend            = var.suspend
+    concurrency_policy = "Replace"
+    job_template {
+      metadata {}
+      spec {
+        template {
+          metadata {}
+          spec {
+            restart_policy = "Never"
+            container {
+              name              = "pruner"
+              image             = var.image
+              image_pull_policy = "Always"
+              args = concat(
+                [
+                  var.s3_uri,
+                  "/etc/passwords.json",
+                ],
+                var.dry_run ? ["--", "--dry-run", ] : []
+              )
+              volume_mount {
+                name       = "passwords"
+                mount_path = "/etc/passwords.json"
+                sub_path   = "passwords"
+              }
+              env {
+                name  = "XDG_CACHE_HOME"
+                value = "/tmp"
+              }
+            }
+            volume {
+              name = "passwords"
+              secret {
+                secret_name = kubernetes_secret_v1.passwords.metadata.0.name
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

requirements-s3.txt

@@ -0,0 +1,7 @@
+boto3==1.29.2
+botocore==1.32.2
+jmespath==1.0.1
+python-dateutil==2.8.2
+s3transfer==0.7.0
+six==1.16.0
+urllib3==1.26.18

resticbackup/__init__.py

@@ -4,4 +4,4 @@ CFG_DIR = os.environ.get("RESTICBACKUP_CONFIG_DIR", "/etc/resticbackup.d")
 RESTIC_BIN = os.environ.get("RESTICBACKUP_RESTIC_BIN_PATH", "restic")
 
 
-__version__ = "0.0.9"
+__version__ = "0.0.12"

resticbackup/s3.py

@@ -0,0 +1,93 @@
+import os
+import json
+import argparse
+import traceback
+import subprocess
+from shutil import which
+from urllib.parse import urlparse
+
+from boto3 import client as boto3_client
+from boto3.session import Config as boto3_config
+
+
+def scan_bucket(s3, s3_bucket, path):
+    for page in s3.get_paginator("list_objects_v2").paginate(
+        Bucket=s3_bucket,
+        Delimiter="/",
+        Prefix=path
+    ):
+        prefixes = set(i["Prefix"].split("/")[-2] for i in page.get("CommonPrefixes", []))
+        contents = set(i["Key"].split("/")[-1] for i in page.get("Contents", []))
+
+        # if it smells like a populated restic restic repo, return it and don't go deeper into it
+        if "config" in contents and set(['data', 'index', 'keys', 'snapshots']).issubset(prefixes):
+            yield path
+        else:
+            for prefix in prefixes:
+                yield from scan_bucket(s3, s3_bucket, path + prefix + "/")
+
+
+def discover_repos(s3, s3_bucket):
+    """
+    using the given s3 client, scan files in the bucket
+    """
+    yield from scan_bucket(s3, s3_bucket, "")
+
+
+def prune(s3, restic_uri, username, password, repo_password, extra_args=None):
+    print("\nPruning: " + restic_uri, "\n")
+
+    env = dict(os.environ)
+    env["AWS_ACCESS_KEY_ID"] = username
+    env["AWS_SECRET_ACCESS_KEY"] = password
+    env["RESTIC_PASSWORD"] = repo_password
+
+    try:
+        subprocess.check_call(
+            [which("restic"), "-r", restic_uri, "prune"] + (extra_args or []),
+            env=env
+        )
+    except subprocess.CalledProcessError:
+        traceback.print_exc()
+        print("assuming we didn't have the repo password, ignoring")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="restic pruning wrapper")
+
+    parser.add_argument("s3uri", help="s3 uri. Same format as restic, without the leading s3:")
+    parser.add_argument("passwordsfile", help="file containing restic encryption passwords")
+    parser.add_argument('args', nargs="*", help="additional arguments to pass to restic")
+
+    args = parser.parse_args()
+    uri = urlparse(args.s3uri)
+
+    with open(args.passwordsfile) as f:
+        passwords = json.load(f)
+
+    default_password = passwords.get("default")
+
+    maybeport = ":{}".format(uri.port) if uri.port else ""
+    s3_addr = "{}://{}{}".format(uri.scheme, uri.hostname, maybeport)
+    s3_bucket = uri.path[1:]
+
+    s3 = boto3_client(
+        "s3",
+        endpoint_url=s3_addr,
+        aws_access_key_id=uri.username,
+        aws_secret_access_key=uri.password,
+        aws_session_token=None,
+        config=boto3_config(signature_version="s3v4"),
+        # verify=False,
+    )
+
+    for repo in discover_repos(s3, s3_bucket):
+        restic_uri = "s3:{}/{}/{}".format(s3_addr, s3_bucket, repo)
+        password = passwords.get("{}/{}".format(s3_bucket, repo)) or passwords.get(s3_bucket + "/") or default_password
+        if not passwords:
+            raise Exception("password not provided for: {}/{}".format(s3_bucket, repo))
+        prune(s3, restic_uri, uri.username, uri.password, password, args.args)
+
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -1,8 +1,16 @@
+import os
 from setuptools import setup
 
 from resticbackup import __version__
 
 
+here = os.path.abspath(os.path.normpath(os.path.dirname(__file__)))
+
+
+with open(os.path.join(here, "requirements-s3.txt")) as f:
+    reqs_s3 = [line.strip() for line in f.readlines()]
+
+
 setup(name='resticbackup',
       version=__version__,
       description='Wrapper around restic for automated backups',
@@ -14,6 +22,10 @@ setup(name='resticbackup',
       entry_points={
           "console_scripts": [
               "resticbackup = resticbackup.cli:main",
+              "resticbackup-s3-scanner = resticbackup.s3:main",
           ]
       },
+      extras_require={
+          "s3": reqs_s3
+      },
       zip_safe=False)