fix: download without buffering volumes in RAM

Fetch each path with 'restic restore' into a temp dir, instead of
buffering whole volume dumps in memory, which crashed on large volumes.
Restore is also much faster than the sequential 'restic dump'.
The archive layout is unchanged.

.drone.yml

@@ -5,9 +5,9 @@ steps:
   - name: publish image
     image: plugins/docker
     settings:
-      username: 3wordchant
+      username: abra-bot
       password:
-        from_secret: git_coopcloud_tech_token_3wc
+        from_secret: git_coopcloud_tech_token_abra_bot
       repo: git.coopcloud.tech/coop-cloud/backup-bot-two
       tags: ${DRONE_SEMVER_BUILD}
       registry: git.coopcloud.tech

.env.sample

@@ -4,6 +4,8 @@ SECRET_RESTIC_PASSWORD_VERSION=v1
 
 COMPOSE_FILE=compose.yml
 
+#TIMEOUT=
+
 RESTIC_REPOSITORY=/backups/restic
 
 CRON_SCHEDULE='30 3 * * *'
@@ -38,10 +40,3 @@ CRON_SCHEDULE='30 3 * * *'
 # it overwrites the RESTIC_REPOSITORY variable
 #SECRET_RESTIC_REPO_VERSION=v1
 #COMPOSE_FILE="$COMPOSE_FILE:compose.secret.yml"
-
-# Restic exporter - prometheus metrics
-#COMPOSE_FILE="$COMPOSE_FILE:compose.exporter.yml"
-#LETS_ENCRYPT_ENV=production
-#REFRESH_INTERVAL=86400 # once per day, be cautious with small numbers, as this can create traffic
-#METRICS_DOMAIN=backup.example.com
-#TIMEZONE=Europe/Berlin

Dockerfile

@@ -1,11 +1,10 @@
-FROM docker:24.0.7-dind
+FROM docker:29.4.3-dind
 
 RUN apk add --upgrade --no-cache restic bash python3 py3-pip py3-click py3-docker-py py3-json-logger curl
 
-# Todo use requirements file with specific versions
-RUN pip install --break-system-packages resticpy==1.0.2
+RUN pip install --break-system-packages resticpy==1.3.0
 
 COPY backupbot.py /usr/bin/backup
 COPY entrypoint.sh /entrypoint.sh
 
-ENTRYPOINT /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

README.md

@@ -79,10 +79,8 @@ Add the key to your `authorized_keys`:
 `ssh-copy-id -i backupkey <user>@<hostname>`
 Add your `SSH_KEY` as docker secret:
 ```
-abra app secret insert <backupbot_name> ssh_key v1 """$(cat backupkey)
-"""
+cat backupkey | abra app secret insert <backupbot_name> ssh_key v1
 ```
-> Attention: This command needs to be executed exactly as stated above, because it places a trailing newline at the end, if this is missing you will get the following error: `Load key "/run/secrets/ssh_key": error in libcrypto`
 
 ### Restic REST server Storage
 
@@ -139,55 +137,57 @@ Enable basic auth in the env file, by uncommenting the following line:
 
 ## Usage
 
-Run the cronjob that creates a backup, including the push notifications and docker logging:
+### Run the cronjob that creates a backup, including the push notifications and docker logging:
+
 `abra app cmd <backupbot_name> app run_cron`
 
-Create a backup of all apps:
+### Create a backup of all apps:
 
 `abra app run <backupbot_name> app -- backup create`
 
 > The apps to backup up need to be deployed
 
-Create an individual backup:
+### Create an individual backup:
 
 `abra app run <backupbot_name> app -- backup --host <target_app_name> create`
 
-Create a backup to a local repository:
+### Create a backup to a local repository:
 
 `abra app run <backupbot_name> app -- backup create -r /backups/restic`
 
 > It is recommended to shutdown/undeploy an app before restoring the data
 
-Restore the latest snapshot of all including apps:
+### Restore the latest snapshot of all including apps:
 
 `abra app run <backupbot_name> app -- backup restore`
 
-Restore a specific snapshot of an individual app:
+### Restore a specific snapshot of an individual app:
 
 `abra app run <backupbot_name> app -- backup --host <target_app_name> restore --snapshot <snapshot_id>`
 
-Show all snapshots:
+### Show all snapshots:
 
 `abra app run <backupbot_name> app -- backup snapshots`
 
-Show all snapshots containing a specific app:
+### Show all snapshots containing a specific app:
 
 `abra app run <backupbot_name> app -- backup --host <target_app_name> snapshots`
 
-Show all files inside the latest snapshot (can be very verbose):
+### Show all files inside the latest snapshot (can be very verbose):
 
 `abra app run <backupbot_name> app -- backup ls`
 
-Show specific files inside a selected snapshot:
+### Show specific files inside a selected snapshot:
 
 `abra app run <backupbot_name> app -- backup ls --snapshot <snapshot_id> /var/lib/docker/volumes/`
 
-Download files from a snapshot:
+### Download files from a snapshot:
 
-```
-filename=$(abra app run <backupbot_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>)
-abra app cp <backupbot_name> app:$filename .
-```
+`abra app run <backupbot_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>`
+
+extract the <path> from the output of the first command and use in the second.
+
+`abra app cp <backupbot_name> app:<path> .`
 
 ## Run restic
 

abra.sh

@@ -1,6 +1,6 @@
 export SSH_CONFIG_VERSION=v1
 export ENTRYPOINT_VERSION=v17
-export CRONJOB_VERSION=v2
+export CRONJOB_VERSION=v3
 
 run_cron () {
     schedule="$(crontab -l | tr -s " " | cut -d ' ' -f-5)"

backupbot.py

@@ -9,7 +9,6 @@ import logging
 import docker
 import restic
 import tarfile
-import io
 from pythonjsonlogger import jsonlogger
 from datetime import datetime, timezone
 from restic.errors import ResticFailedError
@@ -19,6 +18,8 @@ from shutil import copyfile, rmtree
 VOLUME_PATH = "/var/lib/docker/volumes/"
 SECRET_PATH = "/secrets/"
 SERVICE = "ALL"
+BACKUP_FILE = "/tmp/backup.tar.gz"
+DOWNLOAD_TMP_PATH = "/tmp/backup-download"
 
 logger = logging.getLogger("backupbot")
 logging.addLevelName(55, "SUMMARY")
@@ -573,53 +574,85 @@ def download(snapshot, path, volumes, secrets):
         snapshot = latest_snapshot[0]["short_id"]
     if not any([path, volumes, secrets]):
         volumes = secrets = True
-    if path:
-        path = path.removesuffix("/")
-        binary_output = dump(snapshot, path)
-        files = list_files(snapshot, path)
-        filetype = [f.get("type") for f in files if f.get("path") == path][0]
-        filename = Path(path).name
-        if filetype == "dir":
-            filename = filename + ".tar"
-        tarinfo = tarfile.TarInfo(name=filename)
-        tarinfo.size = len(binary_output)
-        file_dumps.append((binary_output, tarinfo))
-    if volumes:
-        if SERVICE == "ALL":
-            logger.error("Please specify '--host' when using '--volumes'")
-            exit(1)
-        files = list_files(snapshot, VOLUME_PATH)
-        for f in files[1:]:
-            path = f["path"]
-            if Path(path).name.startswith(SERVICE) and f["type"] == "dir":
-                binary_output = dump(snapshot, path)
-                filename = f"{Path(path).name}.tar"
-                tarinfo = tarfile.TarInfo(name=filename)
-                tarinfo.size = len(binary_output)
-                file_dumps.append((binary_output, tarinfo))
-    if secrets:
-        if SERVICE == "ALL":
-            logger.error("Please specify '--host' when using '--secrets'")
-            exit(1)
-        filename = f"{SERVICE}.json"
-        files = list_files(snapshot, SECRET_PATH)
-        secrets = {}
-        for f in files[1:]:
-            path = f["path"]
-            if Path(path).name.startswith(SERVICE) and f["type"] == "file":
-                secret = dump(snapshot, path).decode()
-                secret_name = path.removeprefix(f"{SECRET_PATH}{SERVICE}_")
-                secrets[secret_name] = secret
-        binary_output = json.dumps(secrets).encode()
-        tarinfo = tarfile.TarInfo(name=filename)
-        tarinfo.size = len(binary_output)
-        file_dumps.append((binary_output, tarinfo))
-    with tarfile.open("/tmp/backup.tar.gz", "w:gz") as tar:
-        print(f"Writing files to /tmp/backup.tar.gz...")
-        for binary_output, tarinfo in file_dumps:
-            tar.addfile(tarinfo, fileobj=io.BytesIO(binary_output))
-    size = get_formatted_size("/tmp/backup.tar.gz")
-    print(f"Backup has been written to /tmp/backup.tar.gz with a size of {size}")
+    rmtree(DOWNLOAD_TMP_PATH, ignore_errors=True)
+    os.makedirs(DOWNLOAD_TMP_PATH)
+    restore_dir = os.path.join(DOWNLOAD_TMP_PATH, "restore")
+    try:
+        if path:
+            path = path.removesuffix("/")
+            files = list_files(snapshot, path)
+            filetype = [f.get("type") for f in files if f.get("path") == path][0]
+            filename = Path(path).name
+            print(f"Restoring {path} from snapshot '{snapshot}'")
+            restic_restore(snapshot_id=snapshot, include=[path], target_dir=restore_dir)
+            if filetype == "dir":
+                filename = filename + ".tar"
+                create_tar(restore_dir, path, os.path.join(DOWNLOAD_TMP_PATH, filename))
+            else:
+                os.replace(
+                    f"{restore_dir}{path}", os.path.join(DOWNLOAD_TMP_PATH, filename)
+                )
+            rmtree(restore_dir)
+            file_dumps.append(filename)
+        if volumes:
+            if SERVICE == "ALL":
+                logger.error("Please specify '--host' when using '--volumes'")
+                exit(1)
+            files = list_files(snapshot, VOLUME_PATH)
+            for f in files[1:]:
+                path = f["path"]
+                if Path(path).name.startswith(SERVICE) and f["type"] == "dir":
+                    filename = f"{Path(path).name}.tar"
+                    print(f"Restoring {path} from snapshot '{snapshot}'")
+                    restic_restore(snapshot_id=snapshot, include=[path], target_dir=restore_dir)
+                    create_tar(restore_dir, path, os.path.join(DOWNLOAD_TMP_PATH, filename))
+                    rmtree(restore_dir)
+                    file_dumps.append(filename)
+        if secrets:
+            if SERVICE == "ALL":
+                logger.error("Please specify '--host' when using '--secrets'")
+                exit(1)
+            filename = f"{SERVICE}.json"
+            print(f"Restoring {SECRET_PATH} from snapshot '{snapshot}'")
+            restic_restore(
+                snapshot_id=snapshot,
+                include=[SECRET_PATH.removesuffix("/")],
+                target_dir=restore_dir,
+            )
+            secrets = read_secrets(f"{restore_dir}{SECRET_PATH}")
+            rmtree(restore_dir)
+            with open(os.path.join(DOWNLOAD_TMP_PATH, filename), "w") as file:
+                json.dump(secrets, file)
+            file_dumps.append(filename)
+        print(f"Writing files to {BACKUP_FILE}...")
+        with tarfile.open(BACKUP_FILE, "w:gz") as tar:
+            for filename in file_dumps:
+                tar.add(os.path.join(DOWNLOAD_TMP_PATH, filename), arcname=filename)
+    finally:
+        rmtree(DOWNLOAD_TMP_PATH, ignore_errors=True)
+    size = get_formatted_size(BACKUP_FILE)
+    print(f"Backup has been written to {BACKUP_FILE} with a size of {size}")
+
+
+def create_tar(restore_dir, path, tar_file):
+    # The archive contains the children of the given directory,
+    # named by their full path without the leading slash.
+    with tarfile.open(tar_file, "w") as tar:
+        for child in sorted(os.listdir(f"{restore_dir}{path}")):
+            tar.add(
+                f"{restore_dir}{path}/{child}",
+                arcname=f"{path}/{child}".removeprefix("/"),
+            )
+
+
+def read_secrets(secret_dir):
+    secrets = {}
+    for name in sorted(os.listdir(secret_dir)):
+        secret_path = os.path.join(secret_dir, name)
+        if name.startswith(SERVICE) and os.path.isfile(secret_path):
+            with open(secret_path) as file:
+                secrets[name.removeprefix(f"{SERVICE}_")] = file.read()
+    return secrets
 
 
 def get_formatted_size(file_path):
@@ -632,18 +665,6 @@ def get_formatted_size(file_path):
     return f"{round(file_size, 3)} {units[-1]}"
 
 
-def dump(snapshot, path):
-    cmd = restic.cat.base_command() + ["dump"]
-    cmd = cmd + [snapshot, path]
-    print(f"Dumping {path} from snapshot '{snapshot}'")
-    output = subprocess.run(cmd, capture_output=True)
-    if output.returncode:
-        logger.error(
-            f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}"
-        )
-        exit(1)
-    return output.stdout
-
 
 if __name__ == "__main__":
     cli()

compose.exporter.yml

@@ -1,48 +0,0 @@
----
-version: "3.8"
-
-services:
-  restic-exporter:
-    image: ngosang/restic-exporter:2.0.2
-    environment:
-      - TZ=${TIMEZONE}
-      - RESTIC_REPOSITORY
-      - RESTIC_PASSWORD_FILE=/run/secrets/restic_password
-      - REFRESH_INTERVAL
-      - METRICS_DOMAIN
-    volumes:
-      - data:/data
-      - cache:/root/.cache/restic
-    secrets:
-      - restic_password
-    configs:
-      - source: entrypoint
-        target: /entrypoint.sh
-        mode: 666
-    entrypoint: /entrypoint.sh
-    command: /usr/local/bin/python -u /app/exporter.py
-    networks:
-      - proxy
-    deploy:
-      labels:
-        - "traefik.enable=true"
-        - "traefik.http.services.${STACK_NAME}-metrics.loadbalancer.server.port=8001"
-        - "traefik.http.routers.${STACK_NAME}-metrics.rule=Host(`${METRICS_DOMAIN}`)"
-        - "traefik.http.routers.${STACK_NAME}-metrics.entrypoints=web-secure"
-        - "traefik.http.routers.${STACK_NAME}-metrics.tls=true"
-        - "traefik.http.routers.${STACK_NAME}-metrics.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "traefik.http.routers.${STACK_NAME}-metrics.middlewares=basicauth@file"
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8001/metrics"]
-      interval: 1m
-      timeout: 10s
-      retries: 60
-      start_period: 1m
-
-volumes:
-  cache:
-  data:
-
-networks:
-  proxy:
-    external: true

compose.ssh.yml

@@ -1,34 +1,16 @@
 ---
 version: "3.8"
-
-x-ssh_env: &ssh_env
-  - SSH_KEY_FILE=/run/secrets/ssh_key
-  - SSH_HOST_KEY
-
-x-ssh_secrets: &ssh_secrets
-  - source: ssh_key
-    mode: 0400
-
-x-ssh_configs: &ssh_configs
-  - source: ssh_config
-    target: /root/.ssh/config
-
 services:
   app:
     environment:
-      *ssh_env
+      - SSH_KEY_FILE=/run/secrets/ssh_key
+      - SSH_HOST_KEY
     secrets:
-      *ssh_secrets
+      - source: ssh_key
+        mode: 0400
     configs:
-      *ssh_configs
-
-  restic-exporter:
-    environment:
-      *ssh_env
-    secrets:
-      *ssh_secrets
-    configs:
-      *ssh_configs
+      - source: ssh_config
+        target: /root/.ssh/config
 
 secrets:
   ssh_key:

compose.swarm-cronjob.yml

@@ -7,9 +7,7 @@ services:
       replicas: 0
       labels:
         - "swarm.cronjob.enable=true"
-        # Note(3wc): every 5m, testing
-        - "swarm.cronjob.schedule=*/5 * * * *"
-        # Note(3wc): blank label to be picked up by `abra recipe sync`
+        - "swarm.cronjob.schedule=${CRON_SCHEDULE}"
       restart_policy:
         condition: none
     entrypoint: [ "/usr/bin/backup.sh" ]

compose.yml

@@ -2,7 +2,7 @@
 version: "3.8"
 services:
   app:
-    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.3.0-beta
+    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.4.0
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
@@ -23,8 +23,8 @@ services:
         mode: 666
     deploy:
       labels:
-        - coop-cloud.${STACK_NAME}.version=2.3.0+2.3.0-beta
-        - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300}
+        - coop-cloud.${STACK_NAME}.version=2.4.0+2.4.0
+        - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT}
         - coop-cloud.backupbot.enabled=true
     #entrypoint: ['tail', '-f','/dev/null']
     healthcheck:

cronjob.sh

@@ -4,7 +4,7 @@ set -e
 
 CURL_OPTS="-s"
 # Check for basic auth
-if [ -n "$(cat /run/secrets/push_basicauth)" ]
+if [ -s /run/secrets/push_basicauth ]
 then
     CURL_OPTS="$CURL_OPTS -u $(cat /run/secrets/push_basicauth)"
 fi
@@ -32,7 +32,8 @@ else
 fi
 
 eval "$push_start_notification"
-if [ "$(backup --machine-logs create  2>&1 | tee /tmp/backup.log && (grep -q 'backup finished' /tmp/backup.log))" ]
+backup --machine-logs create 2>&1 | tee /tmp/backup.log
+if grep -q 'backup finished' /tmp/backup.log
 then
     eval "$push_success_notification"
 else

entrypoint.sh

@@ -2,25 +2,14 @@
 
 set -e
 
-echo "executing entrypoint.sh..."
-
 if [ -n "$SSH_HOST_KEY" ]
 then
-    echo "setting ssh known hosts"
     echo "$SSH_HOST_KEY" > /root/.ssh/known_hosts
 fi
 
-if [ -n "$CRON_SCHEDULE" ]
-then
-    echo "setting up cronjob..."
-    cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
+cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
 
-    echo "$cron_schedule /cronjob.sh" | crontab -
-    crontab -l
+echo "$cron_schedule /cronjob.sh" | crontab -
+crontab -l
 
-    crond -f -d8 -L /dev/stdout
-else
-    # startup for exporter
-    apk --no-cache add curl
-    exec /sbin/tini -- "$@"
-fi
+crond -f -d8 -L /dev/stdout