fix: download without buffering volumes in RAM

Fetch each path with 'restic restore' into a temp dir, instead of
buffering whole volume dumps in memory, which crashed on large volumes.
Restore is also much faster than the sequential 'restic dump'.
The archive layout is unchanged.

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker:29.3.1-dind
+FROM docker:29.4.3-dind
 
 RUN apk add --upgrade --no-cache restic bash python3 py3-pip py3-click py3-docker-py py3-json-logger curl
 

README.md

@@ -137,55 +137,57 @@ Enable basic auth in the env file, by uncommenting the following line:
 
 ## Usage
 
-Run the cronjob that creates a backup, including the push notifications and docker logging:
+### Run the cronjob that creates a backup, including the push notifications and docker logging:
+
 `abra app cmd <backupbot_name> app run_cron`
 
-Create a backup of all apps:
+### Create a backup of all apps:
 
 `abra app run <backupbot_name> app -- backup create`
 
 > The apps to backup up need to be deployed
 
-Create an individual backup:
+### Create an individual backup:
 
 `abra app run <backupbot_name> app -- backup --host <target_app_name> create`
 
-Create a backup to a local repository:
+### Create a backup to a local repository:
 
 `abra app run <backupbot_name> app -- backup create -r /backups/restic`
 
 > It is recommended to shutdown/undeploy an app before restoring the data
 
-Restore the latest snapshot of all including apps:
+### Restore the latest snapshot of all including apps:
 
 `abra app run <backupbot_name> app -- backup restore`
 
-Restore a specific snapshot of an individual app:
+### Restore a specific snapshot of an individual app:
 
 `abra app run <backupbot_name> app -- backup --host <target_app_name> restore --snapshot <snapshot_id>`
 
-Show all snapshots:
+### Show all snapshots:
 
 `abra app run <backupbot_name> app -- backup snapshots`
 
-Show all snapshots containing a specific app:
+### Show all snapshots containing a specific app:
 
 `abra app run <backupbot_name> app -- backup --host <target_app_name> snapshots`
 
-Show all files inside the latest snapshot (can be very verbose):
+### Show all files inside the latest snapshot (can be very verbose):
 
 `abra app run <backupbot_name> app -- backup ls`
 
-Show specific files inside a selected snapshot:
+### Show specific files inside a selected snapshot:
 
 `abra app run <backupbot_name> app -- backup ls --snapshot <snapshot_id> /var/lib/docker/volumes/`
 
-Download files from a snapshot:
+### Download files from a snapshot:
 
-```
-filename=$(abra app run <backupbot_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>)
-abra app cp <backupbot_name> app:$filename .
-```
+`abra app run <backupbot_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>`
+
+extract the <path> from the output of the first command and use in the second.
+
+`abra app cp <backupbot_name> app:<path> .`
 
 ## Run restic
 

backupbot.py

@@ -9,7 +9,6 @@ import logging
 import docker
 import restic
 import tarfile
-import io
 from pythonjsonlogger import jsonlogger
 from datetime import datetime, timezone
 from restic.errors import ResticFailedError
@@ -19,6 +18,8 @@ from shutil import copyfile, rmtree
 VOLUME_PATH = "/var/lib/docker/volumes/"
 SECRET_PATH = "/secrets/"
 SERVICE = "ALL"
+BACKUP_FILE = "/tmp/backup.tar.gz"
+DOWNLOAD_TMP_PATH = "/tmp/backup-download"
 
 logger = logging.getLogger("backupbot")
 logging.addLevelName(55, "SUMMARY")
@@ -573,53 +574,85 @@ def download(snapshot, path, volumes, secrets):
         snapshot = latest_snapshot[0]["short_id"]
     if not any([path, volumes, secrets]):
         volumes = secrets = True
-    if path:
-        path = path.removesuffix("/")
-        binary_output = dump(snapshot, path)
-        files = list_files(snapshot, path)
-        filetype = [f.get("type") for f in files if f.get("path") == path][0]
-        filename = Path(path).name
-        if filetype == "dir":
-            filename = filename + ".tar"
-        tarinfo = tarfile.TarInfo(name=filename)
-        tarinfo.size = len(binary_output)
-        file_dumps.append((binary_output, tarinfo))
-    if volumes:
-        if SERVICE == "ALL":
-            logger.error("Please specify '--host' when using '--volumes'")
-            exit(1)
-        files = list_files(snapshot, VOLUME_PATH)
-        for f in files[1:]:
-            path = f["path"]
-            if Path(path).name.startswith(SERVICE) and f["type"] == "dir":
-                binary_output = dump(snapshot, path)
-                filename = f"{Path(path).name}.tar"
-                tarinfo = tarfile.TarInfo(name=filename)
-                tarinfo.size = len(binary_output)
-                file_dumps.append((binary_output, tarinfo))
-    if secrets:
-        if SERVICE == "ALL":
-            logger.error("Please specify '--host' when using '--secrets'")
-            exit(1)
-        filename = f"{SERVICE}.json"
-        files = list_files(snapshot, SECRET_PATH)
-        secrets = {}
-        for f in files[1:]:
-            path = f["path"]
-            if Path(path).name.startswith(SERVICE) and f["type"] == "file":
-                secret = dump(snapshot, path).decode()
-                secret_name = path.removeprefix(f"{SECRET_PATH}{SERVICE}_")
-                secrets[secret_name] = secret
-        binary_output = json.dumps(secrets).encode()
-        tarinfo = tarfile.TarInfo(name=filename)
-        tarinfo.size = len(binary_output)
-        file_dumps.append((binary_output, tarinfo))
-    with tarfile.open("/tmp/backup.tar.gz", "w:gz") as tar:
-        print(f"Writing files to /tmp/backup.tar.gz...")
-        for binary_output, tarinfo in file_dumps:
-            tar.addfile(tarinfo, fileobj=io.BytesIO(binary_output))
-    size = get_formatted_size("/tmp/backup.tar.gz")
-    print(f"Backup has been written to /tmp/backup.tar.gz with a size of {size}")
+    rmtree(DOWNLOAD_TMP_PATH, ignore_errors=True)
+    os.makedirs(DOWNLOAD_TMP_PATH)
+    restore_dir = os.path.join(DOWNLOAD_TMP_PATH, "restore")
+    try:
+        if path:
+            path = path.removesuffix("/")
+            files = list_files(snapshot, path)
+            filetype = [f.get("type") for f in files if f.get("path") == path][0]
+            filename = Path(path).name
+            print(f"Restoring {path} from snapshot '{snapshot}'")
+            restic_restore(snapshot_id=snapshot, include=[path], target_dir=restore_dir)
+            if filetype == "dir":
+                filename = filename + ".tar"
+                create_tar(restore_dir, path, os.path.join(DOWNLOAD_TMP_PATH, filename))
+            else:
+                os.replace(
+                    f"{restore_dir}{path}", os.path.join(DOWNLOAD_TMP_PATH, filename)
+                )
+            rmtree(restore_dir)
+            file_dumps.append(filename)
+        if volumes:
+            if SERVICE == "ALL":
+                logger.error("Please specify '--host' when using '--volumes'")
+                exit(1)
+            files = list_files(snapshot, VOLUME_PATH)
+            for f in files[1:]:
+                path = f["path"]
+                if Path(path).name.startswith(SERVICE) and f["type"] == "dir":
+                    filename = f"{Path(path).name}.tar"
+                    print(f"Restoring {path} from snapshot '{snapshot}'")
+                    restic_restore(snapshot_id=snapshot, include=[path], target_dir=restore_dir)
+                    create_tar(restore_dir, path, os.path.join(DOWNLOAD_TMP_PATH, filename))
+                    rmtree(restore_dir)
+                    file_dumps.append(filename)
+        if secrets:
+            if SERVICE == "ALL":
+                logger.error("Please specify '--host' when using '--secrets'")
+                exit(1)
+            filename = f"{SERVICE}.json"
+            print(f"Restoring {SECRET_PATH} from snapshot '{snapshot}'")
+            restic_restore(
+                snapshot_id=snapshot,
+                include=[SECRET_PATH.removesuffix("/")],
+                target_dir=restore_dir,
+            )
+            secrets = read_secrets(f"{restore_dir}{SECRET_PATH}")
+            rmtree(restore_dir)
+            with open(os.path.join(DOWNLOAD_TMP_PATH, filename), "w") as file:
+                json.dump(secrets, file)
+            file_dumps.append(filename)
+        print(f"Writing files to {BACKUP_FILE}...")
+        with tarfile.open(BACKUP_FILE, "w:gz") as tar:
+            for filename in file_dumps:
+                tar.add(os.path.join(DOWNLOAD_TMP_PATH, filename), arcname=filename)
+    finally:
+        rmtree(DOWNLOAD_TMP_PATH, ignore_errors=True)
+    size = get_formatted_size(BACKUP_FILE)
+    print(f"Backup has been written to {BACKUP_FILE} with a size of {size}")
+
+
+def create_tar(restore_dir, path, tar_file):
+    # The archive contains the children of the given directory,
+    # named by their full path without the leading slash.
+    with tarfile.open(tar_file, "w") as tar:
+        for child in sorted(os.listdir(f"{restore_dir}{path}")):
+            tar.add(
+                f"{restore_dir}{path}/{child}",
+                arcname=f"{path}/{child}".removeprefix("/"),
+            )
+
+
+def read_secrets(secret_dir):
+    secrets = {}
+    for name in sorted(os.listdir(secret_dir)):
+        secret_path = os.path.join(secret_dir, name)
+        if name.startswith(SERVICE) and os.path.isfile(secret_path):
+            with open(secret_path) as file:
+                secrets[name.removeprefix(f"{SERVICE}_")] = file.read()
+    return secrets
 
 
 def get_formatted_size(file_path):
@@ -632,18 +665,6 @@ def get_formatted_size(file_path):
     return f"{round(file_size, 3)} {units[-1]}"
 
 
-def dump(snapshot, path):
-    cmd = restic.cat.base_command() + ["dump"]
-    cmd = cmd + [snapshot, path]
-    print(f"Dumping {path} from snapshot '{snapshot}'")
-    output = subprocess.run(cmd, capture_output=True)
-    if output.returncode:
-        logger.error(
-            f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}"
-        )
-        exit(1)
-    return output.stdout
-
 
 if __name__ == "__main__":
     cli()

compose.swarm-cronjob.yml

@@ -7,9 +7,7 @@ services:
       replicas: 0
       labels:
         - "swarm.cronjob.enable=true"
-        # Note(3wc): every 5m, testing
-        - "swarm.cronjob.schedule=*/5 * * * *"
-        # Note(3wc): blank label to be picked up by `abra recipe sync`
+        - "swarm.cronjob.schedule=${CRON_SCHEDULE}"
       restart_policy:
         condition: none
     entrypoint: [ "/usr/bin/backup.sh" ]