chore: publish 2.5.2+2.5.0 release

Reviewed-on: coop-cloud/backup-bot-two#99

.drone.yml

@@ -5,9 +5,10 @@ steps:
   - name: publish image
     image: plugins/docker
     settings:
-      username: abra-bot
+      username:
+        from_secret: release_user
       password:
-        from_secret: git_coopcloud_tech_token_abra_bot
+        from_secret: release_password
       repo: git.coopcloud.tech/coop-cloud/backup-bot-two
       tags: ${DRONE_SEMVER_BUILD}
       registry: git.coopcloud.tech

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker:29.4.3-dind
+FROM docker:29.9.2-dind
 
 RUN apk add --upgrade --no-cache restic bash python3 py3-pip py3-click py3-docker-py py3-json-logger curl
 

README.md

@@ -185,7 +185,7 @@ Enable basic auth in the env file, by uncommenting the following line:
 
 `abra app run <backupbot_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>`
 
-extract the <path> from the output of the first command and use in the second.
+extract the `path` from the output of the first command and use in the second.
 
 `abra app cp <backupbot_name> app:<path> .`
 

backupbot.py

@@ -9,7 +9,6 @@ import logging
 import docker
 import restic
 import tarfile
-import io
 from pythonjsonlogger import jsonlogger
 from datetime import datetime, timezone
 from restic.errors import ResticFailedError
@@ -19,6 +18,8 @@ from shutil import copyfile, rmtree
 VOLUME_PATH = "/var/lib/docker/volumes/"
 SECRET_PATH = "/secrets/"
 SERVICE = "ALL"
+BACKUP_FILE = "/tmp/backup.tar.gz"
+DOWNLOAD_TMP_PATH = "/tmp/backup-download"
 
 logger = logging.getLogger("backupbot")
 logging.addLevelName(55, "SUMMARY")
@@ -224,7 +225,7 @@ def restore_path(snapshot_id, target, noninteractive, paths):
 
 
 def restic_restore(snapshot_id, include=[], target_dir=None):
-    cmd = restic.cat.base_command() + ["restore", snapshot_id]
+    cmd = restic.cat.base_command() + ["restore", snapshot_id, "--delete"]
     for path in include:
         cmd.extend(["--include", path])
     if target_dir:
@@ -573,53 +574,85 @@ def download(snapshot, path, volumes, secrets):
         snapshot = latest_snapshot[0]["short_id"]
     if not any([path, volumes, secrets]):
         volumes = secrets = True
-    if path:
-        path = path.removesuffix("/")
-        binary_output = dump(snapshot, path)
-        files = list_files(snapshot, path)
-        filetype = [f.get("type") for f in files if f.get("path") == path][0]
-        filename = Path(path).name
-        if filetype == "dir":
-            filename = filename + ".tar"
-        tarinfo = tarfile.TarInfo(name=filename)
-        tarinfo.size = len(binary_output)
-        file_dumps.append((binary_output, tarinfo))
-    if volumes:
-        if SERVICE == "ALL":
-            logger.error("Please specify '--host' when using '--volumes'")
-            exit(1)
-        files = list_files(snapshot, VOLUME_PATH)
-        for f in files[1:]:
-            path = f["path"]
-            if Path(path).name.startswith(SERVICE) and f["type"] == "dir":
-                binary_output = dump(snapshot, path)
-                filename = f"{Path(path).name}.tar"
-                tarinfo = tarfile.TarInfo(name=filename)
-                tarinfo.size = len(binary_output)
-                file_dumps.append((binary_output, tarinfo))
-    if secrets:
-        if SERVICE == "ALL":
-            logger.error("Please specify '--host' when using '--secrets'")
-            exit(1)
-        filename = f"{SERVICE}.json"
-        files = list_files(snapshot, SECRET_PATH)
-        secrets = {}
-        for f in files[1:]:
-            path = f["path"]
-            if Path(path).name.startswith(SERVICE) and f["type"] == "file":
-                secret = dump(snapshot, path).decode()
-                secret_name = path.removeprefix(f"{SECRET_PATH}{SERVICE}_")
-                secrets[secret_name] = secret
-        binary_output = json.dumps(secrets).encode()
-        tarinfo = tarfile.TarInfo(name=filename)
-        tarinfo.size = len(binary_output)
-        file_dumps.append((binary_output, tarinfo))
-    with tarfile.open("/tmp/backup.tar.gz", "w:gz") as tar:
-        print(f"Writing files to /tmp/backup.tar.gz...")
-        for binary_output, tarinfo in file_dumps:
-            tar.addfile(tarinfo, fileobj=io.BytesIO(binary_output))
-    size = get_formatted_size("/tmp/backup.tar.gz")
-    print(f"Backup has been written to /tmp/backup.tar.gz with a size of {size}")
+    rmtree(DOWNLOAD_TMP_PATH, ignore_errors=True)
+    os.makedirs(DOWNLOAD_TMP_PATH)
+    restore_dir = os.path.join(DOWNLOAD_TMP_PATH, "restore")
+    try:
+        if path:
+            path = path.removesuffix("/")
+            files = list_files(snapshot, path)
+            filetype = [f.get("type") for f in files if f.get("path") == path][0]
+            filename = Path(path).name
+            print(f"Restoring {path} from snapshot '{snapshot}'")
+            restic_restore(snapshot_id=snapshot, include=[path], target_dir=restore_dir)
+            if filetype == "dir":
+                filename = filename + ".tar"
+                create_tar(restore_dir, path, os.path.join(DOWNLOAD_TMP_PATH, filename))
+            else:
+                os.replace(
+                    f"{restore_dir}{path}", os.path.join(DOWNLOAD_TMP_PATH, filename)
+                )
+            rmtree(restore_dir)
+            file_dumps.append(filename)
+        if volumes:
+            if SERVICE == "ALL":
+                logger.error("Please specify '--host' when using '--volumes'")
+                exit(1)
+            files = list_files(snapshot, VOLUME_PATH)
+            for f in files[1:]:
+                path = f["path"]
+                if Path(path).name.startswith(SERVICE) and f["type"] == "dir":
+                    filename = f"{Path(path).name}.tar"
+                    print(f"Restoring {path} from snapshot '{snapshot}'")
+                    restic_restore(snapshot_id=snapshot, include=[path], target_dir=restore_dir)
+                    create_tar(restore_dir, path, os.path.join(DOWNLOAD_TMP_PATH, filename))
+                    rmtree(restore_dir)
+                    file_dumps.append(filename)
+        if secrets:
+            if SERVICE == "ALL":
+                logger.error("Please specify '--host' when using '--secrets'")
+                exit(1)
+            filename = f"{SERVICE}.json"
+            print(f"Restoring {SECRET_PATH} from snapshot '{snapshot}'")
+            restic_restore(
+                snapshot_id=snapshot,
+                include=[SECRET_PATH.removesuffix("/")],
+                target_dir=restore_dir,
+            )
+            secrets = read_secrets(f"{restore_dir}{SECRET_PATH}")
+            rmtree(restore_dir)
+            with open(os.path.join(DOWNLOAD_TMP_PATH, filename), "w") as file:
+                json.dump(secrets, file)
+            file_dumps.append(filename)
+        print(f"Writing files to {BACKUP_FILE}...")
+        with tarfile.open(BACKUP_FILE, "w:gz") as tar:
+            for filename in file_dumps:
+                tar.add(os.path.join(DOWNLOAD_TMP_PATH, filename), arcname=filename)
+    finally:
+        rmtree(DOWNLOAD_TMP_PATH, ignore_errors=True)
+    size = get_formatted_size(BACKUP_FILE)
+    print(f"Backup has been written to {BACKUP_FILE} with a size of {size}")
+
+
+def create_tar(restore_dir, path, tar_file):
+    # The archive contains the children of the given directory,
+    # named by their full path without the leading slash.
+    with tarfile.open(tar_file, "w") as tar:
+        for child in sorted(os.listdir(f"{restore_dir}{path}")):
+            tar.add(
+                f"{restore_dir}{path}/{child}",
+                arcname=f"{path}/{child}".removeprefix("/"),
+            )
+
+
+def read_secrets(secret_dir):
+    secrets = {}
+    for name in sorted(os.listdir(secret_dir)):
+        secret_path = os.path.join(secret_dir, name)
+        if name.startswith(SERVICE) and os.path.isfile(secret_path):
+            with open(secret_path) as file:
+                secrets[name.removeprefix(f"{SERVICE}_")] = file.read()
+    return secrets
 
 
 def get_formatted_size(file_path):
@@ -632,18 +665,6 @@ def get_formatted_size(file_path):
     return f"{round(file_size, 3)} {units[-1]}"
 
 
-def dump(snapshot, path):
-    cmd = restic.cat.base_command() + ["dump"]
-    cmd = cmd + [snapshot, path]
-    print(f"Dumping {path} from snapshot '{snapshot}'")
-    output = subprocess.run(cmd, capture_output=True)
-    if output.returncode:
-        logger.error(
-            f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}"
-        )
-        exit(1)
-    return output.stdout
-
 
 if __name__ == "__main__":
     cli()

compose.yml

@@ -2,7 +2,7 @@
 version: "3.8"
 services:
   app:
-    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.4.0
+    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.5.0
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
@@ -23,7 +23,7 @@ services:
         mode: 666
     deploy:
       labels:
-        - coop-cloud.${STACK_NAME}.version=2.4.0+2.4.0
+        - coop-cloud.${STACK_NAME}.version=2.5.2+2.5.0
         - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT}
         - coop-cloud.backupbot.enabled=true
     #entrypoint: ['tail', '-f','/dev/null']