Always backup all apps to increase restic performance

This reverts commit 79b7a01dda.

.env.sample

@@ -4,7 +4,7 @@ SECRET_RESTIC_PASSWORD_VERSION=v1
 
 COMPOSE_FILE=compose.yml
 
-RESTIC_REPO=/backups/restic
+RESTIC_REPOSITORY=/backups/restic
 
 CRON_SCHEDULE='30 3 * * *'
 
@@ -22,8 +22,8 @@ CRON_SCHEDULE='30 3 * * *'
 #COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml"
 
 # Secret restic repository
-# use a secret to store the RESTIC_REPO if the repository location contains a secret value
+# use a secret to store the RESTIC_REPOSITORY if the repository location contains a secret value
 # i.E rest:https://user:SECRET_PASSWORD@host:8000/
-# it overwrites the RESTIC_REPO variable
+# it overwrites the RESTIC_REPOSITORY variable
 #SECRET_RESTIC_REPO_VERSION=v1
 #COMPOSE_FILE="$COMPOSE_FILE:compose.secret.yml"

README.md

@@ -45,13 +45,13 @@ Backupbot II tries to help, by
 
 Per default Backupbot stores the backups locally in the repository `/backups/restic`, which is accessible as volume at `/var/lib/docker/volumes/<app_name>_backups/_data/restic/`
 
-The backup location can be changed using the `RESTIC_REPO` env variable.
+The backup location can be changed using the `RESTIC_REPOSITORY` env variable.
 
 ### S3 Storage
 
 To use S3 storage as backup location set the following envs:
 ```
-RESTIC_REPO=s3:<S3-SERVICE-URL>/<BUCKET-NAME>
+RESTIC_REPOSITORY=s3:<S3-SERVICE-URL>/<BUCKET-NAME>
 SECRET_AWS_SECRET_ACCESS_KEY_VERSION=v1
 AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
 COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml"
@@ -67,7 +67,7 @@ See [restic s3 docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new
 
 To use SFTP storage as backup location set the following envs:
 ```
-RESTIC_REPO=sftp:user@host:/restic-repo-path
+RESTIC_REPOSITORY=sftp:user@host:/restic-repo-path
 SECRET_SSH_KEY_VERSION=v1
 SSH_HOST_KEY="hostname ssh-rsa AAAAB3...
 COMPOSE_FILE="$COMPOSE_FILE:compose.ssh.yml"
@@ -85,7 +85,7 @@ abra app secret insert <app_name> ssh_key v1 """$(cat backupkey)
 
 ### Restic REST server Storage
 
-You can simply set the `RESTIC_REPO` variable to your REST server URL `rest:http://host:8000/`.
+You can simply set the `RESTIC_REPOSITORY` variable to your REST server URL `rest:http://host:8000/`.
 If you access the REST server with a password `rest:https://user:pass@host:8000/` you should hide the whole URL containing the password inside a secret.
 Uncomment these lines:
 ```
@@ -96,7 +96,7 @@ Add your REST server url as secret:
 ```
 `abra app secret insert <app_name> restic_repo v1 "rest:https://user:pass@host:8000/"`
 ```
-The secret will overwrite the `RESTIC_REPO` variable.
+The secret will overwrite the `RESTIC_REPOSITORY` variable.
 
 
 See [restic REST docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) for more information.
@@ -151,6 +151,14 @@ filename=$(abra app run <app_name> app -- backup download --snapshot <snapshot_i
 abra app cp <app_name> app:$filename .
 ```
 
+## Run restic
+
+```
+abra app run <app_name> app bash
+export AWS_SECRET_ACCESS_KEY=$(cat $AWS_SECRET_ACCESS_KEY_FILE)
+export RESTIC_PASSWORD=$(cat $RESTIC_PASSWORD_FILE)
+restic snapshots
+```
 
 ## Recipe Configuration
 

backupbot.py

@@ -7,6 +7,8 @@ import subprocess
 import logging
 import docker
 import restic
+import tarfile
+import io
 from datetime import datetime, timezone
 from restic.errors import ResticFailedError
 from pathlib import Path
@@ -21,13 +23,13 @@ SERVICE = None
 @click.group()
 @click.option('-l', '--log', 'loglevel')
 @click.option('service', '--host', '-h', envvar='SERVICE')
-@click.option('repository', '--repo', '-r', envvar='RESTIC_REPO', required=True)
+@click.option('repository', '--repo', '-r', envvar='RESTIC_REPOSITORY', required=True)
 def cli(loglevel, service, repository):
     global SERVICE
     if service:
         SERVICE = service.replace('.', '_')
     if repository:
-        os.environ['RESTIC_REPO'] = repository
+        os.environ['RESTIC_REPOSITORY'] = repository
     if loglevel:
         numeric_level = getattr(logging, loglevel.upper(), None)
         if not isinstance(numeric_level, int):
@@ -38,7 +40,7 @@ def cli(loglevel, service, repository):
 
 
 def init_repo():
-    repo = os.environ['RESTIC_REPO']
+    repo = os.environ['RESTIC_REPOSITORY']
     logging.debug(f"set restic repository location: {repo}")
     restic.repository = repo
     restic.password_file = '/var/run/secrets/restic_password'
@@ -57,7 +59,7 @@ def export_secrets():
         if env.endswith('FILE') and not "COMPOSE_FILE" in env:
             logging.debug(f"exported secret: {env}")
             with open(os.environ[env]) as file:
-                secret =  file.read()
+                secret = file.read()
                 os.environ[env.removesuffix('_FILE')] = secret
                 # logging.debug(f"Read secret value: {secret}")
 
@@ -85,23 +87,26 @@ def get_backup_cmds():
         labels = s.attrs['Spec']['Labels']
         if (backup := labels.get('backupbot.backup')) and bool(backup):
             stack_name = labels['com.docker.stack.namespace']
-            if SERVICE and SERVICE != stack_name:
-                continue
+            # Remove this lines to backup only a specific service
+            # This will unfortenately decrease restice performance
+            # if SERVICE and SERVICE != stack_name:
+            #     continue
             backup_apps.add(stack_name)
-            container = container_by_service.get(s.name)
-            if not container:
+            backup_paths = backup_paths.union(
+                Path(VOLUME_PATH).glob(f"{stack_name}_*"))
+            if not (container:= container_by_service.get(s.name)):
                 logging.error(
                     f"Container {s.name} is not running, hooks can not be executed")
+                continue
             if prehook := labels.get('backupbot.backup.pre-hook'):
                 pre_commands[container] = prehook
             if posthook := labels.get('backupbot.backup.post-hook'):
                 post_commands[container] = posthook
-            backup_paths = backup_paths.union(
-                Path(VOLUME_PATH).glob(f"{stack_name}_*"))
     return pre_commands, post_commands, list(backup_paths), list(backup_apps)
 
 
 def copy_secrets(apps):
+    #TODO: check if it is deployed
     rmtree(SECRET_PATH, ignore_errors=True)
     os.mkdir(SECRET_PATH)
     client = docker.from_env()
@@ -119,6 +124,9 @@ def copy_secrets(apps):
             container_id = container_by_service[s.name].id
             for sec in app_secs:
                 src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}'
+                if not Path(src).exists():
+                    logging.error(f"For the secret {sec['SecretName']} the file {src} does not exist for {s.name}")
+                    continue
                 dst = SECRET_PATH + sec['SecretName']
                 copyfile(src, dst)
 
@@ -127,13 +135,16 @@ def run_commands(commands):
     for container, command in commands.items():
         if not command:
             continue
+        # Remove bash/sh wrapping
+        command = command.removeprefix('bash -c').removeprefix('sh -c')
+        # Remove quotes surrounding the command
+        if (len(command) >= 2 and command[0] == command[-1] and (command[0] == "'" or command[0] == '"')):
+            command[1:-1]
         # Use bash's pipefail to return exit codes inside a pipe to prevent silent failure
-        command = command.removeprefix('bash -c \'').removeprefix('sh -c \'')
-        command = command.removesuffix('\'')
         command = f"bash -c 'set -o pipefail;{command}'"
-        result = container.exec_run(command)
-        logging.info(f"run command in {container.name}")
+        logging.info(f"run command in {container.name}:")
         logging.info(command)
+        result = container.exec_run(command)
         if result.exit_code:
             logging.error(
                 f"Failed to run command {command} in {container.name}: {result.output.decode()}")
@@ -142,9 +153,14 @@ def run_commands(commands):
 
 
 def backup_volumes(backup_paths, apps, dry_run=False):
-    result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
-    print(result)
-    logging.info(result)
+    try:
+        result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
+        print(result)
+        logging.info(result)
+    except ResticFailedError as error:
+        logging.error(f"Backup failed for {apps}. Could not Backup these paths: {backup_paths}")
+        logging.error(error)
+        exit(1)
 
 
 @cli.command()
@@ -163,9 +179,11 @@ def restore(snapshot, target, noninteractive):
     if not noninteractive:
         snapshot_date = datetime.fromisoformat(snapshots[0]['time'])
         delta = datetime.now(tz=timezone.utc) - snapshot_date
-        print(f"You are going to restore Snapshot {snapshot} of {service_paths} at {target}")
+        print(
+            f"You are going to restore Snapshot {snapshot} of {service_paths} at {target}")
         print(f"This snapshot is {delta} old")
-        print(f"THIS COMMAND WILL IRREVERSIBLY OVERWRITES {target}{service_paths.removeprefix('/')}")
+        print(
+            f"THIS COMMAND WILL IRREVERSIBLY OVERWRITES {target}{service_paths.removeprefix('/')}")
         prompt = input("Type YES (uppercase) to continue: ")
         if prompt != 'YES':
             logging.error("Restore aborted")
@@ -179,9 +197,16 @@ def restore(snapshot, target, noninteractive):
 @cli.command()
 def snapshots():
     snapshots = restic.snapshots()
+    no_snapshots = True
     for snap in snapshots:
         if not SERVICE or (tags := snap.get('tags')) and SERVICE in tags:
             print(snap['time'], snap['id'])
+            no_snapshots = False
+    if no_snapshots:
+        err_msg = "No Snapshots found"
+        if SERVICE:
+            err_msg += f' for app {SERVICE}'
+        logging.warning(err_msg)
 
 
 @cli.command()
@@ -201,7 +226,17 @@ def list_files(snapshot, path):
     cmd.append(snapshot)
     if path:
         cmd.append(path)
-    output = restic.internal.command_executor.execute(cmd)
+    try:
+        output = restic.internal.command_executor.execute(cmd)
+    except ResticFailedError as error:
+        if 'no snapshot found' in str(error):
+            err_msg = f'There is no snapshot {snapshot}'
+            if SERVICE:
+                err_msg += f'for the app {SERVICE}'
+            logging.error(err_msg)
+            exit(1)
+        else:
+            raise error
     output = output.replace('}\n{', '}|{')
     results = list(map(json.loads, output.split('|')))
     return results
@@ -213,59 +248,78 @@ def list_files(snapshot, path):
 @click.option('volumes', '--volumes', '-v', is_flag=True)
 @click.option('secrets', '--secrets', '-c', is_flag=True)
 def download(snapshot, path, volumes, secrets):
-    if sum(map(bool, [path, volumes, secrets])) != 1:
-        logging.error("Please specify exactly one of '--path', '--volumes', '--secrets'")
-        exit(1)
+    file_dumps = []
+    if not any([path, volumes, secrets]):
+        volumes = secrets = True
     if path:
         path = path.removesuffix('/')
+        binary_output = dump(snapshot, path)
         files = list_files(snapshot, path)
         filetype = [f.get('type') for f in files if f.get('path') == path][0]
-        filename = "/tmp/" + Path(path).name
+        filename = Path(path).name
         if filetype == 'dir':
             filename = filename + ".tar"
-        output = dump(snapshot, path)
-        with open(filename, "wb") as file:
-            file.write(output)
-        print(filename)
-    elif volumes:
+        tarinfo = tarfile.TarInfo(name=filename)
+        tarinfo.size = len(binary_output)
+        file_dumps.append((binary_output, tarinfo))
+    if volumes:
         if not SERVICE:
             logging.error("Please specify '--host' when using '--volumes'")
             exit(1)
-        filename = f"/tmp/{SERVICE}.tar"
         files = list_files(snapshot, VOLUME_PATH)
         for f in files[1:]:
-            path = f[ 'path' ]
-            if SERVICE in path and f['type'] == 'dir':
-                content = dump(snapshot, path)
-                # Concatenate tar files (extract with tar -xi)
-                with open(filename, "ab") as file:
-                    file.write(content)
-    elif secrets:
+            path = f['path']
+            if Path(path).name.startswith(SERVICE) and f['type'] == 'dir':
+                binary_output = dump(snapshot, path)
+                filename = f"{Path(path).name}.tar"
+                tarinfo = tarfile.TarInfo(name=filename)
+                tarinfo.size = len(binary_output)
+                file_dumps.append((binary_output, tarinfo))
+    if secrets:
         if not SERVICE:
             logging.error("Please specify '--host' when using '--secrets'")
             exit(1)
-        filename = f"/tmp/SECRETS_{SERVICE}.json"
+        filename = f"{SERVICE}.json"
         files = list_files(snapshot, SECRET_PATH)
         secrets = {}
         for f in files[1:]:
-            path = f[ 'path' ]
-            if SERVICE in path and f['type'] == 'file':
+            path = f['path']
+            if Path(path).name.startswith(SERVICE) and f['type'] == 'file':
                 secret = dump(snapshot, path).decode()
                 secret_name = path.removeprefix(f'{SECRET_PATH}{SERVICE}_')
                 secrets[secret_name] = secret
-        with open(filename, "w") as file:
-            json.dump(secrets, file)
-        print(filename)
+        binary_output = json.dumps(secrets).encode()
+        tarinfo = tarfile.TarInfo(name=filename)
+        tarinfo.size = len(binary_output)
+        file_dumps.append((binary_output, tarinfo))
+    with tarfile.open('/tmp/backup.tar.gz', "w:gz") as tar:
+        print(f"Writing files to /tmp/backup.tar.gz...")
+        for binary_output, tarinfo in file_dumps:
+            tar.addfile(tarinfo, fileobj=io.BytesIO(binary_output))
+    size = get_formatted_size('/tmp/backup.tar.gz')
+    print(f"Backup has been written to /tmp/backup.tar.gz with a size of {size}")
+
+
+def get_formatted_size(file_path):
+    file_size = os.path.getsize(file_path)
+    units = ['Bytes', 'KB', 'MB', 'GB', 'TB']
+    for unit in units:
+        if file_size < 1024:
+            return f"{round(file_size, 3)} {unit}"
+        file_size /= 1024
+    return f"{round(file_size, 3)} {units[-1]}"
+
 
 def dump(snapshot, path):
     cmd = restic.cat.base_command() + ['dump']
     if SERVICE:
         cmd = cmd + ['--tag', SERVICE]
-    cmd = cmd +[snapshot, path]
-    logging.debug(f"Dumping {path} from snapshot '{snapshot}'")
+    cmd = cmd + [snapshot, path]
+    print(f"Dumping {path} from snapshot '{snapshot}'")
     output = subprocess.run(cmd, capture_output=True)
     if output.returncode:
-        logging.error(f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}")
+        logging.error(
+            f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}")
         exit(1)
     return output.stdout
 

compose.secret.yml

@@ -3,7 +3,7 @@ version: "3.8"
 services:
   app:
     environment:
-      - RESTIC_REPO_FILE=/run/secrets/restic_repo
+      - RESTIC_REPOSITORY_FILE=/run/secrets/restic_repo
     secrets:
       - restic_repo
 

compose.yml

@@ -2,7 +2,7 @@
 version: "3.8"
 services:
   app:
-    image: docker:24.0.2-dind
+    image: docker:24.0.7-dind
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
@@ -10,7 +10,7 @@ services:
       - backups:/backups
     environment:
       - CRON_SCHEDULE
-      - RESTIC_REPO
+      - RESTIC_REPOSITORY
       - RESTIC_PASSWORD_FILE=/run/secrets/restic_password
     secrets:
       - restic_password

release/1.0.0+latest

@@ -1,3 +1,3 @@
-Breaking Change: the variables `SERVER_NAME` and `RESTIC_HOST` are merged into `RESTIC_REPO`. The format can be looked up here: https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html
+Breaking Change: the variables `SERVER_NAME` and `RESTIC_HOST` are merged into `RESTIC_REPOSITORY`. The format can be looked up here: https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html
 ssh/sftp: `sftp:user@host:/repo-path`
 S3:  `s3:https://s3.example.com/bucket_name`