chore: publish 2.1.0+2.2.0-beta release

without path extraction

.drone.yml

@@ -9,9 +9,9 @@ steps:
       password:
         from_secret: git_coopcloud_tech_token_3wc
       repo: git.coopcloud.tech/coop-cloud/backup-bot-two
-      tags: 2.0.0
+      tags: ${DRONE_SEMVER_BUILD}
       registry: git.coopcloud.tech
     when:
       event:
-        exclude:
-          - pull_request
+        include:
+          - tag

README.md

@@ -95,7 +95,7 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.secret.yml"
 ```
 Add your REST server url as secret:
 ```
-`abra app secret insert <backupbot_name> restic_repo v1 "rest:https://user:pass@host:8000/"`
+abra app secret insert <backupbot_name> restic_repo v1 "rest:https://user:pass@host:8000/"
 ```
 The secret will overwrite the `RESTIC_REPOSITORY` variable.
 
@@ -196,3 +196,20 @@ services:
 As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly.
 
 [abra]: https://git.autonomic.zone/autonomic-cooperative/abra
+
+## Backupbot Development
+
+1. Copy modified backupbot.py into the container:
+
+```
+cp backupbot.py /tmp/backupbot.py; git stash; abra app cp <backupbot_name> /tmp/backupbot.py app:/usr/bin/backupbot.py; git checkout main; git stash pop
+```
+
+2. Testing stuff with the python interpreter inside the container:
+
+```
+abra app run <backupbot_name> app bash
+cd /usr/bin/
+python
+from backupbot import *
+```

backupbot.py

@@ -18,7 +18,7 @@ from shutil import copyfile, rmtree
 
 VOLUME_PATH = "/var/lib/docker/volumes/"
 SECRET_PATH = '/secrets/'
-SERVICE = None
+SERVICE = 'ALL'
 
 logger = logging.getLogger("backupbot")
 logging.addLevelName(55, 'SUMMARY')
@@ -97,44 +97,213 @@ def export_secrets():
 @cli.command()
 @click.option('retries', '--retries', '-r', envvar='RETRIES', default=1)
 def create(retries):
-    pre_commands, post_commands, backup_paths, apps = get_backup_cmds()
-    copy_secrets(apps)
-    backup_paths.append(SECRET_PATH)
+    app_settings = parse_backup_labels()
+    pre_commands, post_commands, backup_paths, apps_versions = get_backup_details(app_settings)
+    copy_secrets(apps_versions)
+    backup_paths.append(Path(SECRET_PATH))
     run_commands(pre_commands)
-    backup_volumes(backup_paths, apps, int(retries))
+    backup_volumes(backup_paths, apps_versions, int(retries))
     run_commands(post_commands)
 
 
-def get_backup_cmds():
+@cli.command()
+@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
+@click.option('target', '--target', '-t', envvar='TARGET', default='/')
+@click.option('noninteractive', '--noninteractive', envvar='NONINTERACTIVE', is_flag=True)
+@click.option('volumes', '--volumes', '-v', envvar='VOLUMES', multiple=True)
+@click.option('container', '--container', '-c', envvar='CONTAINER', multiple=True)
+@click.option('no_commands', '--no-commands', envvar='NO_COMMANDS', is_flag=True)
+def restore(snapshot, target, noninteractive, volumes, container, no_commands):
+    app_settings = parse_backup_labels('restore', container)
+    if SERVICE != 'ALL':
+        app_settings = {SERVICE: app_settings[SERVICE]}
+    pre_commands, post_commands, backup_paths, apps_versions = get_backup_details(app_settings, volumes)
+    snapshots = get_snapshots(snapshot_id=snapshot)
+    if not snapshot:
+        logger.error(f"No Snapshots with ID {snapshots} for {apps_versions.keys()} found.")
+        exit(1)
+    if not noninteractive:
+        print(f"Snapshot to restore: \t{snapshot}")
+        restore_app_versions = app_versions_from_tags(snapshots[0].get('tags'))
+        print("Apps:")
+        for app, version in restore_app_versions.items():
+            print(f"\t{app} \t {version}")
+            if apps_versions.get(app) != version:
+                print(f"WARNING!!! The running app is deployed with version {apps_versions.get(app)}")
+        print("The following volume paths will be restored:")
+        for p in backup_paths:
+            print(f'\t{p}')
+        if not no_commands:
+            print("The following commands will be executed:")
+            for container, cmd in list(pre_commands.items()) + list(post_commands.items()):
+                print(f"\t{container.labels['com.docker.swarm.service.name']}:\t{cmd}")
+        snapshot_date = datetime.fromisoformat(snapshots[0]['time'])
+        delta = datetime.now(tz=timezone.utc) - snapshot_date
+        print(f"This snapshot is {delta} old")
+        print("\nTHIS COMMAND WILL IRREVERSIBLY OVERWRITES FILES")
+        prompt = input("Type YES (uppercase) to continue: ")
+        if prompt != 'YES':
+            logger.error("Restore aborted")
+            exit(1)
+    print(f"Restoring Snapshot {snapshot} at {target}")
+    if not no_commands and pre_commands:
+        print(f"Run pre commands.")
+        run_commands(pre_commands)
+    result = restic_restore(snapshot_id=snapshot, include=backup_paths, target_dir=target)
+    if not no_commands and post_commands:
+        print(f"Run post commands.")
+        run_commands(post_commands)
+    logger.debug(result)
+
+
+def restic_restore(snapshot_id='latest', include=[], target_dir=None):
+    cmd = restic.cat.base_command() + ['restore', snapshot_id]
+    for path in include:
+        cmd.extend(['--include', path])
+    if target_dir:
+        cmd.extend(['--target', target_dir])
+    return restic.internal.command_executor.execute(cmd)
+
+
+def get_snapshots(snapshot_id=None):
+    if snapshot_id and snapshot_id != 'latest':
+        snapshots = restic.snapshots(snapshot_id=snapshot_id)
+        if not SERVICE in app_versions_from_tags(snapshots[0].get('tags')):
+            logger.error(f'Snapshot with ID {snapshot_id} does not contain {SERVICE}')
+            exit(1)
+    else:
+        snapshots = restic.snapshots()
+        snapshots = list(filter(lambda x: SERVICE in app_versions_from_tags(x.get('tags')), snapshots))
+    if snapshot_id == 'latest':
+        return snapshots[-1:]
+    else:
+        return snapshots
+
+
+def app_versions_from_tags(tags):
+    if tags:
+        app_versions = map(lambda x: x.split(':'), tags)
+        return {i[0]: i[1] if len(i) > 1 else None for i in app_versions}
+    else:
+        return {}
+
+def parse_backup_labels(hook_type='backup', selected_container=[]):
     client = docker.from_env()
     container_by_service = {
         c.labels.get('com.docker.swarm.service.name'): c for c in client.containers.list()}
-    backup_paths = set()
-    backup_apps = set()
-    pre_commands = {}
-    post_commands = {}
     services = client.services.list()
+    app_settings = {}
     for s in services:
-        labels = s.attrs['Spec']['Labels']
+        specs = s.attrs['Spec']
+        labels = specs['Labels']
+        stack_name = labels['com.docker.stack.namespace']
+        container_name = s.name.removeprefix(f"{stack_name}_")
+        version = labels.get(f'coop-cloud.{stack_name}.version')
+        settings = app_settings[stack_name] = app_settings.get(stack_name) or {}
         if (backup := labels.get('backupbot.backup')) and bool(backup):
-            # volumes: s.attrs['Spec']['TaskTemplate']['ContainerSpec']['Mounts'][0]['Source']
-            stack_name = labels['com.docker.stack.namespace']
-            # Remove this lines to backup only a specific service
-            # This will unfortenately decrease restice performance
-            # if SERVICE and SERVICE != stack_name:
-            #     continue
-            backup_apps.add(stack_name)
-            backup_paths = backup_paths.union(
-                Path(VOLUME_PATH).glob(f"{stack_name}_*"))
-            if not (container := container_by_service.get(s.name)):
-                logger.error(
-                    f"Container {s.name} is not running, hooks can not be executed")
+            settings['enabled'] = True
+        if version:
+            settings['version'] = version
+        if selected_container and container_name not in selected_container:
+            logger.debug(f"Skipping {s.name} because it's not a selected container")
+            continue
+        if mounts:= specs['TaskTemplate']['ContainerSpec'].get('Mounts'):
+            volumes = parse_volumes(stack_name, mounts)
+            volumes.update(settings.get('volumes') or {})
+            settings['volumes'] = volumes
+            excluded_volumes, included_volume_paths = parse_excludes_includes(labels)
+            settings['excluded_volumes'] = excluded_volumes.union(settings.get('excluded_volumes') or set())
+            settings['included_volume_paths'] = included_volume_paths.union(settings.get('included_volume_paths') or set())
+        if container := container_by_service.get(s.name):
+            if command := labels.get(f'backupbot.{hook_type}.pre-hook'):
+                if not (pre_hooks:= settings.get('pre_hooks')):
+                    pre_hooks = settings['pre_hooks'] = {}
+                pre_hooks[container] = command
+            if command := labels.get(f'backupbot.{hook_type}.post-hook'):
+                if not (post_hooks:= settings.get('post_hooks')):
+                    post_hooks = settings['post_hooks'] = {}
+                post_hooks[container] = command
+        else:
+            logger.debug(f"Container {s.name} is not running.")
+            if labels.get(f'backupbot.{hook_type}.pre-hook') or labels.get(f'backupbot.{hook_type}.post-hook'):
+                logger.error(f"Container {s.name} contain hooks but it's not running")
+    return app_settings
+
+
+def get_backup_details(app_settings, volumes=[]):
+    backup_paths = set()
+    backup_apps_versions = {}
+    pre_hooks= {}
+    post_hooks = {}
+    for app, settings in app_settings.items():
+        if settings.get('enabled'):
+            if SERVICE != 'ALL' and SERVICE != app:
                 continue
-            if prehook := labels.get('backupbot.backup.pre-hook'):
-                pre_commands[container] = prehook
-            if posthook := labels.get('backupbot.backup.post-hook'):
-                post_commands[container] = posthook
-    return pre_commands, post_commands, list(backup_paths), list(backup_apps)
+            backup_apps_versions[app] = settings.get('version')
+            add_backup_paths(backup_paths, settings, app, volumes)
+            if hooks:= settings.get('pre_hooks'):
+                pre_hooks.update(hooks)
+            if hooks:= settings.get('post_hooks'):
+                post_hooks.update(hooks)
+    return pre_hooks, post_hooks, list(backup_paths), backup_apps_versions
+
+
+def add_backup_paths(backup_paths, settings, app, selected_volumes):
+    if (volumes := settings.get('volumes')):
+        if includes:= settings.get('included_volume_paths'):
+            included_volumes = list(zip(*includes))[0]
+            for volume, rel_paths in includes:
+                if not (volume_path:= volumes.get(volume)):
+                    logger.error(f'Can not find volume with the name {volume}')
+                    continue
+                if selected_volumes and volume not in selected_volumes:
+                    logger.debug(f'Skipping {volume}:{rel_paths} because the volume is not selected')
+                    continue
+                for p in rel_paths:
+                    absolute_path = Path(f"{volume_path}/{p}")
+                    backup_paths.add(absolute_path)
+        else:
+            included_volumes = []
+        excluded_volumes = settings.get('excluded_volumes') or []
+        for name, path in volumes.items():
+            if selected_volumes and name not in selected_volumes:
+                logger.debug(f'Skipping volume: {name} because the volume is not selected')
+                continue
+            if name in excluded_volumes:
+                logger.debug(f'Skipping volume: {name} because the volume is excluded')
+                continue
+            if name in included_volumes:
+                logger.debug(f'Skipping volume: {name} because a path is selected')
+                continue
+            backup_paths.add(path)
+    else:
+        logger.warning(f"{app} does not contain any volumes")
+
+
+def parse_volumes(stack_name, mounts):
+    volumes = {}
+    for m in mounts:
+        if m['Type'] != 'volume':
+            continue
+        relative_path = m['Source']
+        name = relative_path.removeprefix(stack_name + '_')
+        absolute_path = Path(f"{VOLUME_PATH}{relative_path}/_data/")
+        volumes[name] = absolute_path
+    return volumes
+
+
+def parse_excludes_includes(labels):
+    excluded_volumes = set()
+    included_volume_paths = set()
+    for label, value in labels.items():
+        if label.startswith('backupbot.backup.volumes.'):
+            volume_name = label.removeprefix('backupbot.backup.volumes.').removesuffix('.path')
+            if label.endswith('path'): 
+                relative_paths = tuple(value.split(','))
+                included_volume_paths.add((volume_name, relative_paths))
+            elif bool(value):
+                excluded_volumes.add(volume_name)
+    return excluded_volumes, included_volume_paths
 
 
 def copy_secrets(apps):
@@ -161,7 +330,7 @@ def copy_secrets(apps):
                         f"For the secret {sec['SecretName']} the file {src} does not exist for {s.name}")
                     continue
                 dst = SECRET_PATH + sec['SecretName']
-                logger.debug("Copy Secret {sec['SecretName']}")
+                logger.debug(f"Copy Secret {sec['SecretName']}")
                 copyfile(src, dst)
 
 
@@ -186,17 +355,26 @@ def run_commands(commands):
             logger.info(result.output.decode())
 
 
-def backup_volumes(backup_paths, apps, retries, dry_run=False):
+def backup_volumes(backup_paths, apps_versions, retries, dry_run=False):
     while True:
         try:
+            logger.info("Backup these paths:")
+            logger.debug("\n".join(map(str, backup_paths)))
+            backup_paths = list(filter(path_exists, backup_paths))
+            cmd = restic.cat.base_command()
+            parent = get_snapshots('latest') 
+            if parent:
+                # https://restic.readthedocs.io/en/stable/040_backup.html#file-change-detection
+                cmd.extend(['--parent', parent[0]['short_id']])
+            tags = [f"{app}:{version}" for app,version in apps_versions.items()]
+            if SERVICE == 'ALL':
+                tags.append(SERVICE)
             logger.info("Start volume backup")
-            logger.debug(backup_paths)
-            result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
+            result = restic.internal.backup.run(cmd, backup_paths, dry_run=dry_run, tags=tags)
             logger.summary("backup finished", extra=result)
             return
         except ResticFailedError as error:
-            logger.error(
-                f"Backup failed for {apps}. Could not Backup these paths: {backup_paths}")
+            logger.error(f"Backup failed for {SERVICE}.")
             logger.error(error, exc_info=True)
             if retries > 0:
                 retries -= 1
@@ -204,49 +382,25 @@ def backup_volumes(backup_paths, apps, retries, dry_run=False):
                 exit(1)
 
 
-@cli.command()
-@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
-@click.option('target', '--target', '-t', envvar='TARGET', default='/')
-@click.option('noninteractive', '--noninteractive', envvar='NONINTERACTIVE', is_flag=True)
-def restore(snapshot, target, noninteractive):
-    # Todo: recommend to shutdown the container
-    service_paths = VOLUME_PATH
-    if SERVICE:
-        service_paths = service_paths + f'{SERVICE}_*'
-    snapshots = restic.snapshots(snapshot_id=snapshot)
-    if not snapshot:
-        logger.error("No Snapshots with ID {snapshots}")
-        exit(1)
-    if not noninteractive:
-        snapshot_date = datetime.fromisoformat(snapshots[0]['time'])
-        delta = datetime.now(tz=timezone.utc) - snapshot_date
-        print(
-            f"You are going to restore Snapshot {snapshot} of {service_paths} at {target}")
-        print(f"This snapshot is {delta} old")
-        print(
-            f"THIS COMMAND WILL IRREVERSIBLY OVERWRITES {target}{service_paths.removeprefix('/')}")
-        prompt = input("Type YES (uppercase) to continue: ")
-        if prompt != 'YES':
-            logger.error("Restore aborted")
-            exit(1)
-    print(f"Restoring Snapshot {snapshot} of {service_paths} at {target}")
-    # TODO: use tags if no snapshot is selected, to use a snapshot including SERVICE
-    result = restic.restore(snapshot_id=snapshot,
-                            include=service_paths, target_dir=target)
-    logger.debug(result)
+def path_exists(path):
+    if not path.exists():
+        logger.error(f'{path} does not exist')
+    return path.exists()
 
 
 @cli.command()
 def snapshots():
-    snapshots = restic.snapshots()
-    no_snapshots = True
+    snapshots = get_snapshots()
     for snap in snapshots:
-        if not SERVICE or (tags := snap.get('tags')) and SERVICE in tags:
-            print(snap['time'], snap['id'])
-            no_snapshots = False
-    if no_snapshots:
+        output = [snap['time'], snap['id']]
+        if tags:= snap.get('tags'):
+            app_versions = app_versions_from_tags(tags)
+            if version:= app_versions.get(SERVICE):
+                output.append(version)
+        print(*output)
+    if not snapshots:
         err_msg = "No Snapshots found"
-        if SERVICE:
+        if SERVICE != 'ALL':
             service_name = SERVICE.replace('_', '.')
             err_msg += f' for app {service_name}'
         logger.warning(err_msg)
@@ -264,8 +418,12 @@ def ls(snapshot, path):
 
 def list_files(snapshot, path):
     cmd = restic.cat.base_command() + ['ls']
-    if SERVICE:
-        cmd = cmd + ['--tag', SERVICE]
+    if snapshot == 'latest':
+        latest_snapshot = get_snapshots('latest')
+        if not latest_snapshot:
+            logger.error(f"There is no latest snapshot for {SERVICE}")
+            exit(1)
+        snapshot = latest_snapshot[0]['short_id']
     cmd.append(snapshot)
     if path:
         cmd.append(path)
@@ -274,7 +432,7 @@ def list_files(snapshot, path):
     except ResticFailedError as error:
         if 'no snapshot found' in str(error):
             err_msg = f'There is no snapshot "{snapshot}"'
-            if SERVICE:
+            if SERVICE != 'ALL':
                 err_msg += f' for the app "{SERVICE}"'
             logger.error(err_msg)
             exit(1)
@@ -306,7 +464,7 @@ def download(snapshot, path, volumes, secrets):
         tarinfo.size = len(binary_output)
         file_dumps.append((binary_output, tarinfo))
     if volumes:
-        if not SERVICE:
+        if SERVICE == 'ALL':
             logger.error("Please specify '--host' when using '--volumes'")
             exit(1)
         files = list_files(snapshot, VOLUME_PATH)
@@ -319,7 +477,7 @@ def download(snapshot, path, volumes, secrets):
                 tarinfo.size = len(binary_output)
                 file_dumps.append((binary_output, tarinfo))
     if secrets:
-        if not SERVICE:
+        if SERVICE == 'ALL':
             logger.error("Please specify '--host' when using '--secrets'")
             exit(1)
         filename = f"{SERVICE}.json"
@@ -356,8 +514,13 @@ def get_formatted_size(file_path):
 
 def dump(snapshot, path):
     cmd = restic.cat.base_command() + ['dump']
-    if SERVICE:
-        cmd = cmd + ['--tag', SERVICE]
+    if snapshot == 'latest':
+        latest_snapshot = get_snapshots('latest')
+        if not latest_snapshot:
+            logger.error(f"There is no latest snapshot for {SERVICE}")
+            exit(1)
+        snapshot = latest_snapshot[0]['short_id']
+    cmd.append(snapshot)
     cmd = cmd + [snapshot, path]
     print(f"Dumping {path} from snapshot '{snapshot}'")
     output = subprocess.run(cmd, capture_output=True)

compose.yml

@@ -2,7 +2,7 @@
 version: "3.8"
 services:
   app:
-    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.0.0
+    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.2.0-beta
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
@@ -16,7 +16,7 @@ services:
       - restic_password
     deploy:
       labels:
-        - coop-cloud.${STACK_NAME}.version=0.1.0+latest
+        - coop-cloud.${STACK_NAME}.version=2.1.0+2.2.0-beta
         - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300}
         - coop-cloud.backupbot.enabled=true
     #entrypoint: ['tail', '-f','/dev/null']

release/1.0.0+2.0.0-beta

@@ -0,0 +1 @@
+This is the first beta release of the new backup-bot-two rewrite in python. Be aware when updating, it can break. Please read the readme and update your config according to it.