chore: publish 2.3.0+2.3.0-beta release

README.md

@@ -157,7 +157,7 @@ Show all files inside the latest snapshot (can be very verbose):
 
 Show specific files inside a selected snapshot:
 
-`abra app run <backupbot_name> app -- backup ls --snapshot <snapshot_id> --path /var/lib/docker/volumes/`
+`abra app run <backupbot_name> app -- backup ls --snapshot <snapshot_id> /var/lib/docker/volumes/`
 
 Download files from a snapshot:
 
@@ -179,20 +179,85 @@ restic snapshots
 
 Like Traefik, or `swarm-cronjob`, Backupbot II uses access to the Docker socket to read labels from running Docker Swarm services:
 
+1. Add `ENABLE_BACKUPS=true` to .env.sample
+
+2. Add backupbot labels to the compose file
+
 ```
 services:
   db:
     deploy:
       labels:
-        backupbot.backup: ${BACKUP:-"true"} 
+        backupbot.backup: "${ENABLE_BACKUPS:-true}"
-        backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /volume_path/dump.db'
+        backupbot.backup.pre-hook: "/pg_backup.sh backup"
-        backupbot.backup.post-hook: "rm -rf /volume_path/dump.db"
+        backupbot.backup.volumes.db.path: "backup.sql"
+        backupbot.restore.post-hook: '/pg_backup.sh restore'
+        backupbot.backup.volumes.redis: "false"
 ```
 
 - `backupbot.backup` -- set to `true` to back up this service (REQUIRED)
-- `backupbot.backup.pre-hook` -- command to run before copying files (optional), save all dumps into the volumes
+    - this is the only required backup label, per default it will backup all volumes
-- `backupbot.backup.post-hook` -- command to run after copying files (optional)
+- `backupbot.backup.volumes.<volume_name>.path` -- only backup the listed relative paths from `<volume_name>`
+- `backupbot.backup.volumes.<volume_name>: false` -- exclude <volume_name> from the backup
+- `backupbot.backup.pre-hook` -- command to run before copying files
+    - i.e. save all database dumps into the volumes
+- `backupbot.backup.post-hook` -- command to run after copying files
+- `backupbot.restore.pre-hook` -- command to run before restoring files
+- `backupbot.restore.post-hook` -- command to run after restoring files
+    - i.e. read all database dumps from the volumes
+
+3. (Optional) add backup/restore scripts to the compose file
+
+```
+services:
+  db:
+    configs:
+        - source: pg_backup
+          target: /pg_backup.sh
+          mode: 0555
+
+
+configs:
+  pg_backup:
+    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION}
+    file: pg_backup.sh
+```
+
+Version the config file in `abra.sh`:
+
+```
+export PG_BACKUP_VERSION=v1
+```
 
 As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly.
 
 [abra]: https://git.autonomic.zone/autonomic-cooperative/abra
+
+## Backupbot Development
+
+1. Copy modified backupbot.py into the container:
+
+```
+cp backupbot.py /tmp/backupbot.py; git stash; abra app cp <backupbot_name> /tmp/backupbot.py app:/usr/bin/backupbot.py; git checkout main; git stash pop
+```
+
+2. Testing stuff with the python interpreter inside the container:
+
+```
+abra app run <backupbot_name> app bash
+cd /usr/bin/
+python
+from backupbot import *
+```
+
+### Versioning
+
+- App version: changes to `backup.py` (build a new image)
+- Co-op Cloud package version: changes to recipe.
+
+        For example, starting with 1.0.0+2.0.0:
+            "patch" change to recipe: 1.0.1+2.0.0
+            "patch" change to backup.py: increment both, so 1.1.0+2.0.1
+                   because bumping the image version would result in a minor recipe release
+
+https://git.coopcloud.tech/coop-cloud/backup-bot-two/issues/4

backupbot.py

@@ -98,34 +98,41 @@ def export_secrets():
 @click.option('retries', '--retries', '-r', envvar='RETRIES', default=1)
 def create(retries):
     app_settings = parse_backup_labels()
-    pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings)
+    pre_commands, post_commands, backup_paths, apps_versions = get_backup_details(app_settings)
-    copy_secrets(apps)
+    copy_secrets(apps_versions)
     backup_paths.append(Path(SECRET_PATH))
     run_commands(pre_commands)
-    backup_volumes(backup_paths, apps, int(retries))
+    backup_volumes(backup_paths, apps_versions, int(retries))
     run_commands(post_commands)
 
 
 @cli.command()
-@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
+@click.option('snapshot_id', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
 @click.option('target', '--target', '-t', envvar='TARGET', default='/')
 @click.option('noninteractive', '--noninteractive', envvar='NONINTERACTIVE', is_flag=True)
 @click.option('volumes', '--volumes', '-v', envvar='VOLUMES', multiple=True)
 @click.option('container', '--container', '-c', envvar='CONTAINER', multiple=True)
 @click.option('no_commands', '--no-commands', envvar='NO_COMMANDS', is_flag=True)
-def restore(snapshot, target, noninteractive, volumes, container, no_commands):
+def restore(snapshot_id, target, noninteractive, volumes, container, no_commands):
     app_settings = parse_backup_labels('restore', container)
     if SERVICE != 'ALL':
         app_settings = {SERVICE: app_settings[SERVICE]}
-    pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings, volumes)
+    pre_commands, post_commands, backup_paths, apps_versions = get_backup_details(app_settings, volumes)
-    snapshots = get_snapshots(snapshot_id=snapshot)
+    snapshots = get_snapshots(snapshot_id)
-    if not snapshot:
+    if not snapshots:
-        logger.error("No Snapshots with ID {snapshots} for {apps} found.")
+        logger.error(f"No Snapshots with ID {snapshot_id} for {apps_versions.keys()} found.")
         exit(1)
+    snapshot = snapshots[0]
+    snapshot_id = snapshot['short_id']
     if not noninteractive:
-        snapshot_date = datetime.fromisoformat(snapshots[0]['time'])
+        print(f"Snapshot to restore: \t{snapshot_id}")
-        delta = datetime.now(tz=timezone.utc) - snapshot_date
+        restore_app_versions = app_versions_from_tags(snapshot.get('tags'))
-        print(f"You are going to restore Snapshot {snapshot} of {apps} at {target}")
+        print("Apps:")
+        for app, version in apps_versions.items():
+            restore_version = restore_app_versions.get(app)
+            print(f"\t{app} \t {restore_version}")
+            if version != restore_version:
+                print(f"WARNING!!! The running app is deployed with version {version}")
         print("The following volume paths will be restored:")
         for p in backup_paths:
             print(f'\t{p}')
@@ -133,24 +140,26 @@ def restore(snapshot, target, noninteractive, volumes, container, no_commands):
             print("The following commands will be executed:")
             for container, cmd in list(pre_commands.items()) + list(post_commands.items()):
                 print(f"\t{container.labels['com.docker.swarm.service.name']}:\t{cmd}")
+        snapshot_date = datetime.fromisoformat(snapshot['time'])
+        delta = datetime.now(tz=timezone.utc) - snapshot_date
         print(f"This snapshot is {delta} old")
         print("\nTHIS COMMAND WILL IRREVERSIBLY OVERWRITES FILES")
         prompt = input("Type YES (uppercase) to continue: ")
         if prompt != 'YES':
             logger.error("Restore aborted")
             exit(1)
-    print(f"Restoring Snapshot {snapshot} at {target}")
+    print(f"Restoring Snapshot {snapshot_id} at {target}")
     if not no_commands and pre_commands:
         print(f"Run pre commands.")
         run_commands(pre_commands)
-    result = restic_restore(snapshot_id=snapshot, include=backup_paths, target_dir=target)
+    result = restic_restore(snapshot_id=snapshot_id, include=backup_paths, target_dir=target)
     if not no_commands and post_commands:
         print(f"Run post commands.")
         run_commands(post_commands)
     logger.debug(result)
 
 
-def restic_restore(snapshot_id='latest', include=[], target_dir=None):
+def restic_restore(snapshot_id, include=[], target_dir=None):
     cmd = restic.cat.base_command() + ['restore', snapshot_id]
     for path in include:
         cmd.extend(['--include', path])
@@ -162,18 +171,29 @@ def restic_restore(snapshot_id='latest', include=[], target_dir=None):
 def get_snapshots(snapshot_id=None):
     if snapshot_id and snapshot_id != 'latest':
         snapshots = restic.snapshots(snapshot_id=snapshot_id)
-        if SERVICE not in snapshots[0]['tags']:
+        if not SERVICE in app_versions_from_tags(snapshots[0].get('tags')):
             logger.error(f'Snapshot with ID {snapshot_id} does not contain {SERVICE}')
             exit(1)
     else:
         snapshots = restic.snapshots()
-        snapshots = list(filter(lambda x: x.get('tags') and SERVICE in x.get('tags'), snapshots))
+        snapshots = list(filter(lambda x: SERVICE in app_versions_from_tags(x.get('tags')), snapshots))
     if snapshot_id == 'latest':
         return snapshots[-1:]
     else:
         return snapshots
 
 
+def app_versions_from_tags(tags):
+    if tags:
+        app_versions = map(lambda x: x.split(':'), tags)
+        return {i[0]: i[1] if len(i) > 1 else None for i in app_versions}
+    else:
+        return {}
+
+def str2bool(value: str) -> bool:
+  return value.lower() in ("yes", "true", "t", "1")
+
+
 def parse_backup_labels(hook_type='backup', selected_container=[]):
     client = docker.from_env()
     container_by_service = {
@@ -185,9 +205,12 @@ def parse_backup_labels(hook_type='backup', selected_container=[]):
         labels = specs['Labels']
         stack_name = labels['com.docker.stack.namespace']
         container_name = s.name.removeprefix(f"{stack_name}_")
+        version = labels.get(f'coop-cloud.{stack_name}.version')
         settings = app_settings[stack_name] = app_settings.get(stack_name) or {}
-        if (backup := labels.get('backupbot.backup')) and bool(backup):
+        if (backup := labels.get('backupbot.backup')) and str2bool(backup):
             settings['enabled'] = True
+        if version:
+            settings['version'] = version
         if selected_container and container_name not in selected_container:
             logger.debug(f"Skipping {s.name} because it's not a selected container")
             continue
@@ -216,20 +239,20 @@ def parse_backup_labels(hook_type='backup', selected_container=[]):
 
 def get_backup_details(app_settings, volumes=[]):
     backup_paths = set()
-    backup_apps = []
+    backup_apps_versions = {}
     pre_hooks= {}
     post_hooks = {}
     for app, settings in app_settings.items():
         if settings.get('enabled'):
             if SERVICE != 'ALL' and SERVICE != app:
                 continue
-            backup_apps.append(app)
+            backup_apps_versions[app] = settings.get('version')
             add_backup_paths(backup_paths, settings, app, volumes)
             if hooks:= settings.get('pre_hooks'):
                 pre_hooks.update(hooks)
             if hooks:= settings.get('post_hooks'):
                 post_hooks.update(hooks)
-    return pre_hooks, post_hooks, list(backup_paths), backup_apps
+    return pre_hooks, post_hooks, list(backup_paths), backup_apps_versions
 
 
 def add_backup_paths(backup_paths, settings, app, selected_volumes):
@@ -285,7 +308,7 @@ def parse_excludes_includes(labels):
             if label.endswith('path'): 
                 relative_paths = tuple(value.split(','))
                 included_volume_paths.add((volume_name, relative_paths))
-            elif bool(value):
+            elif not str2bool(value):
                 excluded_volumes.add(volume_name)
     return excluded_volumes, included_volume_paths
 
@@ -336,21 +359,23 @@ def run_commands(commands):
             logger.error(
                 f"Failed to run command {command} in {container.name}: {result.output.decode()}")
         else:
-            logger.info(result.output.decode())
+            logger.debug(result.output.decode())
 
 
-def backup_volumes(backup_paths, apps, retries, dry_run=False):
+def backup_volumes(backup_paths, apps_versions, retries, dry_run=False):
     while True:
         try:
             logger.info("Backup these paths:")
-            logger.debug("\n".join(map(str, backup_paths)))
+            logger.info("\n".join(map(str, backup_paths)))
             backup_paths = list(filter(path_exists, backup_paths))
             cmd = restic.cat.base_command()
             parent = get_snapshots('latest') 
             if parent:
                 # https://restic.readthedocs.io/en/stable/040_backup.html#file-change-detection
                 cmd.extend(['--parent', parent[0]['short_id']])
-            tags = set(apps + [SERVICE])
+            tags = [f"{app}:{version}" for app,version in apps_versions.items()]
+            if SERVICE == 'ALL':
+                tags.append(SERVICE)
             logger.info("Start volume backup")
             result = restic.internal.backup.run(cmd, backup_paths, dry_run=dry_run, tags=tags)
             logger.summary("backup finished", extra=result)
@@ -374,7 +399,12 @@ def path_exists(path):
 def snapshots():
     snapshots = get_snapshots()
     for snap in snapshots:
-        print(snap['time'], snap['id'])
+        output = [snap['time'], snap['id']]
+        if tags:= snap.get('tags'):
+            app_versions = app_versions_from_tags(tags)
+            if version:= app_versions.get(SERVICE):
+                output.append(version)
+        print(*output)
     if not snapshots:
         err_msg = "No Snapshots found"
         if SERVICE != 'ALL':
@@ -385,17 +415,31 @@ def snapshots():
 
 @cli.command()
 @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
-@click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
+@click.option('show_all', '--all', '-a', envvar='SHOW_ALL', is_flag=True)
-def ls(snapshot, path):
+@click.option('timestamps', '--timestamps', '-t', envvar='TIMESTAMPS', is_flag=True)
+@click.argument('path', required=False, default='/var/lib/docker/volumes/')
+def ls(snapshot, show_all, timestamps, path):
+    if snapshot == 'latest':
+        latest_snapshot = get_snapshots('latest')
+        if not latest_snapshot:
+            logger.error(f"There is no latest snapshot for {SERVICE}")
+            exit(1)
+        snapshot = latest_snapshot[0]['short_id']
+    if os.environ.get('INCLUDE_PATH'):
+        path = os.environ.get('INCLUDE_PATH')
+    if show_all:
+        path = None
     results = list_files(snapshot, path)
     for r in results:
         if r.get('path'):
-            print(f"{r['ctime']}\t{r['path']}")
+            if timestamps:
+                print(f"{r['ctime']}\t{r['path']}")
+            else:
+                print(f"{r['path']}")
 
 
 def list_files(snapshot, path):
     cmd = restic.cat.base_command() + ['ls']
-    cmd = cmd + ['--tag', SERVICE]
     cmd.append(snapshot)
     if path:
         cmd.append(path)
@@ -422,6 +466,12 @@ def list_files(snapshot, path):
 @click.option('secrets', '--secrets', '-c', is_flag=True, envvar='SECRETS')
 def download(snapshot, path, volumes, secrets):
     file_dumps = []
+    if snapshot == 'latest':
+        latest_snapshot = get_snapshots('latest')
+        if not latest_snapshot:
+            logger.error(f"There is no latest snapshot for {SERVICE}")
+            exit(1)
+        snapshot = latest_snapshot[0]['short_id']
     if not any([path, volumes, secrets]):
         volumes = secrets = True
     if path:
@@ -486,7 +536,6 @@ def get_formatted_size(file_path):
 
 def dump(snapshot, path):
     cmd = restic.cat.base_command() + ['dump']
-    cmd = cmd + ['--tag', SERVICE]
     cmd = cmd + [snapshot, path]
     print(f"Dumping {path} from snapshot '{snapshot}'")
     output = subprocess.run(cmd, capture_output=True)

compose.yml

@@ -2,7 +2,7 @@
 version: "3.8"
 services:
   app:
-    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.1.1-beta
+    image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.3.0-beta
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
@@ -16,7 +16,7 @@ services:
       - restic_password
     deploy:
       labels:
-        - coop-cloud.${STACK_NAME}.version=2.0.1+2.1.1-beta
+        - coop-cloud.${STACK_NAME}.version=2.3.0+2.3.0-beta
         - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300}
         - coop-cloud.backupbot.enabled=true
     #entrypoint: ['tail', '-f','/dev/null']

pg_backup.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+BACKUP_FILE='/var/lib/postgresql/data/backup.sql'
+
+function backup {
+  export PGPASSWORD=$(cat $POSTGRES_PASSWORD_FILE)
+  pg_dump -U ${POSTGRES_USER} ${POSTGRES_DB} > $BACKUP_FILE
+}
+
+function restore {
+    cd /var/lib/postgresql/data/
+    restore_config(){
+        # Restore allowed connections
+        cat pg_hba.conf.bak > pg_hba.conf
+        su postgres -c 'pg_ctl reload'
+    }
+    # Don't allow any other connections than local
+    cp pg_hba.conf pg_hba.conf.bak
+    echo "local all all trust" > pg_hba.conf
+    su postgres -c 'pg_ctl reload'
+    trap restore_config EXIT INT TERM
+
+    # Recreate Database
+    psql -U ${POSTGRES_USER} -d postgres -c "DROP DATABASE ${POSTGRES_DB} WITH (FORCE);" 
+    createdb -U ${POSTGRES_USER} ${POSTGRES_DB}
+    psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -1 -f $BACKUP_FILE
+
+    trap - EXIT INT TERM
+    restore_config
+}
+
+$@