From 6355f3572f7aa3ad643dc31b22cfffa81ee4d383 Mon Sep 17 00:00:00 2001 From: Philipp Rothmann Date: Tue, 30 May 2023 14:37:42 +0200 Subject: [PATCH 01/31] Backup volumes from host instead of copying paths * Backupbot will now copy all volumes from a service with backupbot.enabled = 'true' label from the /var/lib/docker/volumes/ path directly. This reduces the resource overhead of copying stuff from one volume to another. Recipes need to be adjustet that db-dumps are saved into a volume now! * Remove the Dockerfile and move stuff into a entrypoint. This simplifies the whole versioning thing and makes this "just" a recipe Co-authored-by: Moritz < moritz.m@local-it.org> --- .drone.yml | 16 ---- Dockerfile | 13 --- README.md | 12 ++- abra.sh | 2 + backup.sh | 139 --------------------------------- compose.yml | 24 ++++-- setup-cron.sh => entrypoint.sh | 7 +- 7 files changed, 29 insertions(+), 184 deletions(-) delete mode 100644 Dockerfile create mode 100644 abra.sh delete mode 100755 backup.sh rename setup-cron.sh => entrypoint.sh (50%) diff --git a/.drone.yml b/.drone.yml index 001bbcd..e2f0fe8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,22 +7,6 @@ steps: commands: - shellcheck backup.sh - - name: publish image - image: plugins/docker - settings: - auto_tag: true - username: thecoopcloud - password: - from_secret: thecoopcloud_password - repo: thecoopcloud/backup-bot-two - tags: latest - depends_on: - - run shellcheck - when: - event: - exclude: - - pull_request - trigger: branch: - main diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f15011b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM docker:24.0.6-dind - -RUN apk add --upgrade --no-cache \ - bash \ - curl \ - jq \ - restic - -COPY backup.sh /usr/bin/backup.sh -COPY setup-cron.sh /usr/bin/setup-cron.sh -RUN chmod +x /usr/bin/backup.sh /usr/bin/setup-cron.sh - -ENTRYPOINT [ "/usr/bin/setup-cron.sh" ] diff --git a/README.md b/README.md index d1d4666..53b9ea6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ _This Time, It's Easily Configurable_ -Automatically take backups from running Docker Swarm services into a volume. +Automatically take backups from all volumes of running Docker Swarm services and runs pre- and post commands. ## Background @@ -49,15 +49,13 @@ services: db: deploy: labels: - backupbot.backup: "true" - backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /tmp/dump/dump.db' - backupbot.backup.post-hook: "rm -rf /tmp/dump/dump.db" - backupbot.backup.path: "/tmp/dump/,/etc/foo/" + backupbot.backup: ${BACKUP:-"true"} + backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /volume_path/dump.db' + backupbot.backup.post-hook: "rm -rf /volume_path/dump.db" ``` - `backupbot.backup` -- set to `true` to back up this service (REQUIRED) -- `backupbot.backup.path` -- comma separated list of file paths within the service to copy (REQUIRED) -- `backupbot.backup.pre-hook` -- command to run before copying files (optional) +- `backupbot.backup.pre-hook` -- command to run before copying files (optional), save all dumps into the volumes - `backupbot.backup.post-hook` -- command to run after copying files (optional) As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly. diff --git a/abra.sh b/abra.sh new file mode 100644 index 0000000..9c54329 --- /dev/null +++ b/abra.sh @@ -0,0 +1,2 @@ +export ENTRYPOINT_VERSION=v1 +export BACKUP_VERSION=v1 diff --git a/backup.sh b/backup.sh deleted file mode 100755 index 6aad3f4..0000000 --- a/backup.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash - -server_name="${SERVER_NAME:?SERVER_NAME not set}" - -restic_password_file="${RESTIC_PASSWORD_FILE:?RESTIC_PASSWORD_FILE not set}" - -restic_host="${RESTIC_HOST:?RESTIC_HOST not set}" - -backup_path="${BACKUP_DEST:?BACKUP_DEST not set}" - -# shellcheck disable=SC2153 -ssh_key_file="${SSH_KEY_FILE}" -s3_key_file="${AWS_SECRET_ACCESS_KEY_FILE}" -# shellcheck disable=SC2153 -https_password_file="${HTTPS_PASSWORD_FILE}" - -restic_repo= -restic_extra_options= - -if [ -n "$ssh_key_file" ] && [ -f "$ssh_key_file" ]; then - restic_repo="sftp:$restic_host:/$server_name" - - # Only check server against provided SSH_HOST_KEY, if set - if [ -n "$SSH_HOST_KEY" ]; then - tmpfile=$(mktemp) - echo "$SSH_HOST_KEY" >>"$tmpfile" - echo "using host key $SSH_HOST_KEY" - ssh_options="-o 'UserKnownHostsFile $tmpfile'" - elif [ "$SSH_HOST_KEY_DISABLE" = "1" ]; then - echo "disabling SSH host key checking" - ssh_options="-o 'StrictHostKeyChecking=No'" - else - echo "neither SSH_HOST_KEY nor SSH_HOST_KEY_DISABLE set" - fi - restic_extra_options="sftp.command=ssh $ssh_options -i $ssh_key_file $restic_host -s sftp" -fi - -if [ -n "$s3_key_file" ] && [ -f "$s3_key_file" ] && [ -n "$AWS_ACCESS_KEY_ID" ]; then - AWS_SECRET_ACCESS_KEY="$(cat "${s3_key_file}")" - export AWS_SECRET_ACCESS_KEY - restic_repo="s3:$restic_host:/$server_name" -fi - -if [ -n "$https_password_file" ] && [ -f "$https_password_file" ]; then - HTTPS_PASSWORD="$(cat "${https_password_file}")" - export HTTPS_PASSWORD - restic_user="${RESTIC_USER:?RESTIC_USER not set}" - restic_repo="rest:https://$restic_user:$HTTPS_PASSWORD@$restic_host" -fi - -if [ -z "$restic_repo" ]; then - echo "you must configure either SFTP, S3, or HTTPS storage, see README" - exit 1 -fi - -echo "restic_repo: $restic_repo" - -# Pre-bake-in some default restic options -_restic() { - if [ -z "$restic_extra_options" ]; then - # shellcheck disable=SC2068 - restic -p "$restic_password_file" \ - --quiet -r "$restic_repo" \ - $@ - else - # shellcheck disable=SC2068 - restic -p "$restic_password_file" \ - --quiet -r "$restic_repo" \ - -o "$restic_extra_options" \ - $@ - fi -} - -if [ -n "$SERVICES_OVERRIDE" ]; then - # this is fine because docker service names should never include spaces or - # glob characters - # shellcheck disable=SC2206 - services=($SERVICES_OVERRIDE) -else - mapfile -t services < <(docker service ls --format '{{ .Name }}') -fi - -if [[ \ $*\ != *\ --skip-backup\ * ]]; then - rm -rf "${backup_path}" - - for service in "${services[@]}"; do - echo "service: $service" - details=$(docker service inspect "$service" --format "{{ json .Spec.Labels }}") - if echo "$details" | jq -r '.["backupbot.backup"]' | grep -q 'true'; then - pre=$(echo "$details" | jq -r '.["backupbot.backup.pre-hook"]') - post=$(echo "$details" | jq -r '.["backupbot.backup.post-hook"]') - path=$(echo "$details" | jq -r '.["backupbot.backup.path"]') - - if [ "$path" = "null" ]; then - echo "ERROR: missing 'path' for $service" - continue # or maybe exit? - fi - - container=$(docker container ls -f "name=$service" --format '{{ .ID }}') - - echo "backing up $service" - - if [ "$pre" != "null" ]; then - # run the precommand - # shellcheck disable=SC2086 - docker exec "$container" sh -c "$pre" - fi - - # run the backup - for p in ${path//,/ }; do - # creates the parent folder, so `docker cp` has reliable behaviour no matter if $p ends with `/` or `/.` - dir=$backup_path/$service/$(dirname "$p") - test -d "$dir" || mkdir -p "$dir" - docker cp -a "$container:$p" "$dir/$(basename "$p")" - done - - if [ "$post" != "null" ]; then - # run the postcommand - # shellcheck disable=SC2086 - docker exec "$container" sh -c "$post" - fi - fi - done - - # check if restic repo exists, initialise if not - if [ -z "$(_restic cat config)" ] 2>/dev/null; then - echo "initializing restic repo" - _restic init - fi -fi - -if [[ \ $*\ != *\ --skip-upload\ * ]]; then - _restic backup --host "$server_name" --tag coop-cloud "$backup_path" - - if [ "$REMOVE_BACKUP_VOLUME_AFTER_UPLOAD" -eq 1 ]; then - echo "Cleaning up ${backup_path}" - rm -rf "${backup_path:?}"/* - fi -fi diff --git a/compose.yml b/compose.yml index 2efc5a4..c273182 100644 --- a/compose.yml +++ b/compose.yml @@ -2,11 +2,10 @@ version: "3.8" services: app: - image: thecoopcloud/backup-bot-two:latest -# build: . + image: docker:24.0.2-dind volumes: - "/var/run/docker.sock:/var/run/docker.sock" - - "backups:/backups" + - "/var/lib/docker/volumes/:/var/lib/docker/volumes/" environment: - CRON_SCHEDULE - RESTIC_REPO @@ -25,11 +24,24 @@ services: - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}" - coop-cloud.${STACK_NAME}.version=0.1.0+latest - -volumes: - backups: + configs: + - source: entrypoint + target: /entrypoint.sh + mode: 0555 + - source: backup + target: /backup.sh + mode: 0555 + entrypoint: ['/entrypoint.sh'] secrets: restic_password: external: true name: ${STACK_NAME}_restic_password_${SECRET_RESTIC_PASSWORD_VERSION} + +configs: + entrypoint: + name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION} + file: entrypoint.sh + backup: + name: ${STACK_NAME}_backup_${BACKUP_VERSION} + file: backup.sh diff --git a/setup-cron.sh b/entrypoint.sh similarity index 50% rename from setup-cron.sh rename to entrypoint.sh index b02c779..6de4aa8 100644 --- a/setup-cron.sh +++ b/entrypoint.sh @@ -1,11 +1,12 @@ -#!/bin/bash +#!/bin/sh set -e -set -o pipefail + +apk add --upgrade --no-cache bash curl jq restic cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}" -echo "$cron_schedule /usr/bin/backup.sh" | crontab - +echo "$cron_schedule /backup.sh" | crontab - crontab -l crond -f -d8 -L /dev/stdout From 3261d67dca1636f919808ef9ddfa608b19b79776 Mon Sep 17 00:00:00 2001 From: Philipp Rothmann Date: Tue, 13 Jun 2023 11:48:25 +0200 Subject: [PATCH 02/31] mount volume ro --- compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index c273182..5060768 100644 --- a/compose.yml +++ b/compose.yml @@ -5,7 +5,7 @@ services: image: docker:24.0.2-dind volumes: - "/var/run/docker.sock:/var/run/docker.sock" - - "/var/lib/docker/volumes/:/var/lib/docker/volumes/" + - "/var/lib/docker/volumes/:/var/lib/docker/volumes/:ro" environment: - CRON_SCHEDULE - RESTIC_REPO From 42ae6a6b9ba0aa63503b8dc8a795ad8bac96be8c Mon Sep 17 00:00:00 2001 From: Philipp Rothmann Date: Tue, 13 Jun 2023 14:34:53 +0200 Subject: [PATCH 03/31] remove unused traefik labels --- compose.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compose.yml b/compose.yml index 5060768..0b13631 100644 --- a/compose.yml +++ b/compose.yml @@ -18,11 +18,6 @@ services: - restic_password deploy: labels: - - "traefik.enable=true" - - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=8008" - - "traefik.http.routers.${STACK_NAME}.rule=" - - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure" - - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}" - coop-cloud.${STACK_NAME}.version=0.1.0+latest configs: - source: entrypoint From 447a808849ea43cebb8b559e2228a76e7b69dc3c Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 01:41:03 +0200 Subject: [PATCH 04/31] initial rewrite --- .env.sample | 4 +- abra.sh | 2 +- backupbot.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++ compose.yml | 18 +++--- entrypoint.sh | 9 ++- 5 files changed, 171 insertions(+), 16 deletions(-) create mode 100755 backupbot.py diff --git a/.env.sample b/.env.sample index d8d915e..8a0ef77 100644 --- a/.env.sample +++ b/.env.sample @@ -4,11 +4,9 @@ SECRET_RESTIC_PASSWORD_VERSION=v1 COMPOSE_FILE=compose.yml -SERVER_NAME=example.com -RESTIC_HOST=minio.example.com +RESTIC_REPO=/backups/restic CRON_SCHEDULE='*/5 * * * *' -REMOVE_BACKUP_VOLUME_AFTER_UPLOAD=1 # swarm-cronjob, instead of built-in cron #COMPOSE_FILE="$COMPOSE_FILE:compose.swarm-cronjob.yml" diff --git a/abra.sh b/abra.sh index 9c54329..b6a1153 100644 --- a/abra.sh +++ b/abra.sh @@ -1,2 +1,2 @@ export ENTRYPOINT_VERSION=v1 -export BACKUP_VERSION=v1 +export BACKUPBOT_VERSION=v1 diff --git a/backupbot.py b/backupbot.py new file mode 100755 index 0000000..0fc12be --- /dev/null +++ b/backupbot.py @@ -0,0 +1,154 @@ +#!/usr/bin/python3 + +import os +import click +import json +import subprocess +# todo json logging +import logging +import docker +import restic +from restic.errors import ResticFailedError +from pathlib import Path +logging.basicConfig(level=logging.INFO) + +VOLUME_PATH = "/var/lib/docker/volumes/" +SERVICE = None + +@click.group() +@click.option('-l', '--log', 'loglevel') +@click.option('service', '--host', '-h', envvar='SERVICE') +def cli(loglevel, service): + global SERVICE + if service: + SERVICE = service.replace('.','_') + if loglevel: + numeric_level = getattr(logging, loglevel.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: %s' % loglevel) + logging.basicConfig(level=numeric_level) + init_repo() + + +def init_repo(): + export_secrets() + restic.repository = os.environ['RESTIC_REPO'] + restic.password_file = '/var/run/secrets/restic_password' + try: + restic.cat.config() + except ResticFailedError as error: + if 'unable to open config file' in str(error): + result = restic.init() + logging.info(f"Initialized restic repo: {result}") + else: + raise error + +def export_secrets(): + for env in os.environ: + if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'): + logging.debug(f"exported secret: {env}") + with open(os.environ[env]) as file: + os.environ[env.removesuffix('_FILE')] = file.read() + +@cli.command() +def create(): + pre_commands, post_commands, backup_paths, apps = get_backup_cmds() + run_commands(pre_commands) + backup_volumes(backup_paths, apps) + run_commands(post_commands) + +def get_backup_cmds(): + client = docker.from_env() + containers = dict(map(lambda c: ( + c.labels['com.docker.swarm.service.name'], c), client.containers.list())) + backup_paths = set() + backup_apps = set() + pre_commands = {} + post_commands = {} + services = client.services.list() + for s in services: + labels = s.attrs['Spec']['Labels'] + if (backup := labels.get('backupbot.backup')) and bool(backup): + stack_name = labels['com.docker.stack.namespace'] + if SERVICE and SERVICE != stack_name: + continue + backup_apps.add(stack_name) + container = containers[s.name] + if prehook:= labels.get('backupbot.backup.pre-hook'): + pre_commands[container] = prehook + if posthook:= labels.get('backupbot.backup.post-hook'): + post_commands[container] = posthook + backup_paths = backup_paths.union( + Path(VOLUME_PATH).glob(f"{stack_name}_*")) + return pre_commands, post_commands, list(backup_paths), list(backup_apps) + +def run_commands(commands): + for container, command in commands.items(): + if not command: + continue + # Use bash's pipefail to return exit codes inside a pipe to prevent silent failure + command = command.removeprefix('bash -c \'').removeprefix('sh -c \'') + command = command.removesuffix('\'') + command = f"bash -c 'set -o pipefail;{command}'" + result = container.exec_run(command) + logging.info(f"run command in {container.name}") + logging.info(command) + if result.exit_code: + logging.error(result.output.decode()) + else: + logging.info(result.output.decode()) + +def backup_volumes(backup_paths, apps, dry_run=False): + result = restic.backup(backup_paths, dry_run=dry_run, tags=apps) + logging.info(result) + +@cli.command() +@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', required=True) +def restore(snapshot): + service_paths = f'/var/lib/docker/volumes/{SERVICE}_*' + result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir='/') + + +@cli.command() +def snapshots(): + snapshots = restic.snapshots() + for snap in snapshots: + if not SERVICE or (tags:= snap.get('tags')) and SERVICE in tags: + print(snap['time'], snap['id']) + +@cli.command() +@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', required=True) +@click.option('path', '--path', '-p', envvar='INCLUDE_PATH') +def ls(snapshot, path): + results = list_files(snapshot, path) + for r in results: + if r.get('path'): + print(f"{r['ctime']}\t{r['path']}") + +def list_files(snapshot, path): + cmd = restic.cat.base_command() + ['ls', snapshot] + if path: + cmd.append(path) + output = restic.internal.command_executor.execute(cmd) + output = output.replace('}\n{', '}|{') + results = list(map(json.loads, output.split('|'))) + return results + +@cli.command() +@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', required=True) +@click.option('path', '--path', '-p', envvar='INCLUDE_PATH') +def download(snapshot, path): + files = list_files(snapshot, path) + filetype = [f.get('type') for f in files if f.get('path') == path][0] + cmd = restic.cat.base_command() + ['dump', snapshot, path] + output = subprocess.run(cmd, capture_output=True).stdout + filename = "/tmp/" + Path(path).name + if filetype == 'dir': + filename = filename + ".tar" + with open(filename, "wb") as file: + file.write(output) + print(filename) + + +if __name__ == '__main__': + cli() diff --git a/compose.yml b/compose.yml index 0b13631..56c909a 100644 --- a/compose.yml +++ b/compose.yml @@ -6,14 +6,11 @@ services: volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "/var/lib/docker/volumes/:/var/lib/docker/volumes/:ro" + - backups:/backups environment: - CRON_SCHEDULE - RESTIC_REPO - RESTIC_PASSWORD_FILE=/run/secrets/restic_password - - BACKUP_DEST=/backups - - RESTIC_HOST - - SERVER_NAME - - REMOVE_BACKUP_VOLUME_AFTER_UPLOAD=1 secrets: - restic_password deploy: @@ -23,8 +20,8 @@ services: - source: entrypoint target: /entrypoint.sh mode: 0555 - - source: backup - target: /backup.sh + - source: backupbot + target: /backup mode: 0555 entrypoint: ['/entrypoint.sh'] @@ -32,11 +29,14 @@ secrets: restic_password: external: true name: ${STACK_NAME}_restic_password_${SECRET_RESTIC_PASSWORD_VERSION} + +volumes: + backups: configs: entrypoint: name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION} file: entrypoint.sh - backup: - name: ${STACK_NAME}_backup_${BACKUP_VERSION} - file: backup.sh + backupbot: + name: ${STACK_NAME}_backupbot_${BACKUPBOT_VERSION} + file: backupbot.py diff --git a/entrypoint.sh b/entrypoint.sh index 6de4aa8..94ba3c3 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,12 +1,15 @@ #!/bin/sh -set -e +set -eu -apk add --upgrade --no-cache bash curl jq restic +apk add --upgrade --no-cache bash curl jq restic python3 py3-pip + +# Todo use requirements file +pip install click docker resticpy cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}" -echo "$cron_schedule /backup.sh" | crontab - +#echo "$cron_schedule /backupbot.py" | crontab - crontab -l crond -f -d8 -L /dev/stdout From 28334a4241766a8b9631d2d797eebbf6365ada9c Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 01:44:38 +0200 Subject: [PATCH 05/31] mount volumes read/write to restore backups --- compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index 56c909a..a25b3f3 100644 --- a/compose.yml +++ b/compose.yml @@ -5,7 +5,7 @@ services: image: docker:24.0.2-dind volumes: - "/var/run/docker.sock:/var/run/docker.sock" - - "/var/lib/docker/volumes/:/var/lib/docker/volumes/:ro" + - "/var/lib/docker/volumes/:/var/lib/docker/volumes/" - backups:/backups environment: - CRON_SCHEDULE From 3009159c8206b22dba925a1d529a966216675035 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 02:03:11 +0200 Subject: [PATCH 06/31] use latest snapshot as default --- backupbot.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backupbot.py b/backupbot.py index 0fc12be..af60e12 100755 --- a/backupbot.py +++ b/backupbot.py @@ -10,7 +10,7 @@ import docker import restic from restic.errors import ResticFailedError from pathlib import Path -logging.basicConfig(level=logging.INFO) +#logging.basicConfig(level=logging.INFO) VOLUME_PATH = "/var/lib/docker/volumes/" SERVICE = None @@ -100,10 +100,11 @@ def run_commands(commands): def backup_volumes(backup_paths, apps, dry_run=False): result = restic.backup(backup_paths, dry_run=dry_run, tags=apps) + print(result) logging.info(result) @cli.command() -@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', required=True) +@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') def restore(snapshot): service_paths = f'/var/lib/docker/volumes/{SERVICE}_*' result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir='/') @@ -117,7 +118,7 @@ def snapshots(): print(snap['time'], snap['id']) @cli.command() -@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', required=True) +@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') def ls(snapshot, path): results = list_files(snapshot, path) @@ -135,7 +136,7 @@ def list_files(snapshot, path): return results @cli.command() -@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', required=True) +@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') def download(snapshot, path): files = list_files(snapshot, path) From 203719c22422c081fb024f78c91ff826c0ad7d1f Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 13:09:25 +0200 Subject: [PATCH 07/31] change repo per option --- backupbot.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/backupbot.py b/backupbot.py index af60e12..e131e54 100755 --- a/backupbot.py +++ b/backupbot.py @@ -18,7 +18,8 @@ SERVICE = None @click.group() @click.option('-l', '--log', 'loglevel') @click.option('service', '--host', '-h', envvar='SERVICE') -def cli(loglevel, service): +@click.option('repository', '--repo', '-r', envvar='RESTIC_REPO', required=True) +def cli(loglevel, service, repository): global SERVICE if service: SERVICE = service.replace('.','_') @@ -27,12 +28,12 @@ def cli(loglevel, service): if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level) - init_repo() - - -def init_repo(): export_secrets() - restic.repository = os.environ['RESTIC_REPO'] + init_repo(repository) + + +def init_repo(repository): + restic.repository = repository restic.password_file = '/var/run/secrets/restic_password' try: restic.cat.config() @@ -94,7 +95,7 @@ def run_commands(commands): logging.info(f"run command in {container.name}") logging.info(command) if result.exit_code: - logging.error(result.output.decode()) + logging.error(f"Failed to run command {command} in {container.name}: {result.output.decode()}") else: logging.info(result.output.decode()) From 5fa8f821c156b1701d61003a0f96c707a5b9a0cb Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 13:10:10 +0200 Subject: [PATCH 08/31] choos specific restore target --- backupbot.py | 13 +++++++++---- compose.yml | 2 +- entrypoint.sh | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/backupbot.py b/backupbot.py index e131e54..9932dde 100755 --- a/backupbot.py +++ b/backupbot.py @@ -4,7 +4,6 @@ import os import click import json import subprocess -# todo json logging import logging import docker import restic @@ -106,9 +105,15 @@ def backup_volumes(backup_paths, apps, dry_run=False): @cli.command() @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') -def restore(snapshot): - service_paths = f'/var/lib/docker/volumes/{SERVICE}_*' - result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir='/') +@click.option('target', '--target', '-t', envvar='TARGET', default='/') +def restore(snapshot, target): + # Todo: recommend to shutdown the container + service_paths = VOLUME_PATH + if SERVICE: + service_paths = service_paths + f'{SERVICE}_*' + print(f"restoring Snapshot {snapshot} of {service_paths} at {target}") + result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir=target) + logging.debug(result) @cli.command() diff --git a/compose.yml b/compose.yml index a25b3f3..2f1c682 100644 --- a/compose.yml +++ b/compose.yml @@ -21,7 +21,7 @@ services: target: /entrypoint.sh mode: 0555 - source: backupbot - target: /backup + target: /usr/bin/backup mode: 0555 entrypoint: ['/entrypoint.sh'] diff --git a/entrypoint.sh b/entrypoint.sh index 94ba3c3..3ed47d0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,14 +2,14 @@ set -eu -apk add --upgrade --no-cache bash curl jq restic python3 py3-pip +apk add --upgrade --no-cache bash restic python3 py3-pip -# Todo use requirements file +# Todo use requirements file with specific versions pip install click docker resticpy cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}" -#echo "$cron_schedule /backupbot.py" | crontab - +echo "$cron_schedule backup create" | crontab - crontab -l crond -f -d8 -L /dev/stdout From a86ac153631715b4ab714f3d707f7ad2a6efed61 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 13:44:38 +0200 Subject: [PATCH 09/31] README --- .env.sample | 2 +- README.md | 106 +++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 94 insertions(+), 14 deletions(-) diff --git a/.env.sample b/.env.sample index 8a0ef77..a382dfb 100644 --- a/.env.sample +++ b/.env.sample @@ -6,7 +6,7 @@ COMPOSE_FILE=compose.yml RESTIC_REPO=/backups/restic -CRON_SCHEDULE='*/5 * * * *' +CRON_SCHEDULE='30 */4 * * *' # swarm-cronjob, instead of built-in cron #COMPOSE_FILE="$COMPOSE_FILE:compose.swarm-cronjob.yml" diff --git a/README.md b/README.md index 53b9ea6..025bedf 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ Backupbot II tries to help, by ### With Co-op Cloud -1. Set up Docker Swarm and [`abra`][abra] -2. `abra app new backup-bot-two` -3. `abra app config `, and set storage options. Either configure `CRON_SCHEDULE`, or set up `swarm-cronjob` -4. `abra app secret generate restic-password v1`, optionally with `--pass` before `` to save the generated secret in `pass`. -5. `abra app secret insert ssh-key v1 ...` or similar, to load required secrets. -4. `abra app deploy ` + +* `abra app new backup-bot-two` +* `abra app config ` + - set storage options. Either configure `CRON_SCHEDULE`, or set up `swarm-cronjob` +* `abra app secret generate -a ` +* `abra app deploy ` @@ -42,6 +42,93 @@ Backupbot II tries to help, by ## Configuration +Per default Backupbot stores the backups locally in the repository `/backups/restic`, which is accessible as volume at `/var/lib/docker/volumes/_backups/_data/restic/` + +The backup location can be changed using the `RESTIC_REPO` env variable. + +### S3 Storage + +To use S3 storage as backup location set the following envs: +``` +RESTIC_REPO=s3:/ +SECRET_AWS_SECRET_ACCESS_KEY_VERSION=v1 +AWS_ACCESS_KEY_ID= +COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml" +``` +and add your `` as docker secret: +`abra app secret insert aws_secret_access_key v1 ` + +See [restic s3 docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for more information. + +### SFTP Storage + +> TODO + +To use SFTP storage as backup location set the following envs: +``` +RESTIC_REPO=sftp:user@host:/restic-repo-path +SECRET_SSH_KEY_VERSION=v1 +SSH_HOST_KEY="hostname ssh-rsa AAAAB3... +COMPOSE_FILE="$COMPOSE_FILE:compose.ssh.yml" +``` + +Generate an ssh keypair: `ssh-keygen -t ed25519 -f backupkey -P ''` +and add your `SSH_KEY` as docker secret: +`abra app secret insert ssh_key v1 "$(cat backupkey)"` + + +## Usage + + +Create a backup of all apps: + +`abra app run app -- backup create` + +> The apps to backup up need to be deployed + +Create an individual backup: + +`abra app run app -- backup --host create` + +Create a backup a local repository: + +`abra app run app -- backup create -r /backups/restic` + +> It is recommended to shutdown/undeploy an app before restoring the data + +Restore the latest backup of all including apps: + +`abra app run app -- backup restore` + +Restore a specific backup of an individual app: + +`abra app run app -- backup --host restore --snapshot ` + +Show all snapshots: + +`abra app run app -- backup snapshots` + +Show all snapshots containing a specific app: + +`abra app run app -- backup --host snapshots` + +Show all files inside the latest snapshot (can be very verbose): + +`abra app run app -- backup ls` + +Show specific files inside a selected snapshot: +`abra app run app -- backup ls --snapshot --path /var/lib/docker/volumes/` + +Download files from a snapshot: + +``` +filename=$(abra app run app -- backup download --snapshot --path ) +abra app cp app:$filename . +``` + + +## Recipe Configuration + Like Traefik, or `swarm-cronjob`, Backupbot II uses access to the Docker socket to read labels from running Docker Swarm services: ``` @@ -60,11 +147,4 @@ services: As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly. -## Development - -1. Install `direnv` -2. `cp .envrc.sample .envrc` -3. Edit `.envrc` as appropriate, including setting `DOCKER_CONTEXT` to a remote Docker context, if you're not running a swarm server locally. -4. Run `./backup.sh` -- you can add the `--skip-backup` or `--skip-upload` options if you just want to test one other step - [abra]: https://git.autonomic.zone/autonomic-cooperative/abra From 61ffb6768607e7fb86598954f948468d213423b0 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 15:26:29 +0200 Subject: [PATCH 10/31] update README --- README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 025bedf..110ce77 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,20 @@ _This Time, It's Easily Configurable_ Automatically take backups from all volumes of running Docker Swarm services and runs pre- and post commands. + + +* **Category**: Utilities +* **Status**: 0, work-in-progress +* **Image**: [`thecoopcloud/backup-bot-two`](https://hub.docker.com/r/thecoopcloud/backup-bot-two), 4, upstream +* **Healthcheck**: No +* **Backups**: N/A +* **Email**: N/A +* **Tests**: No +* **SSO**: N/A + + + + ## Background There are lots of Docker volume backup systems; all of them have one or both of these limitations: @@ -27,19 +41,6 @@ Backupbot II tries to help, by * `abra app secret generate -a ` * `abra app deploy ` - - -* **Category**: Utilities -* **Status**: 0, work-in-progress -* **Image**: [`thecoopcloud/backup-bot-two`](https://hub.docker.com/r/thecoopcloud/backup-bot-two), 4, upstream -* **Healthcheck**: No -* **Backups**: N/A -* **Email**: N/A -* **Tests**: No -* **SSO**: N/A - - - ## Configuration Per default Backupbot stores the backups locally in the repository `/backups/restic`, which is accessible as volume at `/var/lib/docker/volumes/_backups/_data/restic/` From d32337cf3ac71336c548f8bce10b59f08875cf06 Mon Sep 17 00:00:00 2001 From: Moritz Date: Thu, 7 Sep 2023 15:32:57 +0200 Subject: [PATCH 11/31] update README --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 110ce77..43968da 100644 --- a/README.md +++ b/README.md @@ -91,17 +91,17 @@ Create an individual backup: `abra app run app -- backup --host create` -Create a backup a local repository: +Create a backup to a local repository: `abra app run app -- backup create -r /backups/restic` > It is recommended to shutdown/undeploy an app before restoring the data -Restore the latest backup of all including apps: +Restore the latest snapshot of all including apps: `abra app run app -- backup restore` -Restore a specific backup of an individual app: +Restore a specific snapshot of an individual app: `abra app run app -- backup --host restore --snapshot ` @@ -118,6 +118,7 @@ Show all files inside the latest snapshot (can be very verbose): `abra app run app -- backup ls` Show specific files inside a selected snapshot: + `abra app run app -- backup ls --snapshot --path /var/lib/docker/volumes/` Download files from a snapshot: From 75a93c545658ef9f7a4893c0adc2e0a8e139315b Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 8 Sep 2023 01:16:44 +0200 Subject: [PATCH 12/31] add sftp storage --- README.md | 12 +++++++++--- abra.sh | 1 + compose.ssh.yml | 9 ++++++++- entrypoint.sh | 7 ++++++- ssh_config | 4 ++++ 5 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 ssh_config diff --git a/README.md b/README.md index 43968da..661fbda 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ See [restic s3 docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new ### SFTP Storage -> TODO +> With sftp it is not possible to prevent the backupbot from deleting backups in case of a compromised machine. Therefore we recommend to use S3, REST or rclone server without delete permissions. To use SFTP storage as backup location set the following envs: ``` @@ -72,10 +72,16 @@ SECRET_SSH_KEY_VERSION=v1 SSH_HOST_KEY="hostname ssh-rsa AAAAB3... COMPOSE_FILE="$COMPOSE_FILE:compose.ssh.yml" ``` +To get the `SSH_HOST_KEY` run the following command `ssh-keyscan ` Generate an ssh keypair: `ssh-keygen -t ed25519 -f backupkey -P ''` -and add your `SSH_KEY` as docker secret: -`abra app secret insert ssh_key v1 "$(cat backupkey)"` +Add the key to your `authorized_keys`: +`ssh-copy-id -i backupkey @` +Add your `SSH_KEY` as docker secret: +``` +abra app secret insert ssh_key v1 """$(cat backupkey) +""" +``` ## Usage diff --git a/abra.sh b/abra.sh index b6a1153..d806fdb 100644 --- a/abra.sh +++ b/abra.sh @@ -1,2 +1,3 @@ export ENTRYPOINT_VERSION=v1 export BACKUPBOT_VERSION=v1 +export SSH_CONFIG_VERSION=v1 diff --git a/compose.ssh.yml b/compose.ssh.yml index d6b68f3..bb48647 100644 --- a/compose.ssh.yml +++ b/compose.ssh.yml @@ -5,12 +5,19 @@ services: environment: - SSH_KEY_FILE=/run/secrets/ssh_key - SSH_HOST_KEY - - SSH_HOST_KEY_DISABLE secrets: - source: ssh_key mode: 0400 + configs: + - source: ssh_config + target: /root/.ssh/config secrets: ssh_key: external: true name: ${STACK_NAME}_ssh_key_${SECRET_SSH_KEY_VERSION} + +configs: + ssh_config: + name: ${STACK_NAME}_ssh_config_${SSH_CONFIG_VERSION} + file: ssh_config diff --git a/entrypoint.sh b/entrypoint.sh index 3ed47d0..7e8c728 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,12 +1,17 @@ #!/bin/sh -set -eu +set -eu -o pipefail apk add --upgrade --no-cache bash restic python3 py3-pip # Todo use requirements file with specific versions pip install click docker resticpy +if [ -n "$SSH_HOST_KEY" ] +then + echo "$SSH_HOST_KEY" > /root/.ssh/known_hosts +fi + cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}" echo "$cron_schedule backup create" | crontab - diff --git a/ssh_config b/ssh_config new file mode 100644 index 0000000..294dc88 --- /dev/null +++ b/ssh_config @@ -0,0 +1,4 @@ +Host * + IdentityFile /run/secrets/ssh_key + ServerAliveInterval 60 + ServerAliveCountMax 240 From bd8398e7dd7d46aa4cb7a5836c71b59ad2d46c5f Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 8 Sep 2023 01:17:01 +0200 Subject: [PATCH 13/31] add healthcheck --- compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compose.yml b/compose.yml index 2f1c682..cd573b0 100644 --- a/compose.yml +++ b/compose.yml @@ -16,6 +16,7 @@ services: deploy: labels: - coop-cloud.${STACK_NAME}.version=0.1.0+latest + - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120} configs: - source: entrypoint target: /entrypoint.sh @@ -24,6 +25,12 @@ services: target: /usr/bin/backup mode: 0555 entrypoint: ['/entrypoint.sh'] + healthcheck: + test: "pgrep crond" + interval: 30s + timeout: 10s + retries: 10 + start_period: 5m secrets: restic_password: From 06ad03c1d5ccd1e6e9a2a08c2b028d96444b485f Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 8 Sep 2023 01:30:08 +0200 Subject: [PATCH 14/31] specify program versions to prevent future breakage --- entrypoint.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 7e8c728..217f3be 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,10 +2,14 @@ set -eu -o pipefail -apk add --upgrade --no-cache bash restic python3 py3-pip +apk add --upgrade --no-cache \ + bash=5.2.15-r5 \ + restic=0.15.2-r2 \ + python3=3.11.5-r0 \ + py3-pip=23.1.2-r0 # Todo use requirements file with specific versions -pip install click docker resticpy +pip install click==8.1.7 docker==6.1.3 resticpy==1.0.2 if [ -n "$SSH_HOST_KEY" ] then From 33ce3c58aa19cccc7cb561e73088fe4cd2b4b783 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 8 Sep 2023 01:48:09 +0200 Subject: [PATCH 15/31] fix entrypoint --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 217f3be..dd9328b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/sh -set -eu -o pipefail +set -e -o pipefail apk add --upgrade --no-cache \ bash=5.2.15-r5 \ From 6fa9440c76665a377b68948b93f379c6a4a54a69 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 00:50:45 +0200 Subject: [PATCH 16/31] fix restic version, timeout and cron default timer --- .env.sample | 2 +- compose.yml | 2 +- entrypoint.sh | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index a382dfb..048d303 100644 --- a/.env.sample +++ b/.env.sample @@ -6,7 +6,7 @@ COMPOSE_FILE=compose.yml RESTIC_REPO=/backups/restic -CRON_SCHEDULE='30 */4 * * *' +CRON_SCHEDULE='30 3 * * *' # swarm-cronjob, instead of built-in cron #COMPOSE_FILE="$COMPOSE_FILE:compose.swarm-cronjob.yml" diff --git a/compose.yml b/compose.yml index cd573b0..f6c0511 100644 --- a/compose.yml +++ b/compose.yml @@ -16,7 +16,7 @@ services: deploy: labels: - coop-cloud.${STACK_NAME}.version=0.1.0+latest - - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120} + - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300} configs: - source: entrypoint target: /entrypoint.sh diff --git a/entrypoint.sh b/entrypoint.sh index dd9328b..f768e7b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,6 +3,7 @@ set -e -o pipefail apk add --upgrade --no-cache \ + restic=0.15.2-r3 \ bash=5.2.15-r5 \ restic=0.15.2-r2 \ python3=3.11.5-r0 \ From 825565451aa624062988bcfed0c1a9d57d0d85cd Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 00:51:36 +0200 Subject: [PATCH 17/31] feat: Backup Secrets #28 --- backupbot.py | 40 ++++++++++++++++++++++++++++++++++++++++ entrypoint.sh | 4 +++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backupbot.py b/backupbot.py index 9932dde..f3bb38e 100755 --- a/backupbot.py +++ b/backupbot.py @@ -50,6 +50,41 @@ def export_secrets(): with open(os.environ[env]) as file: os.environ[env.removesuffix('_FILE')] = file.read() +@cli.command(help='Attach all secrets to the backupbot container, this can result in a container restart') +def attach_secrets(): + client = docker.from_env() + services = client.services.list() + apps = [] + secrets = [] + secret_ids = [] + # Get all Apps that aktivate backups + for s in services: + labels = s.attrs['Spec']['Labels'] + if (backup := labels.get('backupbot.backup')) and bool(backup): + apps.append(labels['com.docker.stack.namespace']) + # Get all Secrets for these Apps + for s in services: + labels = s.attrs['Spec']['Labels'] + if labels['com.docker.stack.namespace'] in apps: + if app_secs:= s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'): + for sec in app_secs: + if sec['SecretID'] not in secret_ids: + # Move Secret Targets to SecretName to avoid conflicts + secret_ids.append(sec['SecretID']) + sec['File']['Name'] = sec['SecretName'] + secrets.append(sec) + backupbot_service = client.services.get(os.environ['STACK_NAME']+"_app") + # Append the backupbot secrets + backupbot_secrets = backupbot_service.attrs['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] + for sec in backupbot_secrets: + if os.environ['STACK_NAME'] in sec['SecretName']: + secrets.append(sec) + new_sec_ids = set(map(lambda s: s['SecretID'], secrets)) + old_sec_ids = set(map(lambda s: s['SecretID'], backupbot_secrets)) + if new_sec_ids.difference(old_sec_ids): + logging.warning("Backupbot will restart to update the secrets") + backupbot_service.update(secrets=secrets) + @cli.command() def create(): pre_commands, post_commands, backup_paths, apps = get_backup_cmds() @@ -78,8 +113,12 @@ def get_backup_cmds(): pre_commands[container] = prehook if posthook:= labels.get('backupbot.backup.post-hook'): post_commands[container] = posthook + # Backup volumes backup_paths = backup_paths.union( Path(VOLUME_PATH).glob(f"{stack_name}_*")) + # Backup secrets + backup_paths = backup_paths.union( + Path('/var/run/secrets').glob(f"{stack_name}_*")) return pre_commands, post_commands, list(backup_paths), list(backup_apps) def run_commands(commands): @@ -145,6 +184,7 @@ def list_files(snapshot, path): @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') def download(snapshot, path): + path = path.removesuffix('/') files = list_files(snapshot, path) filetype = [f.get('type') for f in files if f.get('path') == path][0] cmd = restic.cat.base_command() + ['dump', snapshot, path] diff --git a/entrypoint.sh b/entrypoint.sh index f768e7b..b5be1cf 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,13 +5,15 @@ set -e -o pipefail apk add --upgrade --no-cache \ restic=0.15.2-r3 \ bash=5.2.15-r5 \ - restic=0.15.2-r2 \ python3=3.11.5-r0 \ py3-pip=23.1.2-r0 # Todo use requirements file with specific versions pip install click==8.1.7 docker==6.1.3 resticpy==1.0.2 +# Attach secrets to backupbot +backup attach-secrets + if [ -n "$SSH_HOST_KEY" ] then echo "$SSH_HOST_KEY" > /root/.ssh/known_hosts From 488c59f667231272d629bdd3d2bd180d47c1cbd7 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 14:15:31 +0200 Subject: [PATCH 18/31] Revert "feat: Backup Secrets #28" This reverts commit 2838a36d43f44f80aa76095863f463d6aae57403. --- backupbot.py | 40 ---------------------------------------- entrypoint.sh | 4 +--- 2 files changed, 1 insertion(+), 43 deletions(-) diff --git a/backupbot.py b/backupbot.py index f3bb38e..9932dde 100755 --- a/backupbot.py +++ b/backupbot.py @@ -50,41 +50,6 @@ def export_secrets(): with open(os.environ[env]) as file: os.environ[env.removesuffix('_FILE')] = file.read() -@cli.command(help='Attach all secrets to the backupbot container, this can result in a container restart') -def attach_secrets(): - client = docker.from_env() - services = client.services.list() - apps = [] - secrets = [] - secret_ids = [] - # Get all Apps that aktivate backups - for s in services: - labels = s.attrs['Spec']['Labels'] - if (backup := labels.get('backupbot.backup')) and bool(backup): - apps.append(labels['com.docker.stack.namespace']) - # Get all Secrets for these Apps - for s in services: - labels = s.attrs['Spec']['Labels'] - if labels['com.docker.stack.namespace'] in apps: - if app_secs:= s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'): - for sec in app_secs: - if sec['SecretID'] not in secret_ids: - # Move Secret Targets to SecretName to avoid conflicts - secret_ids.append(sec['SecretID']) - sec['File']['Name'] = sec['SecretName'] - secrets.append(sec) - backupbot_service = client.services.get(os.environ['STACK_NAME']+"_app") - # Append the backupbot secrets - backupbot_secrets = backupbot_service.attrs['Spec']['TaskTemplate']['ContainerSpec']['Secrets'] - for sec in backupbot_secrets: - if os.environ['STACK_NAME'] in sec['SecretName']: - secrets.append(sec) - new_sec_ids = set(map(lambda s: s['SecretID'], secrets)) - old_sec_ids = set(map(lambda s: s['SecretID'], backupbot_secrets)) - if new_sec_ids.difference(old_sec_ids): - logging.warning("Backupbot will restart to update the secrets") - backupbot_service.update(secrets=secrets) - @cli.command() def create(): pre_commands, post_commands, backup_paths, apps = get_backup_cmds() @@ -113,12 +78,8 @@ def get_backup_cmds(): pre_commands[container] = prehook if posthook:= labels.get('backupbot.backup.post-hook'): post_commands[container] = posthook - # Backup volumes backup_paths = backup_paths.union( Path(VOLUME_PATH).glob(f"{stack_name}_*")) - # Backup secrets - backup_paths = backup_paths.union( - Path('/var/run/secrets').glob(f"{stack_name}_*")) return pre_commands, post_commands, list(backup_paths), list(backup_apps) def run_commands(commands): @@ -184,7 +145,6 @@ def list_files(snapshot, path): @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') def download(snapshot, path): - path = path.removesuffix('/') files = list_files(snapshot, path) filetype = [f.get('type') for f in files if f.get('path') == path][0] cmd = restic.cat.base_command() + ['dump', snapshot, path] diff --git a/entrypoint.sh b/entrypoint.sh index b5be1cf..f768e7b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,15 +5,13 @@ set -e -o pipefail apk add --upgrade --no-cache \ restic=0.15.2-r3 \ bash=5.2.15-r5 \ + restic=0.15.2-r2 \ python3=3.11.5-r0 \ py3-pip=23.1.2-r0 # Todo use requirements file with specific versions pip install click==8.1.7 docker==6.1.3 resticpy==1.0.2 -# Attach secrets to backupbot -backup attach-secrets - if [ -n "$SSH_HOST_KEY" ] then echo "$SSH_HOST_KEY" > /root/.ssh/known_hosts From ebc0ea5d849c879881387bb40c3fc2090da26071 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 14:17:25 +0200 Subject: [PATCH 19/31] small fixes --- backupbot.py | 1 + entrypoint.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/backupbot.py b/backupbot.py index 9932dde..8409e2e 100755 --- a/backupbot.py +++ b/backupbot.py @@ -145,6 +145,7 @@ def list_files(snapshot, path): @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') def download(snapshot, path): + path = path.removesuffix('/') files = list_files(snapshot, path) filetype = [f.get('type') for f in files if f.get('path') == path][0] cmd = restic.cat.base_command() + ['dump', snapshot, path] diff --git a/entrypoint.sh b/entrypoint.sh index f768e7b..4493054 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,7 +5,6 @@ set -e -o pipefail apk add --upgrade --no-cache \ restic=0.15.2-r3 \ bash=5.2.15-r5 \ - restic=0.15.2-r2 \ python3=3.11.5-r0 \ py3-pip=23.1.2-r0 From 5d4def6143c0538f8c8706671fc6599078a9ba10 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 16:39:40 +0200 Subject: [PATCH 20/31] feat: Backup Secrets (copy secrets) #28 --- backupbot.py | 30 +++++++++++++++++++++++++++--- compose.yml | 1 + 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/backupbot.py b/backupbot.py index 8409e2e..b57e536 100755 --- a/backupbot.py +++ b/backupbot.py @@ -9,9 +9,11 @@ import docker import restic from restic.errors import ResticFailedError from pathlib import Path +from shutil import copyfile, rmtree #logging.basicConfig(level=logging.INFO) VOLUME_PATH = "/var/lib/docker/volumes/" +SECRET_PATH = '/secrets/' SERVICE = None @click.group() @@ -53,14 +55,15 @@ def export_secrets(): @cli.command() def create(): pre_commands, post_commands, backup_paths, apps = get_backup_cmds() + copy_secrets(apps) + backup_paths.append(SECRET_PATH) run_commands(pre_commands) backup_volumes(backup_paths, apps) run_commands(post_commands) def get_backup_cmds(): client = docker.from_env() - containers = dict(map(lambda c: ( - c.labels['com.docker.swarm.service.name'], c), client.containers.list())) + container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} backup_paths = set() backup_apps = set() pre_commands = {} @@ -73,7 +76,9 @@ def get_backup_cmds(): if SERVICE and SERVICE != stack_name: continue backup_apps.add(stack_name) - container = containers[s.name] + container = container_by_service.get(s.name) + if not container: + logging.error("Container {s.name} is not running, hooks can not be executed") if prehook:= labels.get('backupbot.backup.pre-hook'): pre_commands[container] = prehook if posthook:= labels.get('backupbot.backup.post-hook'): @@ -82,6 +87,25 @@ def get_backup_cmds(): Path(VOLUME_PATH).glob(f"{stack_name}_*")) return pre_commands, post_commands, list(backup_paths), list(backup_apps) +def copy_secrets(apps): + rmtree(SECRET_PATH, ignore_errors=True) + os.mkdir(SECRET_PATH) + client = docker.from_env() + container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} + services = client.services.list() + for s in services: + app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace'] + if app_name in apps: + if app_secs:= s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'): + if not container_by_service.get(s.name): + logging.error("Container {s.name} is not running, secrets can not be copied.") + continue + container_id = container_by_service[s.name].id + for sec in app_secs: + src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}' + dst = SECRET_PATH + sec['SecretName'] + copyfile(src, dst) + def run_commands(commands): for container, command in commands.items(): if not command: diff --git a/compose.yml b/compose.yml index f6c0511..13c591d 100644 --- a/compose.yml +++ b/compose.yml @@ -6,6 +6,7 @@ services: volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "/var/lib/docker/volumes/:/var/lib/docker/volumes/" + - "/var/lib/docker/containers/:/var/lib/docker/containers/:ro" - backups:/backups environment: - CRON_SCHEDULE From 15a552ef8b707545c6639aa2b20ec861d22d03b2 Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 16:50:45 +0200 Subject: [PATCH 21/31] formatting --- backupbot.py | 57 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/backupbot.py b/backupbot.py index b57e536..3501b23 100755 --- a/backupbot.py +++ b/backupbot.py @@ -10,12 +10,13 @@ import restic from restic.errors import ResticFailedError from pathlib import Path from shutil import copyfile, rmtree -#logging.basicConfig(level=logging.INFO) +# logging.basicConfig(level=logging.INFO) VOLUME_PATH = "/var/lib/docker/volumes/" SECRET_PATH = '/secrets/' SERVICE = None + @click.group() @click.option('-l', '--log', 'loglevel') @click.option('service', '--host', '-h', envvar='SERVICE') @@ -23,7 +24,7 @@ SERVICE = None def cli(loglevel, service, repository): global SERVICE if service: - SERVICE = service.replace('.','_') + SERVICE = service.replace('.', '_') if loglevel: numeric_level = getattr(logging, loglevel.upper(), None) if not isinstance(numeric_level, int): @@ -45,6 +46,7 @@ def init_repo(repository): else: raise error + def export_secrets(): for env in os.environ: if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'): @@ -52,6 +54,7 @@ def export_secrets(): with open(os.environ[env]) as file: os.environ[env.removesuffix('_FILE')] = file.read() + @cli.command() def create(): pre_commands, post_commands, backup_paths, apps = get_backup_cmds() @@ -61,9 +64,11 @@ def create(): backup_volumes(backup_paths, apps) run_commands(post_commands) + def get_backup_cmds(): client = docker.from_env() - container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} + container_by_service = { + c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} backup_paths = set() backup_apps = set() pre_commands = {} @@ -78,33 +83,38 @@ def get_backup_cmds(): backup_apps.add(stack_name) container = container_by_service.get(s.name) if not container: - logging.error("Container {s.name} is not running, hooks can not be executed") - if prehook:= labels.get('backupbot.backup.pre-hook'): + logging.error( + "Container {s.name} is not running, hooks can not be executed") + if prehook := labels.get('backupbot.backup.pre-hook'): pre_commands[container] = prehook - if posthook:= labels.get('backupbot.backup.post-hook'): + if posthook := labels.get('backupbot.backup.post-hook'): post_commands[container] = posthook backup_paths = backup_paths.union( Path(VOLUME_PATH).glob(f"{stack_name}_*")) return pre_commands, post_commands, list(backup_paths), list(backup_apps) + def copy_secrets(apps): rmtree(SECRET_PATH, ignore_errors=True) os.mkdir(SECRET_PATH) client = docker.from_env() - container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} + container_by_service = { + c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} services = client.services.list() for s in services: app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace'] - if app_name in apps: - if app_secs:= s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'): - if not container_by_service.get(s.name): - logging.error("Container {s.name} is not running, secrets can not be copied.") - continue - container_id = container_by_service[s.name].id - for sec in app_secs: - src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}' - dst = SECRET_PATH + sec['SecretName'] - copyfile(src, dst) + if (app_name in apps and + app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets')): + if not container_by_service.get(s.name): + logging.error( + "Container {s.name} is not running, secrets can not be copied.") + continue + container_id = container_by_service[s.name].id + for sec in app_secs: + src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}' + dst = SECRET_PATH + sec['SecretName'] + copyfile(src, dst) + def run_commands(commands): for container, command in commands.items(): @@ -118,15 +128,18 @@ def run_commands(commands): logging.info(f"run command in {container.name}") logging.info(command) if result.exit_code: - logging.error(f"Failed to run command {command} in {container.name}: {result.output.decode()}") + logging.error( + f"Failed to run command {command} in {container.name}: {result.output.decode()}") else: logging.info(result.output.decode()) + def backup_volumes(backup_paths, apps, dry_run=False): result = restic.backup(backup_paths, dry_run=dry_run, tags=apps) print(result) logging.info(result) + @cli.command() @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('target', '--target', '-t', envvar='TARGET', default='/') @@ -136,7 +149,8 @@ def restore(snapshot, target): if SERVICE: service_paths = service_paths + f'{SERVICE}_*' print(f"restoring Snapshot {snapshot} of {service_paths} at {target}") - result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir=target) + result = restic.restore(snapshot_id=snapshot, + include=service_paths, target_dir=target) logging.debug(result) @@ -144,9 +158,10 @@ def restore(snapshot, target): def snapshots(): snapshots = restic.snapshots() for snap in snapshots: - if not SERVICE or (tags:= snap.get('tags')) and SERVICE in tags: + if not SERVICE or (tags := snap.get('tags')) and SERVICE in tags: print(snap['time'], snap['id']) + @cli.command() @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') @@ -156,6 +171,7 @@ def ls(snapshot, path): if r.get('path'): print(f"{r['ctime']}\t{r['path']}") + def list_files(snapshot, path): cmd = restic.cat.base_command() + ['ls', snapshot] if path: @@ -165,6 +181,7 @@ def list_files(snapshot, path): results = list(map(json.loads, output.split('|'))) return results + @cli.command() @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') From 1f06af95eb642da686e13a3d047afc0c173aeccb Mon Sep 17 00:00:00 2001 From: Moritz Date: Fri, 22 Sep 2023 18:33:42 +0200 Subject: [PATCH 22/31] fix error messages --- backupbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backupbot.py b/backupbot.py index 3501b23..89f27a1 100755 --- a/backupbot.py +++ b/backupbot.py @@ -84,7 +84,7 @@ def get_backup_cmds(): container = container_by_service.get(s.name) if not container: logging.error( - "Container {s.name} is not running, hooks can not be executed") + f"Container {s.name} is not running, hooks can not be executed") if prehook := labels.get('backupbot.backup.pre-hook'): pre_commands[container] = prehook if posthook := labels.get('backupbot.backup.post-hook'): @@ -107,7 +107,7 @@ def copy_secrets(apps): app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets')): if not container_by_service.get(s.name): logging.error( - "Container {s.name} is not running, secrets can not be copied.") + f"Container {s.name} is not running, secrets can not be copied.") continue container_id = container_by_service[s.name].id for sec in app_secs: From 6fc62b551629cc93e02bc9dc077be2c880c66d26 Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 3 Oct 2023 18:19:43 +0200 Subject: [PATCH 23/31] fix typo --- backupbot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backupbot.py b/backupbot.py index 89f27a1..30ce01b 100755 --- a/backupbot.py +++ b/backupbot.py @@ -104,7 +104,7 @@ def copy_secrets(apps): for s in services: app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace'] if (app_name in apps and - app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets')): + (app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'))): if not container_by_service.get(s.name): logging.error( f"Container {s.name} is not running, secrets can not be copied.") From 9398e0d83ded105efa5c03044fa2f9ce10efa589 Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 3 Oct 2023 18:41:16 +0200 Subject: [PATCH 24/31] release note for migration --- release/1.0.0+latest | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 release/1.0.0+latest diff --git a/release/1.0.0+latest b/release/1.0.0+latest new file mode 100644 index 0000000..47876a2 --- /dev/null +++ b/release/1.0.0+latest @@ -0,0 +1,3 @@ +Breaking Change: the variables `SERVER_NAME` and `RESTIC_HOST` are merged into `RESTIC_REPO`. The format can be looked up here: https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html +ssh/sftp: `sftp:user@host:/repo-path` +S3: `s3:https://s3.example.com/bucket_name` From ab6c06d4234c4d0dd3c869c225963bf6a9ff45f3 Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 3 Oct 2023 22:04:15 +0200 Subject: [PATCH 25/31] Prompt before restore --- backupbot.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/backupbot.py b/backupbot.py index 30ce01b..a15d848 100755 --- a/backupbot.py +++ b/backupbot.py @@ -7,6 +7,7 @@ import subprocess import logging import docker import restic +from datetime import datetime, timezone from restic.errors import ResticFailedError from pathlib import Path from shutil import copyfile, rmtree @@ -143,12 +144,27 @@ def backup_volumes(backup_paths, apps, dry_run=False): @cli.command() @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('target', '--target', '-t', envvar='TARGET', default='/') -def restore(snapshot, target): +@click.option('noninteractive', '--noninteractive', envvar='NONINTERACTIVE', default=False) +def restore(snapshot, target, noninteractive): # Todo: recommend to shutdown the container service_paths = VOLUME_PATH if SERVICE: service_paths = service_paths + f'{SERVICE}_*' - print(f"restoring Snapshot {snapshot} of {service_paths} at {target}") + snapshots = restic.snapshots(snapshot_id=snapshot) + if not snapshot: + logging.error("No Snapshots with ID {snapshots}") + exit(1) + if not noninteractive: + snapshot_date = datetime.fromisoformat(snapshots[0]['time']) + delta = datetime.now(tz=timezone.utc) - snapshot_date + print(f"You are going to restore Snapshot {snapshot} of {service_paths} at {target}") + print(f"This snapshot is {delta} old") + print(f"THIS COMMAND WILL IRREVERSIBLY OVERWRITES {target}{service_paths.removeprefix('/')}") + prompt = input("Type YES (uppercase) to continue: ") + if prompt != 'YES': + logging.error("Restore aborted") + exit(1) + print(f"Restoring Snapshot {snapshot} of {service_paths} at {target}") result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir=target) logging.debug(result) From c3f3d1a6fe870081fa904fc539866943a04f4a7b Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 3 Oct 2023 22:39:06 +0200 Subject: [PATCH 26/31] restic_repo as secret option #31 --- .env.sample | 10 ++++++---- README.md | 17 +++++++++++++++++ backupbot.py | 16 +++++++++++----- compose.secret.yml | 13 +++++++++++++ 4 files changed, 47 insertions(+), 9 deletions(-) create mode 100644 compose.secret.yml diff --git a/.env.sample b/.env.sample index 048d303..172d156 100644 --- a/.env.sample +++ b/.env.sample @@ -21,7 +21,9 @@ CRON_SCHEDULE='30 3 * * *' #AWS_ACCESS_KEY_ID=something-secret #COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml" -# HTTPS storage -#SECRET_HTTPS_PASSWORD_VERSION=v1 -#COMPOSE_FILE="$COMPOSE_FILE:compose.https.yml" -#RESTIC_USER= +# Secret restic repository +# use a secret to store the RESTIC_REPO if the repository location contains a secret value +# i.E rest:https://user:SECRET_PASSWORD@host:8000/ +# it overwrites the RESTIC_REPO variable +#SECRET_RESTIC_REPO_VERSION=v1 +#COMPOSE_FILE="$COMPOSE_FILE:compose.secret.yml" diff --git a/README.md b/README.md index 661fbda..e772270 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,23 @@ abra app secret insert ssh_key v1 """$(cat backupkey) """ ``` +### Restic REST server Storage + +You can simply set the `RESTIC_REPO` variable to your REST server URL `rest:http://host:8000/`. +If you access the REST server with a password `rest:https://user:pass@host:8000/` you should hide the whole URL containing the password inside a secret. +Uncomment these lines: +``` +SECRET_RESTIC_REPO_VERSION=v1 +COMPOSE_FILE="$COMPOSE_FILE:compose.secret.yml" +``` +Add your REST server url as secret: +``` +`abra app secret insert restic_repo v1 "rest:https://user:pass@host:8000/"` +``` +The secret will overwrite the `RESTIC_REPO` variable. + + +See [restic REST docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) for more information. ## Usage diff --git a/backupbot.py b/backupbot.py index a15d848..0f4286c 100755 --- a/backupbot.py +++ b/backupbot.py @@ -26,17 +26,21 @@ def cli(loglevel, service, repository): global SERVICE if service: SERVICE = service.replace('.', '_') + if repository: + os.environ['RESTIC_REPO'] = repository if loglevel: numeric_level = getattr(logging, loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level) export_secrets() - init_repo(repository) + init_repo() -def init_repo(repository): - restic.repository = repository +def init_repo(): + repo = os.environ['RESTIC_REPO'] + logging.debug(f"set restic repository location: {repo}") + restic.repository = repo restic.password_file = '/var/run/secrets/restic_password' try: restic.cat.config() @@ -50,10 +54,12 @@ def init_repo(repository): def export_secrets(): for env in os.environ: - if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'): + if env.endswith('FILE') and not "COMPOSE_FILE" in env: logging.debug(f"exported secret: {env}") with open(os.environ[env]) as file: - os.environ[env.removesuffix('_FILE')] = file.read() + secret = file.read() + os.environ[env.removesuffix('_FILE')] = secret + # logging.debug(f"Read secret value: {secret}") @cli.command() diff --git a/compose.secret.yml b/compose.secret.yml new file mode 100644 index 0000000..ab649ae --- /dev/null +++ b/compose.secret.yml @@ -0,0 +1,13 @@ +--- +version: "3.8" +services: + app: + environment: + - RESTIC_REPO_FILE=/run/secrets/restic_repo + secrets: + - restic_repo + +secrets: + restic_repo: + external: true + name: ${STACK_NAME}_restic_repo_${SECRET_RESTIC_REPO_VERSION} From 4240318d2065e1e34e91e2b099978cd80d885ed9 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 4 Oct 2023 13:42:33 +0200 Subject: [PATCH 27/31] remove package versions, to avoid conflicts --- entrypoint.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 4493054..8d1048c 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,11 +2,7 @@ set -e -o pipefail -apk add --upgrade --no-cache \ - restic=0.15.2-r3 \ - bash=5.2.15-r5 \ - python3=3.11.5-r0 \ - py3-pip=23.1.2-r0 +apk add --upgrade --no-cache restic bash python3 py3-pip # Todo use requirements file with specific versions pip install click==8.1.7 docker==6.1.3 resticpy==1.0.2 From 972a2c2314c9478df935709a94bbbcea4d659a92 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 4 Oct 2023 17:11:55 +0200 Subject: [PATCH 28/31] extend download to download the secrets or all app volumes at once --- backupbot.py | 74 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/backupbot.py b/backupbot.py index 0f4286c..47f48b4 100755 --- a/backupbot.py +++ b/backupbot.py @@ -195,7 +195,10 @@ def ls(snapshot, path): def list_files(snapshot, path): - cmd = restic.cat.base_command() + ['ls', snapshot] + cmd = restic.cat.base_command() + ['ls'] + if SERVICE: + cmd = cmd + ['--tag', SERVICE] + cmd.append(snapshot) if path: cmd.append(path) output = restic.internal.command_executor.execute(cmd) @@ -207,18 +210,63 @@ def list_files(snapshot, path): @cli.command() @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH') -def download(snapshot, path): - path = path.removesuffix('/') - files = list_files(snapshot, path) - filetype = [f.get('type') for f in files if f.get('path') == path][0] - cmd = restic.cat.base_command() + ['dump', snapshot, path] - output = subprocess.run(cmd, capture_output=True).stdout - filename = "/tmp/" + Path(path).name - if filetype == 'dir': - filename = filename + ".tar" - with open(filename, "wb") as file: - file.write(output) - print(filename) +@click.option('volumes', '--volumes', '-v', is_flag=True) +@click.option('secrets', '--secrets', '-c', is_flag=True) +def download(snapshot, path, volumes, secrets): + if sum(map(bool, [path, volumes, secrets])) != 1: + logging.error("Please specify exactly one of '--path', '--volumes', '--secrets'") + exit(1) + if path: + path = path.removesuffix('/') + files = list_files(snapshot, path) + filetype = [f.get('type') for f in files if f.get('path') == path][0] + filename = "/tmp/" + Path(path).name + if filetype == 'dir': + filename = filename + ".tar" + output = dump(snapshot, path) + with open(filename, "wb") as file: + file.write(output) + print(filename) + elif volumes: + if not SERVICE: + logging.error("Please specify '--host' when using '--volumes'") + exit(1) + filename = f"/tmp/{SERVICE}.tar" + files = list_files(snapshot, VOLUME_PATH) + for f in files[1:]: + path = f[ 'path' ] + if SERVICE in path and f['type'] == 'dir': + content = dump(snapshot, path) + # Concatenate tar files (extract with tar -xi) + with open(filename, "ab") as file: + file.write(content) + elif secrets: + if not SERVICE: + logging.error("Please specify '--host' when using '--secrets'") + exit(1) + filename = f"/tmp/SECRETS_{SERVICE}.json" + files = list_files(snapshot, SECRET_PATH) + secrets = {} + for f in files[1:]: + path = f[ 'path' ] + if SERVICE in path and f['type'] == 'file': + secret = dump(snapshot, path).decode() + secrets[path.removeprefix(SERVICE)] = secret + with open(filename, "w") as file: + json.dump(secrets, file) + print(filename) + +def dump(snapshot, path): + cmd = restic.cat.base_command() + ['dump'] + if SERVICE: + cmd = cmd + ['--tag', SERVICE] + cmd = cmd +[snapshot, path] + logging.debug(f"Dumping {path} from snapshot '{snapshot}'") + output = subprocess.run(cmd, capture_output=True) + if output.returncode: + logging.error(f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}") + exit(1) + return output.stdout if __name__ == '__main__': From bb1237f9ade28eff9a39120925f45311d06e3c80 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 4 Oct 2023 18:48:06 +0200 Subject: [PATCH 29/31] fix secret name --- backupbot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backupbot.py b/backupbot.py index 47f48b4..2675cd6 100755 --- a/backupbot.py +++ b/backupbot.py @@ -251,7 +251,8 @@ def download(snapshot, path, volumes, secrets): path = f[ 'path' ] if SERVICE in path and f['type'] == 'file': secret = dump(snapshot, path).decode() - secrets[path.removeprefix(SERVICE)] = secret + secret_name = path.removeprefix(f'{SECRET_PATH}{SERVICE}_') + secrets[secret_name] = secret with open(filename, "w") as file: json.dump(secrets, file) print(filename) From b3cbb8bb46fdda7138041a8fb035501dadcad070 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 4 Oct 2023 19:11:42 +0200 Subject: [PATCH 30/31] rm unused compose.https.yml as its replaced with compose.secret.yml --- compose.https.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 compose.https.yml diff --git a/compose.https.yml b/compose.https.yml deleted file mode 100644 index f8cfe20..0000000 --- a/compose.https.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -version: "3.8" -services: - app: - environment: - - HTTPS_PASSWORD_FILE=/run/secrets/https_password - - RESTIC_USER - secrets: - - source: https_password - mode: 0400 - -secrets: - https_password: - external: true - name: ${STACK_NAME}_https_password_${SECRET_HTTPS_PASSWORD_VERSION} From d25688f312150327e53090e9849503c9c0c70296 Mon Sep 17 00:00:00 2001 From: Moritz Date: Mon, 9 Oct 2023 12:53:28 +0200 Subject: [PATCH 31/31] add backupbot label --- compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yml b/compose.yml index 13c591d..888c3e3 100644 --- a/compose.yml +++ b/compose.yml @@ -18,6 +18,7 @@ services: labels: - coop-cloud.${STACK_NAME}.version=0.1.0+latest - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300} + - coop-cloud.backupbot.enabled=true configs: - source: entrypoint target: /entrypoint.sh