Backup volumes from host instead of copying paths

* Backupbot will now copy all volumes from a service with
  backupbot.enabled = 'true' label from the /var/lib/docker/volumes/
  path directly. This reduces the resource overhead of copying
  stuff from one volume to another.
  Recipes need to be adjustet that db-dumps are saved into a volume
  now!
* Remove the Dockerfile and move stuff into a entrypoint. This
  simplifies the whole versioning thing and makes this "just"
  a recipe

Co-authored-by: Moritz < moritz.m@local-it.org>

.drone.yml

@@ -7,22 +7,6 @@ steps:
     commands:
       - shellcheck backup.sh
 
-  - name: publish image
-    image: plugins/docker
-    settings:
-      auto_tag: true
-      username: thecoopcloud
-      password:
-        from_secret: thecoopcloud_password
-      repo: thecoopcloud/backup-bot-two
-      tags: latest
-    depends_on:
-      - run shellcheck
-    when:
-      event:
-        exclude:
-          - pull_request
-
 trigger:
   branch:
     - main

Dockerfile

@@ -1,13 +0,0 @@
-FROM docker:24.0.2-dind
-
-RUN apk add --upgrade --no-cache \
-  bash \
-  curl \
-  jq \
-  restic
-
-COPY backup.sh /usr/bin/backup.sh
-COPY setup-cron.sh /usr/bin/setup-cron.sh
-RUN chmod +x /usr/bin/backup.sh /usr/bin/setup-cron.sh
-
-ENTRYPOINT [ "/usr/bin/setup-cron.sh" ]

README.md

@@ -4,7 +4,7 @@
 
 _This Time, It's Easily Configurable_
 
-Automatically take backups from running Docker Swarm services into a volume.
+Automatically take backups from all volumes of running Docker Swarm services and runs pre- and post commands.
 
 ## Background
 
@@ -49,15 +49,13 @@ services:
   db:
     deploy:
       labels:
-        backupbot.backup: "true"
-        backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /tmp/dump/dump.db'
-        backupbot.backup.post-hook: "rm -rf /tmp/dump/dump.db"
-        backupbot.backup.path: "/tmp/dump/,/etc/foo/"
+        backupbot.backup: ${BACKUP:-"true"} 
+        backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /volume_path/dump.db'
+        backupbot.backup.post-hook: "rm -rf /volume_path/dump.db"
 ```
 
 - `backupbot.backup` -- set to `true` to back up this service (REQUIRED)
-- `backupbot.backup.path` -- comma separated list of file paths within the service to copy (REQUIRED)
-- `backupbot.backup.pre-hook` -- command to run before copying files (optional)
+- `backupbot.backup.pre-hook` -- command to run before copying files (optional), save all dumps into the volumes
 - `backupbot.backup.post-hook` -- command to run after copying files (optional)
 
 As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly.

abra.sh

@@ -0,0 +1,2 @@
+export ENTRYPOINT_VERSION=v1
+export BACKUP_VERSION=v1

backup.sh

@@ -1,12 +1,14 @@
 #!/bin/bash
 
+set -e
+
 server_name="${SERVER_NAME:?SERVER_NAME not set}"
 
 restic_password_file="${RESTIC_PASSWORD_FILE:?RESTIC_PASSWORD_FILE not set}"
 
 restic_host="${RESTIC_HOST:?RESTIC_HOST not set}"
 
-backup_path="${BACKUP_DEST:?BACKUP_DEST not set}"
+backup_paths=()
 
 # shellcheck disable=SC2153
 ssh_key_file="${SSH_KEY_FILE}"
@@ -71,8 +73,8 @@ else
 	mapfile -t services < <(docker service ls --format '{{ .Name }}')
 fi
 
+post_commands=()
 if [[ \ $*\  != *\ --skip-backup\ * ]]; then
-	rm -rf "${backup_path}"
 
 	for service in "${services[@]}"; do
 		echo "service: $service"
@@ -80,36 +82,21 @@ if [[ \ $*\  != *\ --skip-backup\ * ]]; then
 		if echo "$details" | jq -r '.["backupbot.backup"]' | grep -q 'true'; then
 			pre=$(echo "$details" | jq -r '.["backupbot.backup.pre-hook"]')
 			post=$(echo "$details" | jq -r '.["backupbot.backup.post-hook"]')
-			path=$(echo "$details" | jq -r '.["backupbot.backup.path"]')
-
-			if [ "$path" = "null" ]; then
-				echo "ERROR: missing 'path' for $service"
-				continue # or maybe exit?
-			fi
-
 			container=$(docker container ls -f "name=$service" --format '{{ .ID }}')
-
-			echo "backing up $service"
+			stack_name=$(echo "$details" | jq -r '.["com.docker.stack.namespace"]')
 
 			if [ "$pre" != "null" ]; then
 				# run the precommand
-				# shellcheck disable=SC2086
+				echo "executing precommand $pre in container $container"
 				docker exec "$container" sh -c "$pre"
 			fi
-
-			# run the backup
-			for p in ${path//,/ }; do
-				# creates the parent folder, so `docker cp` has reliable behaviour no matter if $p ends with `/` or `/.`
-				dir=$backup_path/$service/$(dirname "$p")
-				test -d "$dir" || mkdir -p "$dir"
-				docker cp -a "$container:$p" "$dir/$(basename "$p")"
-			done
-
 			if [ "$post" != "null" ]; then
-				# run the postcommand
-				# shellcheck disable=SC2086
-				docker exec "$container" sh -c "$post"
+				# append post command
+				post_commands+=("docker exec $container sh -c \"$post\"")
 			fi
+
+			# add volume paths to backup path
+			backup_paths+=(/var/lib/docker/volumes/${stack_name}_*)
 		fi
 	done
 
@@ -121,10 +108,11 @@ if [[ \ $*\  != *\ --skip-backup\ * ]]; then
 fi
 
 if [[ \ $*\  != *\ --skip-upload\ * ]]; then
-	_restic backup --host "$server_name" --tag coop-cloud "$backup_path"
-
-	if [ "$REMOVE_BACKUP_VOLUME_AFTER_UPLOAD" -eq 1 ]; then
-		echo "Cleaning up ${backup_path}"
-		rm -rf "${backup_path}"
-	fi
+	_restic backup --host "$server_name" --tag coop-cloud "${backup_paths[@]}"
 fi
+
+# run post commands
+for post in "${post_commands[@]}"; do
+	echo "executing postcommand $post"
+	eval "$post"
+done

compose.yml

@@ -2,11 +2,10 @@
 version: "3.8"
 services:
   app:
-    image: thecoopcloud/backup-bot-two:latest
-#    build: .
+    image: docker:24.0.2-dind
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
-      - "backups:/backups"
+      - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
     environment:
       - CRON_SCHEDULE
       - RESTIC_REPO
@@ -25,11 +24,24 @@ services:
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - coop-cloud.${STACK_NAME}.version=0.1.0+latest
-
-volumes:
-  backups:
+    configs:
+      - source: entrypoint
+        target: /entrypoint.sh
+        mode: 0555
+      - source: backup
+        target: /backup.sh
+        mode: 0555
+    entrypoint: ['/entrypoint.sh']
 
 secrets:
   restic_password:
     external: true
     name: ${STACK_NAME}_restic_password_${SECRET_RESTIC_PASSWORD_VERSION}
+
+configs:
+  entrypoint:
+    name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
+    file: entrypoint.sh
+  backup:
+    name: ${STACK_NAME}_backup_${BACKUP_VERSION}
+    file: backup.sh

entrypoint.sh

@@ -1,11 +1,12 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e
-set -o pipefail
+
+apk add --upgrade --no-cache bash curl jq restic
 
 cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
 
-echo "$cron_schedule /usr/bin/backup.sh" | crontab -
+echo "$cron_schedule /backup.sh" | crontab -
 crontab -l
 
 crond -f -d8 -L /dev/stdout