Backup volumes from host instead of copying paths
* Backupbot will now copy all volumes from a service with backupbot.enabled = 'true' label from the /var/lib/docker/volumes/ path directly. This reduces the resource overhead of copying stuff from one volume to another. Recipes need to be adjustet that db-dumps are saved into a volume now! * Remove the Dockerfile and move stuff into a entrypoint. This simplifies the whole versioning thing and makes this "just" a recipe Co-authored-by: Moritz < moritz.m@local-it.org>
This commit is contained in:
parent
451c511554
commit
6355f3572f
16
.drone.yml
16
.drone.yml
|
@ -7,22 +7,6 @@ steps:
|
||||||
commands:
|
commands:
|
||||||
- shellcheck backup.sh
|
- shellcheck backup.sh
|
||||||
|
|
||||||
- name: publish image
|
|
||||||
image: plugins/docker
|
|
||||||
settings:
|
|
||||||
auto_tag: true
|
|
||||||
username: thecoopcloud
|
|
||||||
password:
|
|
||||||
from_secret: thecoopcloud_password
|
|
||||||
repo: thecoopcloud/backup-bot-two
|
|
||||||
tags: latest
|
|
||||||
depends_on:
|
|
||||||
- run shellcheck
|
|
||||||
when:
|
|
||||||
event:
|
|
||||||
exclude:
|
|
||||||
- pull_request
|
|
||||||
|
|
||||||
trigger:
|
trigger:
|
||||||
branch:
|
branch:
|
||||||
- main
|
- main
|
||||||
|
|
13
Dockerfile
13
Dockerfile
|
@ -1,13 +0,0 @@
|
||||||
FROM docker:24.0.6-dind
|
|
||||||
|
|
||||||
RUN apk add --upgrade --no-cache \
|
|
||||||
bash \
|
|
||||||
curl \
|
|
||||||
jq \
|
|
||||||
restic
|
|
||||||
|
|
||||||
COPY backup.sh /usr/bin/backup.sh
|
|
||||||
COPY setup-cron.sh /usr/bin/setup-cron.sh
|
|
||||||
RUN chmod +x /usr/bin/backup.sh /usr/bin/setup-cron.sh
|
|
||||||
|
|
||||||
ENTRYPOINT [ "/usr/bin/setup-cron.sh" ]
|
|
12
README.md
12
README.md
|
@ -4,7 +4,7 @@
|
||||||
|
|
||||||
_This Time, It's Easily Configurable_
|
_This Time, It's Easily Configurable_
|
||||||
|
|
||||||
Automatically take backups from running Docker Swarm services into a volume.
|
Automatically take backups from all volumes of running Docker Swarm services and runs pre- and post commands.
|
||||||
|
|
||||||
## Background
|
## Background
|
||||||
|
|
||||||
|
@ -49,15 +49,13 @@ services:
|
||||||
db:
|
db:
|
||||||
deploy:
|
deploy:
|
||||||
labels:
|
labels:
|
||||||
backupbot.backup: "true"
|
backupbot.backup: ${BACKUP:-"true"}
|
||||||
backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /tmp/dump/dump.db'
|
backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /volume_path/dump.db'
|
||||||
backupbot.backup.post-hook: "rm -rf /tmp/dump/dump.db"
|
backupbot.backup.post-hook: "rm -rf /volume_path/dump.db"
|
||||||
backupbot.backup.path: "/tmp/dump/,/etc/foo/"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- `backupbot.backup` -- set to `true` to back up this service (REQUIRED)
|
- `backupbot.backup` -- set to `true` to back up this service (REQUIRED)
|
||||||
- `backupbot.backup.path` -- comma separated list of file paths within the service to copy (REQUIRED)
|
- `backupbot.backup.pre-hook` -- command to run before copying files (optional), save all dumps into the volumes
|
||||||
- `backupbot.backup.pre-hook` -- command to run before copying files (optional)
|
|
||||||
- `backupbot.backup.post-hook` -- command to run after copying files (optional)
|
- `backupbot.backup.post-hook` -- command to run after copying files (optional)
|
||||||
|
|
||||||
As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly.
|
As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly.
|
||||||
|
|
|
@ -0,0 +1,2 @@
|
||||||
|
export ENTRYPOINT_VERSION=v1
|
||||||
|
export BACKUP_VERSION=v1
|
139
backup.sh
139
backup.sh
|
@ -1,139 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
server_name="${SERVER_NAME:?SERVER_NAME not set}"
|
|
||||||
|
|
||||||
restic_password_file="${RESTIC_PASSWORD_FILE:?RESTIC_PASSWORD_FILE not set}"
|
|
||||||
|
|
||||||
restic_host="${RESTIC_HOST:?RESTIC_HOST not set}"
|
|
||||||
|
|
||||||
backup_path="${BACKUP_DEST:?BACKUP_DEST not set}"
|
|
||||||
|
|
||||||
# shellcheck disable=SC2153
|
|
||||||
ssh_key_file="${SSH_KEY_FILE}"
|
|
||||||
s3_key_file="${AWS_SECRET_ACCESS_KEY_FILE}"
|
|
||||||
# shellcheck disable=SC2153
|
|
||||||
https_password_file="${HTTPS_PASSWORD_FILE}"
|
|
||||||
|
|
||||||
restic_repo=
|
|
||||||
restic_extra_options=
|
|
||||||
|
|
||||||
if [ -n "$ssh_key_file" ] && [ -f "$ssh_key_file" ]; then
|
|
||||||
restic_repo="sftp:$restic_host:/$server_name"
|
|
||||||
|
|
||||||
# Only check server against provided SSH_HOST_KEY, if set
|
|
||||||
if [ -n "$SSH_HOST_KEY" ]; then
|
|
||||||
tmpfile=$(mktemp)
|
|
||||||
echo "$SSH_HOST_KEY" >>"$tmpfile"
|
|
||||||
echo "using host key $SSH_HOST_KEY"
|
|
||||||
ssh_options="-o 'UserKnownHostsFile $tmpfile'"
|
|
||||||
elif [ "$SSH_HOST_KEY_DISABLE" = "1" ]; then
|
|
||||||
echo "disabling SSH host key checking"
|
|
||||||
ssh_options="-o 'StrictHostKeyChecking=No'"
|
|
||||||
else
|
|
||||||
echo "neither SSH_HOST_KEY nor SSH_HOST_KEY_DISABLE set"
|
|
||||||
fi
|
|
||||||
restic_extra_options="sftp.command=ssh $ssh_options -i $ssh_key_file $restic_host -s sftp"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$s3_key_file" ] && [ -f "$s3_key_file" ] && [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
|
||||||
AWS_SECRET_ACCESS_KEY="$(cat "${s3_key_file}")"
|
|
||||||
export AWS_SECRET_ACCESS_KEY
|
|
||||||
restic_repo="s3:$restic_host:/$server_name"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$https_password_file" ] && [ -f "$https_password_file" ]; then
|
|
||||||
HTTPS_PASSWORD="$(cat "${https_password_file}")"
|
|
||||||
export HTTPS_PASSWORD
|
|
||||||
restic_user="${RESTIC_USER:?RESTIC_USER not set}"
|
|
||||||
restic_repo="rest:https://$restic_user:$HTTPS_PASSWORD@$restic_host"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$restic_repo" ]; then
|
|
||||||
echo "you must configure either SFTP, S3, or HTTPS storage, see README"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "restic_repo: $restic_repo"
|
|
||||||
|
|
||||||
# Pre-bake-in some default restic options
|
|
||||||
_restic() {
|
|
||||||
if [ -z "$restic_extra_options" ]; then
|
|
||||||
# shellcheck disable=SC2068
|
|
||||||
restic -p "$restic_password_file" \
|
|
||||||
--quiet -r "$restic_repo" \
|
|
||||||
$@
|
|
||||||
else
|
|
||||||
# shellcheck disable=SC2068
|
|
||||||
restic -p "$restic_password_file" \
|
|
||||||
--quiet -r "$restic_repo" \
|
|
||||||
-o "$restic_extra_options" \
|
|
||||||
$@
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
if [ -n "$SERVICES_OVERRIDE" ]; then
|
|
||||||
# this is fine because docker service names should never include spaces or
|
|
||||||
# glob characters
|
|
||||||
# shellcheck disable=SC2206
|
|
||||||
services=($SERVICES_OVERRIDE)
|
|
||||||
else
|
|
||||||
mapfile -t services < <(docker service ls --format '{{ .Name }}')
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ \ $*\ != *\ --skip-backup\ * ]]; then
|
|
||||||
rm -rf "${backup_path}"
|
|
||||||
|
|
||||||
for service in "${services[@]}"; do
|
|
||||||
echo "service: $service"
|
|
||||||
details=$(docker service inspect "$service" --format "{{ json .Spec.Labels }}")
|
|
||||||
if echo "$details" | jq -r '.["backupbot.backup"]' | grep -q 'true'; then
|
|
||||||
pre=$(echo "$details" | jq -r '.["backupbot.backup.pre-hook"]')
|
|
||||||
post=$(echo "$details" | jq -r '.["backupbot.backup.post-hook"]')
|
|
||||||
path=$(echo "$details" | jq -r '.["backupbot.backup.path"]')
|
|
||||||
|
|
||||||
if [ "$path" = "null" ]; then
|
|
||||||
echo "ERROR: missing 'path' for $service"
|
|
||||||
continue # or maybe exit?
|
|
||||||
fi
|
|
||||||
|
|
||||||
container=$(docker container ls -f "name=$service" --format '{{ .ID }}')
|
|
||||||
|
|
||||||
echo "backing up $service"
|
|
||||||
|
|
||||||
if [ "$pre" != "null" ]; then
|
|
||||||
# run the precommand
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
docker exec "$container" sh -c "$pre"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# run the backup
|
|
||||||
for p in ${path//,/ }; do
|
|
||||||
# creates the parent folder, so `docker cp` has reliable behaviour no matter if $p ends with `/` or `/.`
|
|
||||||
dir=$backup_path/$service/$(dirname "$p")
|
|
||||||
test -d "$dir" || mkdir -p "$dir"
|
|
||||||
docker cp -a "$container:$p" "$dir/$(basename "$p")"
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ "$post" != "null" ]; then
|
|
||||||
# run the postcommand
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
docker exec "$container" sh -c "$post"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# check if restic repo exists, initialise if not
|
|
||||||
if [ -z "$(_restic cat config)" ] 2>/dev/null; then
|
|
||||||
echo "initializing restic repo"
|
|
||||||
_restic init
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ \ $*\ != *\ --skip-upload\ * ]]; then
|
|
||||||
_restic backup --host "$server_name" --tag coop-cloud "$backup_path"
|
|
||||||
|
|
||||||
if [ "$REMOVE_BACKUP_VOLUME_AFTER_UPLOAD" -eq 1 ]; then
|
|
||||||
echo "Cleaning up ${backup_path}"
|
|
||||||
rm -rf "${backup_path:?}"/*
|
|
||||||
fi
|
|
||||||
fi
|
|
24
compose.yml
24
compose.yml
|
@ -2,11 +2,10 @@
|
||||||
version: "3.8"
|
version: "3.8"
|
||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
image: thecoopcloud/backup-bot-two:latest
|
image: docker:24.0.2-dind
|
||||||
# build: .
|
|
||||||
volumes:
|
volumes:
|
||||||
- "/var/run/docker.sock:/var/run/docker.sock"
|
- "/var/run/docker.sock:/var/run/docker.sock"
|
||||||
- "backups:/backups"
|
- "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
|
||||||
environment:
|
environment:
|
||||||
- CRON_SCHEDULE
|
- CRON_SCHEDULE
|
||||||
- RESTIC_REPO
|
- RESTIC_REPO
|
||||||
|
@ -25,11 +24,24 @@ services:
|
||||||
- "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
|
- "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
|
||||||
- "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
- "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||||
- coop-cloud.${STACK_NAME}.version=0.1.0+latest
|
- coop-cloud.${STACK_NAME}.version=0.1.0+latest
|
||||||
|
configs:
|
||||||
volumes:
|
- source: entrypoint
|
||||||
backups:
|
target: /entrypoint.sh
|
||||||
|
mode: 0555
|
||||||
|
- source: backup
|
||||||
|
target: /backup.sh
|
||||||
|
mode: 0555
|
||||||
|
entrypoint: ['/entrypoint.sh']
|
||||||
|
|
||||||
secrets:
|
secrets:
|
||||||
restic_password:
|
restic_password:
|
||||||
external: true
|
external: true
|
||||||
name: ${STACK_NAME}_restic_password_${SECRET_RESTIC_PASSWORD_VERSION}
|
name: ${STACK_NAME}_restic_password_${SECRET_RESTIC_PASSWORD_VERSION}
|
||||||
|
|
||||||
|
configs:
|
||||||
|
entrypoint:
|
||||||
|
name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
|
||||||
|
file: entrypoint.sh
|
||||||
|
backup:
|
||||||
|
name: ${STACK_NAME}_backup_${BACKUP_VERSION}
|
||||||
|
file: backup.sh
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
#!/bin/bash
|
#!/bin/sh
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
set -o pipefail
|
|
||||||
|
apk add --upgrade --no-cache bash curl jq restic
|
||||||
|
|
||||||
cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
|
cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
|
||||||
|
|
||||||
echo "$cron_schedule /usr/bin/backup.sh" | crontab -
|
echo "$cron_schedule /backup.sh" | crontab -
|
||||||
crontab -l
|
crontab -l
|
||||||
|
|
||||||
crond -f -d8 -L /dev/stdout
|
crond -f -d8 -L /dev/stdout
|
Loading…
Reference in New Issue