Backup volumes from host instead of copying paths
* Backupbot will now copy all volumes from a service with backupbot.enabled = 'true' label from the /var/lib/docker/volumes/ path directly. This reduces the resource overhead of copying stuff from one volume to another. Recipes need to be adjustet that db-dumps are saved into a volume now! * Remove the Dockerfile and move stuff into a entrypoint. This simplifies the whole versioning thing and makes this "just" a recipe Co-authored-by: Moritz < moritz.m@local-it.org>
This commit is contained in:
parent
451c511554
commit
6355f3572f
16
.drone.yml
16
.drone.yml
|
|
@ -7,22 +7,6 @@ steps:
|
|||
commands:
|
||||
- shellcheck backup.sh
|
||||
|
||||
- name: publish image
|
||||
image: plugins/docker
|
||||
settings:
|
||||
auto_tag: true
|
||||
username: thecoopcloud
|
||||
password:
|
||||
from_secret: thecoopcloud_password
|
||||
repo: thecoopcloud/backup-bot-two
|
||||
tags: latest
|
||||
depends_on:
|
||||
- run shellcheck
|
||||
when:
|
||||
event:
|
||||
exclude:
|
||||
- pull_request
|
||||
|
||||
trigger:
|
||||
branch:
|
||||
- main
|
||||
|
|
|
|||
13
Dockerfile
13
Dockerfile
|
|
@ -1,13 +0,0 @@
|
|||
FROM docker:24.0.6-dind
|
||||
|
||||
RUN apk add --upgrade --no-cache \
|
||||
bash \
|
||||
curl \
|
||||
jq \
|
||||
restic
|
||||
|
||||
COPY backup.sh /usr/bin/backup.sh
|
||||
COPY setup-cron.sh /usr/bin/setup-cron.sh
|
||||
RUN chmod +x /usr/bin/backup.sh /usr/bin/setup-cron.sh
|
||||
|
||||
ENTRYPOINT [ "/usr/bin/setup-cron.sh" ]
|
||||
12
README.md
12
README.md
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
_This Time, It's Easily Configurable_
|
||||
|
||||
Automatically take backups from running Docker Swarm services into a volume.
|
||||
Automatically take backups from all volumes of running Docker Swarm services and runs pre- and post commands.
|
||||
|
||||
## Background
|
||||
|
||||
|
|
@ -49,15 +49,13 @@ services:
|
|||
db:
|
||||
deploy:
|
||||
labels:
|
||||
backupbot.backup: "true"
|
||||
backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /tmp/dump/dump.db'
|
||||
backupbot.backup.post-hook: "rm -rf /tmp/dump/dump.db"
|
||||
backupbot.backup.path: "/tmp/dump/,/etc/foo/"
|
||||
backupbot.backup: ${BACKUP:-"true"}
|
||||
backupbot.backup.pre-hook: 'mysqldump -u root -p"$(cat /run/secrets/db_root_password)" -f /volume_path/dump.db'
|
||||
backupbot.backup.post-hook: "rm -rf /volume_path/dump.db"
|
||||
```
|
||||
|
||||
- `backupbot.backup` -- set to `true` to back up this service (REQUIRED)
|
||||
- `backupbot.backup.path` -- comma separated list of file paths within the service to copy (REQUIRED)
|
||||
- `backupbot.backup.pre-hook` -- command to run before copying files (optional)
|
||||
- `backupbot.backup.pre-hook` -- command to run before copying files (optional), save all dumps into the volumes
|
||||
- `backupbot.backup.post-hook` -- command to run after copying files (optional)
|
||||
|
||||
As in the above example, you can reference Docker Secrets, e.g. for looking up database passwords, by reading the files in `/run/secrets` directly.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
export ENTRYPOINT_VERSION=v1
|
||||
export BACKUP_VERSION=v1
|
||||
139
backup.sh
139
backup.sh
|
|
@ -1,139 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
server_name="${SERVER_NAME:?SERVER_NAME not set}"
|
||||
|
||||
restic_password_file="${RESTIC_PASSWORD_FILE:?RESTIC_PASSWORD_FILE not set}"
|
||||
|
||||
restic_host="${RESTIC_HOST:?RESTIC_HOST not set}"
|
||||
|
||||
backup_path="${BACKUP_DEST:?BACKUP_DEST not set}"
|
||||
|
||||
# shellcheck disable=SC2153
|
||||
ssh_key_file="${SSH_KEY_FILE}"
|
||||
s3_key_file="${AWS_SECRET_ACCESS_KEY_FILE}"
|
||||
# shellcheck disable=SC2153
|
||||
https_password_file="${HTTPS_PASSWORD_FILE}"
|
||||
|
||||
restic_repo=
|
||||
restic_extra_options=
|
||||
|
||||
if [ -n "$ssh_key_file" ] && [ -f "$ssh_key_file" ]; then
|
||||
restic_repo="sftp:$restic_host:/$server_name"
|
||||
|
||||
# Only check server against provided SSH_HOST_KEY, if set
|
||||
if [ -n "$SSH_HOST_KEY" ]; then
|
||||
tmpfile=$(mktemp)
|
||||
echo "$SSH_HOST_KEY" >>"$tmpfile"
|
||||
echo "using host key $SSH_HOST_KEY"
|
||||
ssh_options="-o 'UserKnownHostsFile $tmpfile'"
|
||||
elif [ "$SSH_HOST_KEY_DISABLE" = "1" ]; then
|
||||
echo "disabling SSH host key checking"
|
||||
ssh_options="-o 'StrictHostKeyChecking=No'"
|
||||
else
|
||||
echo "neither SSH_HOST_KEY nor SSH_HOST_KEY_DISABLE set"
|
||||
fi
|
||||
restic_extra_options="sftp.command=ssh $ssh_options -i $ssh_key_file $restic_host -s sftp"
|
||||
fi
|
||||
|
||||
if [ -n "$s3_key_file" ] && [ -f "$s3_key_file" ] && [ -n "$AWS_ACCESS_KEY_ID" ]; then
|
||||
AWS_SECRET_ACCESS_KEY="$(cat "${s3_key_file}")"
|
||||
export AWS_SECRET_ACCESS_KEY
|
||||
restic_repo="s3:$restic_host:/$server_name"
|
||||
fi
|
||||
|
||||
if [ -n "$https_password_file" ] && [ -f "$https_password_file" ]; then
|
||||
HTTPS_PASSWORD="$(cat "${https_password_file}")"
|
||||
export HTTPS_PASSWORD
|
||||
restic_user="${RESTIC_USER:?RESTIC_USER not set}"
|
||||
restic_repo="rest:https://$restic_user:$HTTPS_PASSWORD@$restic_host"
|
||||
fi
|
||||
|
||||
if [ -z "$restic_repo" ]; then
|
||||
echo "you must configure either SFTP, S3, or HTTPS storage, see README"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "restic_repo: $restic_repo"
|
||||
|
||||
# Pre-bake-in some default restic options
|
||||
_restic() {
|
||||
if [ -z "$restic_extra_options" ]; then
|
||||
# shellcheck disable=SC2068
|
||||
restic -p "$restic_password_file" \
|
||||
--quiet -r "$restic_repo" \
|
||||
$@
|
||||
else
|
||||
# shellcheck disable=SC2068
|
||||
restic -p "$restic_password_file" \
|
||||
--quiet -r "$restic_repo" \
|
||||
-o "$restic_extra_options" \
|
||||
$@
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -n "$SERVICES_OVERRIDE" ]; then
|
||||
# this is fine because docker service names should never include spaces or
|
||||
# glob characters
|
||||
# shellcheck disable=SC2206
|
||||
services=($SERVICES_OVERRIDE)
|
||||
else
|
||||
mapfile -t services < <(docker service ls --format '{{ .Name }}')
|
||||
fi
|
||||
|
||||
if [[ \ $*\ != *\ --skip-backup\ * ]]; then
|
||||
rm -rf "${backup_path}"
|
||||
|
||||
for service in "${services[@]}"; do
|
||||
echo "service: $service"
|
||||
details=$(docker service inspect "$service" --format "{{ json .Spec.Labels }}")
|
||||
if echo "$details" | jq -r '.["backupbot.backup"]' | grep -q 'true'; then
|
||||
pre=$(echo "$details" | jq -r '.["backupbot.backup.pre-hook"]')
|
||||
post=$(echo "$details" | jq -r '.["backupbot.backup.post-hook"]')
|
||||
path=$(echo "$details" | jq -r '.["backupbot.backup.path"]')
|
||||
|
||||
if [ "$path" = "null" ]; then
|
||||
echo "ERROR: missing 'path' for $service"
|
||||
continue # or maybe exit?
|
||||
fi
|
||||
|
||||
container=$(docker container ls -f "name=$service" --format '{{ .ID }}')
|
||||
|
||||
echo "backing up $service"
|
||||
|
||||
if [ "$pre" != "null" ]; then
|
||||
# run the precommand
|
||||
# shellcheck disable=SC2086
|
||||
docker exec "$container" sh -c "$pre"
|
||||
fi
|
||||
|
||||
# run the backup
|
||||
for p in ${path//,/ }; do
|
||||
# creates the parent folder, so `docker cp` has reliable behaviour no matter if $p ends with `/` or `/.`
|
||||
dir=$backup_path/$service/$(dirname "$p")
|
||||
test -d "$dir" || mkdir -p "$dir"
|
||||
docker cp -a "$container:$p" "$dir/$(basename "$p")"
|
||||
done
|
||||
|
||||
if [ "$post" != "null" ]; then
|
||||
# run the postcommand
|
||||
# shellcheck disable=SC2086
|
||||
docker exec "$container" sh -c "$post"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# check if restic repo exists, initialise if not
|
||||
if [ -z "$(_restic cat config)" ] 2>/dev/null; then
|
||||
echo "initializing restic repo"
|
||||
_restic init
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ \ $*\ != *\ --skip-upload\ * ]]; then
|
||||
_restic backup --host "$server_name" --tag coop-cloud "$backup_path"
|
||||
|
||||
if [ "$REMOVE_BACKUP_VOLUME_AFTER_UPLOAD" -eq 1 ]; then
|
||||
echo "Cleaning up ${backup_path}"
|
||||
rm -rf "${backup_path:?}"/*
|
||||
fi
|
||||
fi
|
||||
24
compose.yml
24
compose.yml
|
|
@ -2,11 +2,10 @@
|
|||
version: "3.8"
|
||||
services:
|
||||
app:
|
||||
image: thecoopcloud/backup-bot-two:latest
|
||||
# build: .
|
||||
image: docker:24.0.2-dind
|
||||
volumes:
|
||||
- "/var/run/docker.sock:/var/run/docker.sock"
|
||||
- "backups:/backups"
|
||||
- "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
|
||||
environment:
|
||||
- CRON_SCHEDULE
|
||||
- RESTIC_REPO
|
||||
|
|
@ -25,11 +24,24 @@ services:
|
|||
- "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
|
||||
- "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||
- coop-cloud.${STACK_NAME}.version=0.1.0+latest
|
||||
|
||||
volumes:
|
||||
backups:
|
||||
configs:
|
||||
- source: entrypoint
|
||||
target: /entrypoint.sh
|
||||
mode: 0555
|
||||
- source: backup
|
||||
target: /backup.sh
|
||||
mode: 0555
|
||||
entrypoint: ['/entrypoint.sh']
|
||||
|
||||
secrets:
|
||||
restic_password:
|
||||
external: true
|
||||
name: ${STACK_NAME}_restic_password_${SECRET_RESTIC_PASSWORD_VERSION}
|
||||
|
||||
configs:
|
||||
entrypoint:
|
||||
name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
|
||||
file: entrypoint.sh
|
||||
backup:
|
||||
name: ${STACK_NAME}_backup_${BACKUP_VERSION}
|
||||
file: backup.sh
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
#!/bin/bash
|
||||
#!/bin/sh
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
apk add --upgrade --no-cache bash curl jq restic
|
||||
|
||||
cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
|
||||
|
||||
echo "$cron_schedule /usr/bin/backup.sh" | crontab -
|
||||
echo "$cron_schedule /backup.sh" | crontab -
|
||||
crontab -l
|
||||
|
||||
crond -f -d8 -L /dev/stdout
|
||||
Loading…
Reference in New Issue