From a990dc27c7c7322c751467d205db9322c0bc1858 Mon Sep 17 00:00:00 2001 From: 3wc <3wc@doesthisthing.work> Date: Wed, 10 Nov 2021 22:01:57 +0200 Subject: [PATCH] SSH host keys, split out swarm-cronjob --- .env.sample | 4 ++ README.md | 4 +- backup.sh | 87 ++++++++++++++++++++++---------------- compose.ssh.yml | 5 ++- compose.swarmm-cronjob.yml | 14 ++++++ compose.yml | 9 ---- 6 files changed, 75 insertions(+), 48 deletions(-) create mode 100644 compose.swarmm-cronjob.yml diff --git a/.env.sample b/.env.sample index 6aa0e3d..9eec677 100644 --- a/.env.sample +++ b/.env.sample @@ -9,8 +9,12 @@ RESTIC_HOST=minio.example.com CRON_SCHEDULE='*/5 * * * *' +# swarm-cronjob, instead of built-in cron +#COMPOSE_FILE="$COMPOSE_FILE:compose.swarm-cronjob.yml" + # SSH storage #SECRET_SSH_KEY_VERSION=v1 +#SSH_HOST_KEY="hostname ssh-rsa AAAAB3... #COMPOSE_FILE="$COMPOSE_FILE:compose.ssh.yml" # S3 storage diff --git a/README.md b/README.md index 82a666a..92be861 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ A first stab: - [ ] SSH remote storage - [ ] Add SSH key handling - [ ] SSH host key checking -- [ ] S3 remote storage -- [ ] Re-add `crond` support +- [x] S3 remote storage +- [x] Re-add `crond` support Future: - [ ] Continuous linting with shellcheck diff --git a/backup.sh b/backup.sh index 82cb95c..5e2278a 100755 --- a/backup.sh +++ b/backup.sh @@ -16,7 +16,18 @@ restic_extra_options= if [ -n "$ssh_key_file" ] && [ -f "$ssh_key_file" ]; then restic_repo="sftp:$restic_host:/$server_name" - restic_extra_options="sftp.command=ssh -i $ssh_key_file $restic_host -s sftp" + + # Only check server against provided SSH_HOST_KEY, if set + if [ -n "$SSH_HOST_KEY" ]; then + tmpfile=$(mktemp) + echo "$SSH_HOST_KEY" >> "$tmpfile" + ssh_options="-o 'UserKnownHostsFile $tmpfile'" + elif [ "$SSH_HOST_KEY_DISABLE" = "1" ]; then + ssh_options="-o 'StrictHostKeyChecking=No'" + else + echo "Neither SSH_HOST_KEY nor SSH_HOST_KEY_DISABLE set" + fi + restic_extra_options="sftp.command=ssh $ssh_options -i $ssh_key_file $restic_host -s sftp" fi if [ -n "$s3_key_file" ] && [ -f "$s3_key_file" ] && [ -n "$AWS_ACCESS_KEY_ID" ]; then @@ -57,45 +68,49 @@ else mapfile -t services < <(docker service ls --format '{{ .Name }}') fi -for service in "${services[@]}"; do - echo "service: $service" - details=$(docker service inspect "$service" --format "{{ json .Spec.Labels }}") - if echo "$details" | jq -r '.["backupbot.backup"]' | grep -q 'true'; then - pre=$(echo "$details" | jq -r '.["backupbot.backup.pre-hook"]') - post=$(echo "$details" | jq -r '.["backupbot.backup.post-hook"]') - path=$(echo "$details" | jq -r '.["backupbot.backup.path"]') - - if [ "$path" = "null" ]; then - echo "ERROR: missing 'path' for $service" - continue # or maybe exit? - fi +if [[ \ $*\ != *\ --skip-backup\ * ]]; then + for service in "${services[@]}"; do + echo "service: $service" + details=$(docker service inspect "$service" --format "{{ json .Spec.Labels }}") + if echo "$details" | jq -r '.["backupbot.backup"]' | grep -q 'true'; then + pre=$(echo "$details" | jq -r '.["backupbot.backup.pre-hook"]') + post=$(echo "$details" | jq -r '.["backupbot.backup.post-hook"]') + path=$(echo "$details" | jq -r '.["backupbot.backup.path"]') + + if [ "$path" = "null" ]; then + echo "ERROR: missing 'path' for $service" + continue # or maybe exit? + fi - container=$(docker container ls -f "name=$service" --format '{{ .ID }}') - - echo "backing up $service" - test -d "$backup_path/$service" || mkdir "$backup_path/$service" - - if [ "$pre" != "null" ]; then - # run the precommand - # shellcheck disable=SC2086 - docker exec "$container" sh -c "$pre" - fi + container=$(docker container ls -f "name=$service" --format '{{ .ID }}') + + echo "backing up $service" + test -d "$backup_path/$service" || mkdir "$backup_path/$service" + + if [ "$pre" != "null" ]; then + # run the precommand + # shellcheck disable=SC2086 + docker exec "$container" sh -c "$pre" + fi - # run the backup - docker cp "$container:$path" "$backup_path/$service" + # run the backup + docker cp "$container:$path" "$backup_path/$service" - if [ "$post" != "null" ]; then - # run the postcommand - # shellcheck disable=SC2086 - docker exec "$container" sh -c "$post" + if [ "$post" != "null" ]; then + # run the postcommand + # shellcheck disable=SC2086 + docker exec "$container" sh -c "$post" + fi fi + done + + # check if restic repo exists, initialise if not + if [ -z "$(_restic cat config)" ] 2>/dev/null; then + echo "initializing restic repo" + _restic init fi -done - -# check if restic repo exists, initialise if not -if [ -z "$(_restic cat config)" ] 2>/dev/null; then - echo "initializing restic repo" - _restic init fi -_restic backup --tag coop-cloud "$backup_path" +if [[ \ $*\ != *\ --skip-upload\ * ]]; then + _restic backup --tag coop-cloud "$backup_path" +fi diff --git a/compose.ssh.yml b/compose.ssh.yml index 9667101..ee143d0 100644 --- a/compose.ssh.yml +++ b/compose.ssh.yml @@ -4,8 +4,11 @@ services: app: environment: - SSH_KEY_FILE=/run/secrets/ssh_key + - SSH_HOST_KEY secrets: - - ssh_key + - source: ssh_key + mode: 0400 + entrypoint: [ "/usr/bin/backup.sh" ] secrets: ssh_key: diff --git a/compose.swarmm-cronjob.yml b/compose.swarmm-cronjob.yml new file mode 100644 index 0000000..df997e0 --- /dev/null +++ b/compose.swarmm-cronjob.yml @@ -0,0 +1,14 @@ +--- +version: "3.8" +services: + app: + deploy: + mode: replicated + replicas: 0 + labels: + - "swarm.cronjob.enable=true" + # Note(3wc): every 5m, testing + - "swarm.cronjob.schedule=*/5 * * * *" + # Note(3wc): blank label to be picked up by `abra recipe sync` + restart_policy: + condition: none diff --git a/compose.yml b/compose.yml index b536cea..9050c8f 100644 --- a/compose.yml +++ b/compose.yml @@ -16,18 +16,9 @@ services: - SERVER_NAME secrets: - restic_password - deploy: - mode: replicated - replicas: 0 labels: - - "swarm.cronjob.enable=true" - # Note(3wc): every 5m, testing - - "swarm.cronjob.schedule=*/5 * * * *" - # Note(3wc): blank label to be picked up by `abra recipe sync` - coop-cloud.${STACK_NAME}.app.version= - restart_policy: - condition: none volumes: backups: