Compare commits

...

37 Commits

Author SHA1 Message Date
7e8c598eb5 Update .env.sample 2024-08-11 03:33:49 +00:00
cc049b858b README: new image location 2024-07-05 16:30:40 +02:00
b7bc8ed58f fix: ignore none swarm container #52 2024-07-05 14:41:50 +02:00
68e37f5c23 Merge pull request 'Add dockerfile, and compose.yml to use it' (#49) from feature/dockerfile into main
Reviewed-on: coop-cloud/backup-bot-two#49
2024-06-01 03:36:07 +00:00
3wc
4d39d84733 Switch ENTRYPOINT to try to resolve loop on start 2024-06-01 03:35:52 +00:00
3wc
e5b9bc0446 Update requirements 2024-06-01 03:35:52 +00:00
3wc
ec4c4509dc Make entrypoint executable 2024-06-01 03:35:52 +00:00
3wc
26162a9e38 Add --break-system-packages, surely we don't need a virtualenv 2024-06-01 03:35:52 +00:00
3wc
bd581fd8d7 Move entrypoint script into Docker image 2024-06-01 03:35:52 +00:00
3wc
e77432e3ab Move /entrypoint.sh to Dockerfile 2024-06-01 03:35:52 +00:00
3wc
001a654e37 Remove redundant stuff from entrypoint 2024-06-01 03:35:52 +00:00
3wc
c5574edc54 Whoops, wrong image 2024-06-01 03:35:52 +00:00
3wc
50e4d68717 Switch to backup-bot-two image 2024-06-01 03:35:52 +00:00
3wc
c7830ceb6f Whoops skip shellcheck 2024-06-01 03:35:52 +00:00
3wc
b6f859efbb Reinstate Docker image 2024-06-01 03:35:52 +00:00
7f14698824 change loglever to warning for not running container 2024-05-06 11:31:40 +02:00
2a9a98172f Add debug infos 2024-04-30 15:27:17 +02:00
282215cf9c Add debug infos 2024-04-30 14:59:59 +02:00
ae7a14b6f1 Fix usage of RESTIC_REPOSITORY_FILE #51 2024-04-30 14:51:46 +02:00
8acdb20e5b Fix loghandler 2024-04-29 14:18:32 +02:00
5582744073 Fix usage of RESTIC_REPOSITORY_FILE #51 2024-04-29 14:16:13 +02:00
3wc
84d606fa80 Add CHANGELOG.md
[ci skip]
2024-04-09 22:51:09 -03:00
7865907811 fix push notification precendence race condition 2024-03-08 15:42:00 +01:00
dc66c02e23 make run_cron cmd independent from push_success_notifiaction 2024-02-13 11:53:27 +01:00
f730c70bfe feat: add retry option 2024-01-18 18:01:30 +01:00
faa7ae3dd1 fix Readme 2024-01-17 20:36:06 +01:00
79eeec428a Push Notifications #24 2024-01-16 19:40:31 +01:00
4164760dc6 Sepcify secret and volume donwload via env, fixes #44 2024-01-11 18:46:58 +01:00
e644679b8b Clearer service name in warning message. Fixes #46 2024-01-11 18:39:26 +01:00
0c587ac926 add spaces for missing snapshot, fixes #45 2024-01-11 18:34:58 +01:00
65686cd891 Fix python package install error 2023-12-19 01:16:12 +01:00
ac055c932e fix: remove bash/sh wrapping 2023-12-13 18:27:12 +01:00
64328c79b1 make --noninteractive a flag 2023-12-12 13:39:26 +01:00
15275b2571 structured json logging with -m flag 2023-11-23 20:16:15 +01:00
4befebba38 Merge pull request 'fix removing quotes' (#40) from p4u1/backup-bot-two:fix-quotes into main
Reviewed-on: coop-cloud/backup-bot-two#40
2023-11-11 08:15:12 +00:00
d2087a441e fix removing quotes 2023-11-11 08:55:12 +01:00
f4d96b0875 update README 2023-11-10 20:04:05 +01:00
9 changed files with 190 additions and 100 deletions

View File

@ -2,11 +2,16 @@
kind: pipeline kind: pipeline
name: linters name: linters
steps: steps:
- name: run shellcheck - name: publish image
image: koalaman/shellcheck-alpine image: plugins/docker
commands: settings:
- shellcheck backup.sh username: 3wordchant
password:
trigger: from_secret: git_coopcloud_tech_token_3wc
branch: repo: git.coopcloud.tech/coop-cloud/backup-bot-two
- main tags: 2.0.0
registry: git.coopcloud.tech
when:
event:
exclude:
- pull_request

View File

@ -4,10 +4,17 @@ SECRET_RESTIC_PASSWORD_VERSION=v1
COMPOSE_FILE=compose.yml COMPOSE_FILE=compose.yml
DOMAIN=backup-bot-two.example.com
RESTIC_REPOSITORY=/backups/restic RESTIC_REPOSITORY=/backups/restic
CRON_SCHEDULE='30 3 * * *' CRON_SCHEDULE='30 3 * * *'
# Push Notifiactions
#PUSH_URL_START=https://status.example.com/api/push/xxxxxxxxxx?status=up&msg=start
#PUSH_URL_SUCCESS=https://status.example.com/api/push/xxxxxxxxxx?status=up&msg=OK
#PUSH_URL_FAIL=https://status.example.com/api/push/xxxxxxxxxx?status=down&msg=fail
# swarm-cronjob, instead of built-in cron # swarm-cronjob, instead of built-in cron
#COMPOSE_FILE="$COMPOSE_FILE:compose.swarm-cronjob.yml" #COMPOSE_FILE="$COMPOSE_FILE:compose.swarm-cronjob.yml"

6
CHANGELOG.md Normal file
View File

@ -0,0 +1,6 @@
# Change log
## 2.0.0 (unreleased)
- Rewrite from Bash to Python
- Add support for push notifications (#24)

11
Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM docker:24.0.7-dind
RUN apk add --upgrade --no-cache restic bash python3 py3-pip py3-click py3-docker-py py3-json-logger curl
# Todo use requirements file with specific versions
RUN pip install --break-system-packages resticpy==1.0.2
COPY backupbot.py /usr/bin/backup
COPY entrypoint.sh /entrypoint.sh
ENTRYPOINT /entrypoint.sh

View File

@ -10,7 +10,7 @@ Automatically take backups from all volumes of running Docker Swarm services and
* **Category**: Utilities * **Category**: Utilities
* **Status**: 0, work-in-progress * **Status**: 0, work-in-progress
* **Image**: [`thecoopcloud/backup-bot-two`](https://hub.docker.com/r/thecoopcloud/backup-bot-two), 4, upstream * **Image**: [`git.coopcloud.tech/coop-cloud/backup-bot-two`](https://git.coopcloud.tech/coop-cloud/-/packages/container/backup-bot-two), 4, upstream
* **Healthcheck**: No * **Healthcheck**: No
* **Backups**: N/A * **Backups**: N/A
* **Email**: N/A * **Email**: N/A
@ -38,12 +38,12 @@ Backupbot II tries to help, by
* `abra app new backup-bot-two` * `abra app new backup-bot-two`
* `abra app config <app-name>` * `abra app config <app-name>`
- set storage options. Either configure `CRON_SCHEDULE`, or set up `swarm-cronjob` - set storage options. Either configure `CRON_SCHEDULE`, or set up `swarm-cronjob`
* `abra app secret generate -a <app_name>` * `abra app secret generate -a <backupbot_name>`
* `abra app deploy <app-name>` * `abra app deploy <app-name>`
## Configuration ## Configuration
Per default Backupbot stores the backups locally in the repository `/backups/restic`, which is accessible as volume at `/var/lib/docker/volumes/<app_name>_backups/_data/restic/` Per default Backupbot stores the backups locally in the repository `/backups/restic`, which is accessible as volume at `/var/lib/docker/volumes/<backupbot_name>_backups/_data/restic/`
The backup location can be changed using the `RESTIC_REPOSITORY` env variable. The backup location can be changed using the `RESTIC_REPOSITORY` env variable.
@ -57,7 +57,7 @@ AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml" COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml"
``` ```
and add your `<SECRET_ACCESS_KEY>` as docker secret: and add your `<SECRET_ACCESS_KEY>` as docker secret:
`abra app secret insert <app_name> aws_secret_access_key v1 <SECRET_ACCESS_KEY>` `abra app secret insert <backupbot_name> aws_secret_access_key v1 <SECRET_ACCESS_KEY>`
See [restic s3 docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for more information. See [restic s3 docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#amazon-s3) for more information.
@ -79,9 +79,10 @@ Add the key to your `authorized_keys`:
`ssh-copy-id -i backupkey <user>@<hostname>` `ssh-copy-id -i backupkey <user>@<hostname>`
Add your `SSH_KEY` as docker secret: Add your `SSH_KEY` as docker secret:
``` ```
abra app secret insert <app_name> ssh_key v1 """$(cat backupkey) abra app secret insert <backupbot_name> ssh_key v1 """$(cat backupkey)
""" """
``` ```
> Attention: This command needs to be executed exactly as stated above, because it places a trailing newline at the end, if this is missing you will get the following error: `Load key "/run/secrets/ssh_key": error in libcrypto`
### Restic REST server Storage ### Restic REST server Storage
@ -94,67 +95,81 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.secret.yml"
``` ```
Add your REST server url as secret: Add your REST server url as secret:
``` ```
`abra app secret insert <app_name> restic_repo v1 "rest:https://user:pass@host:8000/"` `abra app secret insert <backupbot_name> restic_repo v1 "rest:https://user:pass@host:8000/"`
``` ```
The secret will overwrite the `RESTIC_REPOSITORY` variable. The secret will overwrite the `RESTIC_REPOSITORY` variable.
See [restic REST docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) for more information. See [restic REST docs](https://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) for more information.
## Push notifications
The following env variables can be used to setup push notifications for backups. `PUSH_URL_START` is requested just before the backups starts, `PUSH_URL_SUCCESS` is only requested if the backup was successful and if the backup fails `PUSH_URL_FAIL` will be requested.
Each variable is optional and independent of the other.
```
PUSH_URL_START=https://status.example.com/api/push/xxxxxxxxxx?status=up&msg=start
PUSH_URL_SUCCESS=https://status.example.com/api/push/xxxxxxxxxx?status=up&msg=OK
PUSH_URL_FAIL=https://status.example.com/api/push/xxxxxxxxxx?status=down&msg=fail
```
## Usage ## Usage
Run the cronjob that creates a backup, including the push notifications and docker logging:
`abra app cmd <backupbot_name> app run_cron`
Create a backup of all apps: Create a backup of all apps:
`abra app run <app_name> app -- backup create` `abra app run <backupbot_name> app -- backup create`
> The apps to backup up need to be deployed > The apps to backup up need to be deployed
Create an individual backup: Create an individual backup:
`abra app run <app_name> app -- backup --host <target_app_name> create` `abra app run <backupbot_name> app -- backup --host <target_app_name> create`
Create a backup to a local repository: Create a backup to a local repository:
`abra app run <app_name> app -- backup create -r /backups/restic` `abra app run <backupbot_name> app -- backup create -r /backups/restic`
> It is recommended to shutdown/undeploy an app before restoring the data > It is recommended to shutdown/undeploy an app before restoring the data
Restore the latest snapshot of all including apps: Restore the latest snapshot of all including apps:
`abra app run <app_name> app -- backup restore` `abra app run <backupbot_name> app -- backup restore`
Restore a specific snapshot of an individual app: Restore a specific snapshot of an individual app:
`abra app run <app_name> app -- backup --host <target_app_name> restore --snapshot <snapshot_id>` `abra app run <backupbot_name> app -- backup --host <target_app_name> restore --snapshot <snapshot_id>`
Show all snapshots: Show all snapshots:
`abra app run <app_name> app -- backup snapshots` `abra app run <backupbot_name> app -- backup snapshots`
Show all snapshots containing a specific app: Show all snapshots containing a specific app:
`abra app run <app_name> app -- backup --host <target_app_name> snapshots` `abra app run <backupbot_name> app -- backup --host <target_app_name> snapshots`
Show all files inside the latest snapshot (can be very verbose): Show all files inside the latest snapshot (can be very verbose):
`abra app run <app_name> app -- backup ls` `abra app run <backupbot_name> app -- backup ls`
Show specific files inside a selected snapshot: Show specific files inside a selected snapshot:
`abra app run <app_name> app -- backup ls --snapshot <snapshot_id> --path /var/lib/docker/volumes/` `abra app run <backupbot_name> app -- backup ls --snapshot <snapshot_id> --path /var/lib/docker/volumes/`
Download files from a snapshot: Download files from a snapshot:
``` ```
filename=$(abra app run <app_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>) filename=$(abra app run <backupbot_name> app -- backup download --snapshot <snapshot_id> --path <absolute_path>)
abra app cp <app_name> app:$filename . abra app cp <backupbot_name> app:$filename .
``` ```
## Run restic ## Run restic
``` ```
abra app run <app_name> app bash abra app run <backupbot_name> app bash
export AWS_SECRET_ACCESS_KEY=$(cat $AWS_SECRET_ACCESS_KEY_FILE) export AWS_SECRET_ACCESS_KEY=$(cat $AWS_SECRET_ACCESS_KEY_FILE)
export RESTIC_PASSWORD=$(cat $RESTIC_PASSWORD_FILE) export RESTIC_PASSWORD=$(cat $RESTIC_PASSWORD_FILE)
restic snapshots restic snapshots

View File

@ -1,3 +1,10 @@
export ENTRYPOINT_VERSION=v1
export BACKUPBOT_VERSION=v1 export BACKUPBOT_VERSION=v1
export SSH_CONFIG_VERSION=v1 export SSH_CONFIG_VERSION=v1
run_cron () {
schedule="$(crontab -l | tr -s " " | cut -d ' ' -f-5)"
rm -f /tmp/backup.log
echo "* * * * * $(crontab -l | tr -s " " | cut -d ' ' -f6-)" | crontab -
while [ ! -f /tmp/backup.log ]; do sleep 1; done
echo "$schedule $(crontab -l | tr -s " " | cut -d ' ' -f6-)" | crontab -
}

View File

@ -1,6 +1,7 @@
#!/usr/bin/python3 #!/usr/bin/python3
import os import os
import sys
import click import click
import json import json
import subprocess import subprocess
@ -9,22 +10,40 @@ import docker
import restic import restic
import tarfile import tarfile
import io import io
from pythonjsonlogger import jsonlogger
from datetime import datetime, timezone from datetime import datetime, timezone
from restic.errors import ResticFailedError from restic.errors import ResticFailedError
from pathlib import Path from pathlib import Path
from shutil import copyfile, rmtree from shutil import copyfile, rmtree
# logging.basicConfig(level=logging.INFO)
VOLUME_PATH = "/var/lib/docker/volumes/" VOLUME_PATH = "/var/lib/docker/volumes/"
SECRET_PATH = '/secrets/' SECRET_PATH = '/secrets/'
SERVICE = None SERVICE = None
logger = logging.getLogger("backupbot")
logging.addLevelName(55, 'SUMMARY')
setattr(logging, 'SUMMARY', 55)
setattr(logger, 'summary', lambda message, *args, **
kwargs: logger.log(55, message, *args, **kwargs))
def handle_exception(exc_type, exc_value, exc_traceback):
if issubclass(exc_type, KeyboardInterrupt):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
logger.critical("Uncaught exception", exc_info=(
exc_type, exc_value, exc_traceback))
sys.excepthook = handle_exception
@click.group() @click.group()
@click.option('-l', '--log', 'loglevel') @click.option('-l', '--log', 'loglevel')
@click.option('-m', '--machine-logs', 'machine_logs', is_flag=True)
@click.option('service', '--host', '-h', envvar='SERVICE') @click.option('service', '--host', '-h', envvar='SERVICE')
@click.option('repository', '--repo', '-r', envvar='RESTIC_REPOSITORY', required=True) @click.option('repository', '--repo', '-r', envvar='RESTIC_REPOSITORY')
def cli(loglevel, service, repository): def cli(loglevel, service, repository, machine_logs):
global SERVICE global SERVICE
if service: if service:
SERVICE = service.replace('.', '_') SERVICE = service.replace('.', '_')
@ -34,22 +53,33 @@ def cli(loglevel, service, repository):
numeric_level = getattr(logging, loglevel.upper(), None) numeric_level = getattr(logging, loglevel.upper(), None)
if not isinstance(numeric_level, int): if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel) raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level) logger.setLevel(numeric_level)
logHandler = logging.StreamHandler()
if machine_logs:
formatter = jsonlogger.JsonFormatter(
"%(levelname)s %(filename)s %(lineno)s %(process)d %(message)s", rename_fields={"levelname": "message_type"})
logHandler.setFormatter(formatter)
logger.addHandler(logHandler)
export_secrets() export_secrets()
init_repo() init_repo()
def init_repo(): def init_repo():
repo = os.environ['RESTIC_REPOSITORY'] if repo:= os.environ.get('RESTIC_REPOSITORY_FILE'):
logging.debug(f"set restic repository location: {repo}") # RESTIC_REPOSITORY_FILE and RESTIC_REPOSITORY are mutually exclusive
restic.repository = repo del os.environ['RESTIC_REPOSITORY']
else:
repo = os.environ['RESTIC_REPOSITORY']
restic.repository = repo
logger.debug(f"set restic repository location: {repo}")
restic.password_file = '/var/run/secrets/restic_password' restic.password_file = '/var/run/secrets/restic_password'
try: try:
restic.cat.config() restic.cat.config()
except ResticFailedError as error: except ResticFailedError as error:
if 'unable to open config file' in str(error): if 'unable to open config file' in str(error):
result = restic.init() result = restic.init()
logging.info(f"Initialized restic repo: {result}") logger.info(f"Initialized restic repo: {result}")
else: else:
raise error raise error
@ -57,27 +87,28 @@ def init_repo():
def export_secrets(): def export_secrets():
for env in os.environ: for env in os.environ:
if env.endswith('FILE') and not "COMPOSE_FILE" in env: if env.endswith('FILE') and not "COMPOSE_FILE" in env:
logging.debug(f"exported secret: {env}") logger.debug(f"exported secret: {env}")
with open(os.environ[env]) as file: with open(os.environ[env]) as file:
secret = file.read() secret = file.read()
os.environ[env.removesuffix('_FILE')] = secret os.environ[env.removesuffix('_FILE')] = secret
# logging.debug(f"Read secret value: {secret}") # logger.debug(f"Read secret value: {secret}")
@cli.command() @cli.command()
def create(): @click.option('retries', '--retries', '-r', envvar='RETRIES', default=1)
def create(retries):
pre_commands, post_commands, backup_paths, apps = get_backup_cmds() pre_commands, post_commands, backup_paths, apps = get_backup_cmds()
copy_secrets(apps) copy_secrets(apps)
backup_paths.append(SECRET_PATH) backup_paths.append(SECRET_PATH)
run_commands(pre_commands) run_commands(pre_commands)
backup_volumes(backup_paths, apps) backup_volumes(backup_paths, apps, int(retries))
run_commands(post_commands) run_commands(post_commands)
def get_backup_cmds(): def get_backup_cmds():
client = docker.from_env() client = docker.from_env()
container_by_service = { container_by_service = {
c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} c.labels.get('com.docker.swarm.service.name'): c for c in client.containers.list()}
backup_paths = set() backup_paths = set()
backup_apps = set() backup_apps = set()
pre_commands = {} pre_commands = {}
@ -86,6 +117,7 @@ def get_backup_cmds():
for s in services: for s in services:
labels = s.attrs['Spec']['Labels'] labels = s.attrs['Spec']['Labels']
if (backup := labels.get('backupbot.backup')) and bool(backup): if (backup := labels.get('backupbot.backup')) and bool(backup):
# volumes: s.attrs['Spec']['TaskTemplate']['ContainerSpec']['Mounts'][0]['Source']
stack_name = labels['com.docker.stack.namespace'] stack_name = labels['com.docker.stack.namespace']
# Remove this lines to backup only a specific service # Remove this lines to backup only a specific service
# This will unfortenately decrease restice performance # This will unfortenately decrease restice performance
@ -94,8 +126,8 @@ def get_backup_cmds():
backup_apps.add(stack_name) backup_apps.add(stack_name)
backup_paths = backup_paths.union( backup_paths = backup_paths.union(
Path(VOLUME_PATH).glob(f"{stack_name}_*")) Path(VOLUME_PATH).glob(f"{stack_name}_*"))
if not (container:= container_by_service.get(s.name)): if not (container := container_by_service.get(s.name)):
logging.error( logger.error(
f"Container {s.name} is not running, hooks can not be executed") f"Container {s.name} is not running, hooks can not be executed")
continue continue
if prehook := labels.get('backupbot.backup.pre-hook'): if prehook := labels.get('backupbot.backup.pre-hook'):
@ -106,28 +138,30 @@ def get_backup_cmds():
def copy_secrets(apps): def copy_secrets(apps):
#TODO: check if it is deployed # TODO: check if it is deployed
rmtree(SECRET_PATH, ignore_errors=True) rmtree(SECRET_PATH, ignore_errors=True)
os.mkdir(SECRET_PATH) os.mkdir(SECRET_PATH)
client = docker.from_env() client = docker.from_env()
container_by_service = { container_by_service = {
c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} c.labels.get('com.docker.swarm.service.name'): c for c in client.containers.list()}
services = client.services.list() services = client.services.list()
for s in services: for s in services:
app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace'] app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace']
if (app_name in apps and if (app_name in apps and
(app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'))): (app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'))):
if not container_by_service.get(s.name): if not container_by_service.get(s.name):
logging.error( logger.warning(
f"Container {s.name} is not running, secrets can not be copied.") f"Container {s.name} is not running, secrets can not be copied.")
continue continue
container_id = container_by_service[s.name].id container_id = container_by_service[s.name].id
for sec in app_secs: for sec in app_secs:
src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}' src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}'
if not Path(src).exists(): if not Path(src).exists():
logging.error(f"For the secret {sec['SecretName']} the file {src} does not exist for {s.name}") logger.error(
f"For the secret {sec['SecretName']} the file {src} does not exist for {s.name}")
continue continue
dst = SECRET_PATH + sec['SecretName'] dst = SECRET_PATH + sec['SecretName']
logger.debug("Copy Secret {sec['SecretName']}")
copyfile(src, dst) copyfile(src, dst)
@ -136,37 +170,44 @@ def run_commands(commands):
if not command: if not command:
continue continue
# Remove bash/sh wrapping # Remove bash/sh wrapping
command = command.removeprefix('bash -c').removeprefix('sh -c') command = command.removeprefix('bash -c').removeprefix('sh -c').removeprefix(' ')
# Remove quotes surrounding the command # Remove quotes surrounding the command
if (len(command) >= 2 and command[0] == command[-1] and (command[0] == "'" or command[0] == '"')): if (len(command) >= 2 and command[0] == command[-1] and (command[0] == "'" or command[0] == '"')):
command[1:-1] command = command[1:-1]
# Use bash's pipefail to return exit codes inside a pipe to prevent silent failure # Use bash's pipefail to return exit codes inside a pipe to prevent silent failure
command = f"bash -c 'set -o pipefail;{command}'" command = f"bash -c 'set -o pipefail;{command}'"
logging.info(f"run command in {container.name}:") logger.info(f"run command in {container.name}:")
logging.info(command) logger.info(command)
result = container.exec_run(command) result = container.exec_run(command)
if result.exit_code: if result.exit_code:
logging.error( logger.error(
f"Failed to run command {command} in {container.name}: {result.output.decode()}") f"Failed to run command {command} in {container.name}: {result.output.decode()}")
else: else:
logging.info(result.output.decode()) logger.info(result.output.decode())
def backup_volumes(backup_paths, apps, dry_run=False): def backup_volumes(backup_paths, apps, retries, dry_run=False):
try: while True:
result = restic.backup(backup_paths, dry_run=dry_run, tags=apps) try:
print(result) logger.info("Start volume backup")
logging.info(result) logger.debug(backup_paths)
except ResticFailedError as error: result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
logging.error(f"Backup failed for {apps}. Could not Backup these paths: {backup_paths}") logger.summary("backup finished", extra=result)
logging.error(error) return
exit(1) except ResticFailedError as error:
logger.error(
f"Backup failed for {apps}. Could not Backup these paths: {backup_paths}")
logger.error(error, exc_info=True)
if retries > 0:
retries -= 1
else:
exit(1)
@cli.command() @cli.command()
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
@click.option('target', '--target', '-t', envvar='TARGET', default='/') @click.option('target', '--target', '-t', envvar='TARGET', default='/')
@click.option('noninteractive', '--noninteractive', envvar='NONINTERACTIVE', default=False) @click.option('noninteractive', '--noninteractive', envvar='NONINTERACTIVE', is_flag=True)
def restore(snapshot, target, noninteractive): def restore(snapshot, target, noninteractive):
# Todo: recommend to shutdown the container # Todo: recommend to shutdown the container
service_paths = VOLUME_PATH service_paths = VOLUME_PATH
@ -174,7 +215,7 @@ def restore(snapshot, target, noninteractive):
service_paths = service_paths + f'{SERVICE}_*' service_paths = service_paths + f'{SERVICE}_*'
snapshots = restic.snapshots(snapshot_id=snapshot) snapshots = restic.snapshots(snapshot_id=snapshot)
if not snapshot: if not snapshot:
logging.error("No Snapshots with ID {snapshots}") logger.error("No Snapshots with ID {snapshots}")
exit(1) exit(1)
if not noninteractive: if not noninteractive:
snapshot_date = datetime.fromisoformat(snapshots[0]['time']) snapshot_date = datetime.fromisoformat(snapshots[0]['time'])
@ -186,12 +227,13 @@ def restore(snapshot, target, noninteractive):
f"THIS COMMAND WILL IRREVERSIBLY OVERWRITES {target}{service_paths.removeprefix('/')}") f"THIS COMMAND WILL IRREVERSIBLY OVERWRITES {target}{service_paths.removeprefix('/')}")
prompt = input("Type YES (uppercase) to continue: ") prompt = input("Type YES (uppercase) to continue: ")
if prompt != 'YES': if prompt != 'YES':
logging.error("Restore aborted") logger.error("Restore aborted")
exit(1) exit(1)
print(f"Restoring Snapshot {snapshot} of {service_paths} at {target}") print(f"Restoring Snapshot {snapshot} of {service_paths} at {target}")
# TODO: use tags if no snapshot is selected, to use a snapshot including SERVICE
result = restic.restore(snapshot_id=snapshot, result = restic.restore(snapshot_id=snapshot,
include=service_paths, target_dir=target) include=service_paths, target_dir=target)
logging.debug(result) logger.debug(result)
@cli.command() @cli.command()
@ -205,8 +247,9 @@ def snapshots():
if no_snapshots: if no_snapshots:
err_msg = "No Snapshots found" err_msg = "No Snapshots found"
if SERVICE: if SERVICE:
err_msg += f' for app {SERVICE}' service_name = SERVICE.replace('_', '.')
logging.warning(err_msg) err_msg += f' for app {service_name}'
logger.warning(err_msg)
@cli.command() @cli.command()
@ -230,10 +273,10 @@ def list_files(snapshot, path):
output = restic.internal.command_executor.execute(cmd) output = restic.internal.command_executor.execute(cmd)
except ResticFailedError as error: except ResticFailedError as error:
if 'no snapshot found' in str(error): if 'no snapshot found' in str(error):
err_msg = f'There is no snapshot {snapshot}' err_msg = f'There is no snapshot "{snapshot}"'
if SERVICE: if SERVICE:
err_msg += f'for the app {SERVICE}' err_msg += f' for the app "{SERVICE}"'
logging.error(err_msg) logger.error(err_msg)
exit(1) exit(1)
else: else:
raise error raise error
@ -245,8 +288,8 @@ def list_files(snapshot, path):
@cli.command() @cli.command()
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
@click.option('volumes', '--volumes', '-v', is_flag=True) @click.option('volumes', '--volumes', '-v', envvar='VOLUMES')
@click.option('secrets', '--secrets', '-c', is_flag=True) @click.option('secrets', '--secrets', '-c', is_flag=True, envvar='SECRETS')
def download(snapshot, path, volumes, secrets): def download(snapshot, path, volumes, secrets):
file_dumps = [] file_dumps = []
if not any([path, volumes, secrets]): if not any([path, volumes, secrets]):
@ -264,7 +307,7 @@ def download(snapshot, path, volumes, secrets):
file_dumps.append((binary_output, tarinfo)) file_dumps.append((binary_output, tarinfo))
if volumes: if volumes:
if not SERVICE: if not SERVICE:
logging.error("Please specify '--host' when using '--volumes'") logger.error("Please specify '--host' when using '--volumes'")
exit(1) exit(1)
files = list_files(snapshot, VOLUME_PATH) files = list_files(snapshot, VOLUME_PATH)
for f in files[1:]: for f in files[1:]:
@ -277,7 +320,7 @@ def download(snapshot, path, volumes, secrets):
file_dumps.append((binary_output, tarinfo)) file_dumps.append((binary_output, tarinfo))
if secrets: if secrets:
if not SERVICE: if not SERVICE:
logging.error("Please specify '--host' when using '--secrets'") logger.error("Please specify '--host' when using '--secrets'")
exit(1) exit(1)
filename = f"{SERVICE}.json" filename = f"{SERVICE}.json"
files = list_files(snapshot, SECRET_PATH) files = list_files(snapshot, SECRET_PATH)
@ -297,7 +340,8 @@ def download(snapshot, path, volumes, secrets):
for binary_output, tarinfo in file_dumps: for binary_output, tarinfo in file_dumps:
tar.addfile(tarinfo, fileobj=io.BytesIO(binary_output)) tar.addfile(tarinfo, fileobj=io.BytesIO(binary_output))
size = get_formatted_size('/tmp/backup.tar.gz') size = get_formatted_size('/tmp/backup.tar.gz')
print(f"Backup has been written to /tmp/backup.tar.gz with a size of {size}") print(
f"Backup has been written to /tmp/backup.tar.gz with a size of {size}")
def get_formatted_size(file_path): def get_formatted_size(file_path):
@ -318,7 +362,7 @@ def dump(snapshot, path):
print(f"Dumping {path} from snapshot '{snapshot}'") print(f"Dumping {path} from snapshot '{snapshot}'")
output = subprocess.run(cmd, capture_output=True) output = subprocess.run(cmd, capture_output=True)
if output.returncode: if output.returncode:
logging.error( logger.error(
f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}") f"error while dumping {path} from snapshot '{snapshot}': {output.stderr}")
exit(1) exit(1)
return output.stdout return output.stdout

View File

@ -2,7 +2,7 @@
version: "3.8" version: "3.8"
services: services:
app: app:
image: docker:24.0.7-dind image: git.coopcloud.tech/coop-cloud/backup-bot-two:2.0.0
volumes: volumes:
- "/var/run/docker.sock:/var/run/docker.sock" - "/var/run/docker.sock:/var/run/docker.sock"
- "/var/lib/docker/volumes/:/var/lib/docker/volumes/" - "/var/lib/docker/volumes/:/var/lib/docker/volumes/"
@ -19,14 +19,7 @@ services:
- coop-cloud.${STACK_NAME}.version=0.1.0+latest - coop-cloud.${STACK_NAME}.version=0.1.0+latest
- coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300} - coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-300}
- coop-cloud.backupbot.enabled=true - coop-cloud.backupbot.enabled=true
configs: #entrypoint: ['tail', '-f','/dev/null']
- source: entrypoint
target: /entrypoint.sh
mode: 0555
- source: backupbot
target: /usr/bin/backup
mode: 0555
entrypoint: ['/entrypoint.sh']
healthcheck: healthcheck:
test: "pgrep crond" test: "pgrep crond"
interval: 30s interval: 30s
@ -41,11 +34,3 @@ secrets:
volumes: volumes:
backups: backups:
configs:
entrypoint:
name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
file: entrypoint.sh
backupbot:
name: ${STACK_NAME}_backupbot_${BACKUPBOT_VERSION}
file: backupbot.py

24
entrypoint.sh Normal file → Executable file
View File

@ -1,11 +1,6 @@
#!/bin/sh #!/bin/sh
set -e -o pipefail set -e
apk add --upgrade --no-cache restic bash python3 py3-pip
# Todo use requirements file with specific versions
pip install click==8.1.7 docker==6.1.3 resticpy==1.0.2
if [ -n "$SSH_HOST_KEY" ] if [ -n "$SSH_HOST_KEY" ]
then then
@ -14,7 +9,22 @@ fi
cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}" cron_schedule="${CRON_SCHEDULE:?CRON_SCHEDULE not set}"
echo "$cron_schedule backup create" | crontab - if [ -n "$PUSH_URL_START" ]
then
push_start_notification="curl -s '$PUSH_URL_START' &&"
fi
if [ -n "$PUSH_URL_FAIL" ]
then
push_fail_notification="|| curl -s '$PUSH_URL_FAIL'"
fi
if [ -n "$PUSH_URL_SUCCESS" ]
then
push_notification=" && (grep -q 'backup finished' /tmp/backup.log && curl -s '$PUSH_URL_SUCCESS' $push_fail_notification)"
fi
echo "$cron_schedule $push_start_notification backup --machine-logs create 2>&1 | tee /tmp/backup.log $push_notification" | crontab -
crontab -l crontab -l
crond -f -d8 -L /dev/stdout crond -f -d8 -L /dev/stdout