coop-cloud/backup-bot-two
coop-cloud
/
backup-bot-two
6
0
Fork 3
Code Issues 12 Pull Requests 1 Packages Projects Releases Wiki Activity

formatting

This commit is contained in:
Moritz 2023-09-22 16:50:45 +02:00
parent 5d4def6143
commit 15a552ef8b
1 changed files with 37 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
backupbot.py
View File

@ -10,12 +10,13 @@ import restic
from restic.errors import ResticFailedError from restic.errors import ResticFailedError
from pathlib import Path from pathlib import Path
from shutil import copyfile, rmtree from shutil import copyfile, rmtree
#logging.basicConfig(level=logging.INFO) # logging.basicConfig(level=logging.INFO)
VOLUME_PATH = "/var/lib/docker/volumes/" VOLUME_PATH = "/var/lib/docker/volumes/"
SECRET_PATH = '/secrets/' SECRET_PATH = '/secrets/'
SERVICE = None SERVICE = None
@click.group() @click.group()
@click.option('-l', '--log', 'loglevel') @click.option('-l', '--log', 'loglevel')
@click.option('service', '--host', '-h', envvar='SERVICE') @click.option('service', '--host', '-h', envvar='SERVICE')
@ -23,7 +24,7 @@ SERVICE = None
def cli(loglevel, service, repository): def cli(loglevel, service, repository):
global SERVICE global SERVICE
if service: if service:
SERVICE = service.replace('.','_') SERVICE = service.replace('.', '_')
if loglevel: if loglevel:
numeric_level = getattr(logging, loglevel.upper(), None) numeric_level = getattr(logging, loglevel.upper(), None)
if not isinstance(numeric_level, int): if not isinstance(numeric_level, int):
@ -45,6 +46,7 @@ def init_repo(repository):
else: else:
raise error raise error
def export_secrets(): def export_secrets():
for env in os.environ: for env in os.environ:
if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'): if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'):
@ -52,6 +54,7 @@ def export_secrets():
with open(os.environ[env]) as file: with open(os.environ[env]) as file:
os.environ[env.removesuffix('_FILE')] = file.read() os.environ[env.removesuffix('_FILE')] = file.read()
@cli.command() @cli.command()
def create(): def create():
pre_commands, post_commands, backup_paths, apps = get_backup_cmds() pre_commands, post_commands, backup_paths, apps = get_backup_cmds()
@ -61,9 +64,11 @@ def create():
backup_volumes(backup_paths, apps) backup_volumes(backup_paths, apps)
run_commands(post_commands) run_commands(post_commands)
def get_backup_cmds(): def get_backup_cmds():
client = docker.from_env() client = docker.from_env()
container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} container_by_service = {
c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()}
backup_paths = set() backup_paths = set()
backup_apps = set() backup_apps = set()
pre_commands = {} pre_commands = {}
@ -78,33 +83,38 @@ def get_backup_cmds():
backup_apps.add(stack_name) backup_apps.add(stack_name)
container = container_by_service.get(s.name) container = container_by_service.get(s.name)
if not container: if not container:
logging.error("Container {s.name} is not running, hooks can not be executed") logging.error(
if prehook:= labels.get('backupbot.backup.pre-hook'): "Container {s.name} is not running, hooks can not be executed")
if prehook := labels.get('backupbot.backup.pre-hook'):
pre_commands[container] = prehook pre_commands[container] = prehook
if posthook:= labels.get('backupbot.backup.post-hook'): if posthook := labels.get('backupbot.backup.post-hook'):
post_commands[container] = posthook post_commands[container] = posthook
backup_paths = backup_paths.union( backup_paths = backup_paths.union(
Path(VOLUME_PATH).glob(f"{stack_name}_*")) Path(VOLUME_PATH).glob(f"{stack_name}_*"))
return pre_commands, post_commands, list(backup_paths), list(backup_apps) return pre_commands, post_commands, list(backup_paths), list(backup_apps)
def copy_secrets(apps): def copy_secrets(apps):
rmtree(SECRET_PATH, ignore_errors=True) rmtree(SECRET_PATH, ignore_errors=True)
os.mkdir(SECRET_PATH) os.mkdir(SECRET_PATH)
client = docker.from_env() client = docker.from_env()
container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()} container_by_service = {
c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()}
services = client.services.list() services = client.services.list()
for s in services: for s in services:
app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace'] app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace']
if app_name in apps: if (app_name in apps and
if app_secs:= s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'): app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets')):
if not container_by_service.get(s.name): if not container_by_service.get(s.name):
logging.error("Container {s.name} is not running, secrets can not be copied.") logging.error(
continue "Container {s.name} is not running, secrets can not be copied.")
container_id = container_by_service[s.name].id continue
for sec in app_secs: container_id = container_by_service[s.name].id
src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}' for sec in app_secs:
dst = SECRET_PATH + sec['SecretName'] src = f'/var/lib/docker/containers/{container_id}/mounts/secrets/{sec["SecretID"]}'
copyfile(src, dst) dst = SECRET_PATH + sec['SecretName']
copyfile(src, dst)
def run_commands(commands): def run_commands(commands):
for container, command in commands.items(): for container, command in commands.items():
@ -118,15 +128,18 @@ def run_commands(commands):
logging.info(f"run command in {container.name}") logging.info(f"run command in {container.name}")
logging.info(command) logging.info(command)
if result.exit_code: if result.exit_code:
logging.error(f"Failed to run command {command} in {container.name}: {result.output.decode()}") logging.error(
f"Failed to run command {command} in {container.name}: {result.output.decode()}")
else: else:
logging.info(result.output.decode()) logging.info(result.output.decode())
def backup_volumes(backup_paths, apps, dry_run=False): def backup_volumes(backup_paths, apps, dry_run=False):
result = restic.backup(backup_paths, dry_run=dry_run, tags=apps) result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
print(result) print(result)
logging.info(result) logging.info(result)
@cli.command() @cli.command()
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
@click.option('target', '--target', '-t', envvar='TARGET', default='/') @click.option('target', '--target', '-t', envvar='TARGET', default='/')
@ -136,7 +149,8 @@ def restore(snapshot, target):
if SERVICE: if SERVICE:
service_paths = service_paths + f'{SERVICE}_*' service_paths = service_paths + f'{SERVICE}_*'
print(f"restoring Snapshot {snapshot} of {service_paths} at {target}") print(f"restoring Snapshot {snapshot} of {service_paths} at {target}")
result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir=target) result = restic.restore(snapshot_id=snapshot,
include=service_paths, target_dir=target)
logging.debug(result) logging.debug(result)
@ -144,9 +158,10 @@ def restore(snapshot, target):
def snapshots(): def snapshots():
snapshots = restic.snapshots() snapshots = restic.snapshots()
for snap in snapshots: for snap in snapshots:
if not SERVICE or (tags:= snap.get('tags')) and SERVICE in tags: if not SERVICE or (tags := snap.get('tags')) and SERVICE in tags:
print(snap['time'], snap['id']) print(snap['time'], snap['id'])
@cli.command() @cli.command()
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
@ -156,6 +171,7 @@ def ls(snapshot, path):
if r.get('path'): if r.get('path'):
print(f"{r['ctime']}\t{r['path']}") print(f"{r['ctime']}\t{r['path']}")
def list_files(snapshot, path): def list_files(snapshot, path):
cmd = restic.cat.base_command() + ['ls', snapshot] cmd = restic.cat.base_command() + ['ls', snapshot]
if path: if path:
@ -165,6 +181,7 @@ def list_files(snapshot, path):
results = list(map(json.loads, output.split('|'))) results = list(map(json.loads, output.split('|')))
return results return results
@cli.command() @cli.command()
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest') @click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH') @click.option('path', '--path', '-p', envvar='INCLUDE_PATH')