formatting
This commit is contained in:
parent
5d4def6143
commit
15a552ef8b
43
backupbot.py
43
backupbot.py
|
@ -10,12 +10,13 @@ import restic
|
||||||
from restic.errors import ResticFailedError
|
from restic.errors import ResticFailedError
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from shutil import copyfile, rmtree
|
from shutil import copyfile, rmtree
|
||||||
#logging.basicConfig(level=logging.INFO)
|
# logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
VOLUME_PATH = "/var/lib/docker/volumes/"
|
VOLUME_PATH = "/var/lib/docker/volumes/"
|
||||||
SECRET_PATH = '/secrets/'
|
SECRET_PATH = '/secrets/'
|
||||||
SERVICE = None
|
SERVICE = None
|
||||||
|
|
||||||
|
|
||||||
@click.group()
|
@click.group()
|
||||||
@click.option('-l', '--log', 'loglevel')
|
@click.option('-l', '--log', 'loglevel')
|
||||||
@click.option('service', '--host', '-h', envvar='SERVICE')
|
@click.option('service', '--host', '-h', envvar='SERVICE')
|
||||||
|
@ -23,7 +24,7 @@ SERVICE = None
|
||||||
def cli(loglevel, service, repository):
|
def cli(loglevel, service, repository):
|
||||||
global SERVICE
|
global SERVICE
|
||||||
if service:
|
if service:
|
||||||
SERVICE = service.replace('.','_')
|
SERVICE = service.replace('.', '_')
|
||||||
if loglevel:
|
if loglevel:
|
||||||
numeric_level = getattr(logging, loglevel.upper(), None)
|
numeric_level = getattr(logging, loglevel.upper(), None)
|
||||||
if not isinstance(numeric_level, int):
|
if not isinstance(numeric_level, int):
|
||||||
|
@ -45,6 +46,7 @@ def init_repo(repository):
|
||||||
else:
|
else:
|
||||||
raise error
|
raise error
|
||||||
|
|
||||||
|
|
||||||
def export_secrets():
|
def export_secrets():
|
||||||
for env in os.environ:
|
for env in os.environ:
|
||||||
if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'):
|
if env.endswith('PASSWORD_FILE') or env.endswith('KEY_FILE'):
|
||||||
|
@ -52,6 +54,7 @@ def export_secrets():
|
||||||
with open(os.environ[env]) as file:
|
with open(os.environ[env]) as file:
|
||||||
os.environ[env.removesuffix('_FILE')] = file.read()
|
os.environ[env.removesuffix('_FILE')] = file.read()
|
||||||
|
|
||||||
|
|
||||||
@cli.command()
|
@cli.command()
|
||||||
def create():
|
def create():
|
||||||
pre_commands, post_commands, backup_paths, apps = get_backup_cmds()
|
pre_commands, post_commands, backup_paths, apps = get_backup_cmds()
|
||||||
|
@ -61,9 +64,11 @@ def create():
|
||||||
backup_volumes(backup_paths, apps)
|
backup_volumes(backup_paths, apps)
|
||||||
run_commands(post_commands)
|
run_commands(post_commands)
|
||||||
|
|
||||||
|
|
||||||
def get_backup_cmds():
|
def get_backup_cmds():
|
||||||
client = docker.from_env()
|
client = docker.from_env()
|
||||||
container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()}
|
container_by_service = {
|
||||||
|
c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()}
|
||||||
backup_paths = set()
|
backup_paths = set()
|
||||||
backup_apps = set()
|
backup_apps = set()
|
||||||
pre_commands = {}
|
pre_commands = {}
|
||||||
|
@ -78,27 +83,31 @@ def get_backup_cmds():
|
||||||
backup_apps.add(stack_name)
|
backup_apps.add(stack_name)
|
||||||
container = container_by_service.get(s.name)
|
container = container_by_service.get(s.name)
|
||||||
if not container:
|
if not container:
|
||||||
logging.error("Container {s.name} is not running, hooks can not be executed")
|
logging.error(
|
||||||
if prehook:= labels.get('backupbot.backup.pre-hook'):
|
"Container {s.name} is not running, hooks can not be executed")
|
||||||
|
if prehook := labels.get('backupbot.backup.pre-hook'):
|
||||||
pre_commands[container] = prehook
|
pre_commands[container] = prehook
|
||||||
if posthook:= labels.get('backupbot.backup.post-hook'):
|
if posthook := labels.get('backupbot.backup.post-hook'):
|
||||||
post_commands[container] = posthook
|
post_commands[container] = posthook
|
||||||
backup_paths = backup_paths.union(
|
backup_paths = backup_paths.union(
|
||||||
Path(VOLUME_PATH).glob(f"{stack_name}_*"))
|
Path(VOLUME_PATH).glob(f"{stack_name}_*"))
|
||||||
return pre_commands, post_commands, list(backup_paths), list(backup_apps)
|
return pre_commands, post_commands, list(backup_paths), list(backup_apps)
|
||||||
|
|
||||||
|
|
||||||
def copy_secrets(apps):
|
def copy_secrets(apps):
|
||||||
rmtree(SECRET_PATH, ignore_errors=True)
|
rmtree(SECRET_PATH, ignore_errors=True)
|
||||||
os.mkdir(SECRET_PATH)
|
os.mkdir(SECRET_PATH)
|
||||||
client = docker.from_env()
|
client = docker.from_env()
|
||||||
container_by_service = {c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()}
|
container_by_service = {
|
||||||
|
c.labels['com.docker.swarm.service.name']: c for c in client.containers.list()}
|
||||||
services = client.services.list()
|
services = client.services.list()
|
||||||
for s in services:
|
for s in services:
|
||||||
app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace']
|
app_name = s.attrs['Spec']['Labels']['com.docker.stack.namespace']
|
||||||
if app_name in apps:
|
if (app_name in apps and
|
||||||
if app_secs:= s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets'):
|
app_secs := s.attrs['Spec']['TaskTemplate']['ContainerSpec'].get('Secrets')):
|
||||||
if not container_by_service.get(s.name):
|
if not container_by_service.get(s.name):
|
||||||
logging.error("Container {s.name} is not running, secrets can not be copied.")
|
logging.error(
|
||||||
|
"Container {s.name} is not running, secrets can not be copied.")
|
||||||
continue
|
continue
|
||||||
container_id = container_by_service[s.name].id
|
container_id = container_by_service[s.name].id
|
||||||
for sec in app_secs:
|
for sec in app_secs:
|
||||||
|
@ -106,6 +115,7 @@ def copy_secrets(apps):
|
||||||
dst = SECRET_PATH + sec['SecretName']
|
dst = SECRET_PATH + sec['SecretName']
|
||||||
copyfile(src, dst)
|
copyfile(src, dst)
|
||||||
|
|
||||||
|
|
||||||
def run_commands(commands):
|
def run_commands(commands):
|
||||||
for container, command in commands.items():
|
for container, command in commands.items():
|
||||||
if not command:
|
if not command:
|
||||||
|
@ -118,15 +128,18 @@ def run_commands(commands):
|
||||||
logging.info(f"run command in {container.name}")
|
logging.info(f"run command in {container.name}")
|
||||||
logging.info(command)
|
logging.info(command)
|
||||||
if result.exit_code:
|
if result.exit_code:
|
||||||
logging.error(f"Failed to run command {command} in {container.name}: {result.output.decode()}")
|
logging.error(
|
||||||
|
f"Failed to run command {command} in {container.name}: {result.output.decode()}")
|
||||||
else:
|
else:
|
||||||
logging.info(result.output.decode())
|
logging.info(result.output.decode())
|
||||||
|
|
||||||
|
|
||||||
def backup_volumes(backup_paths, apps, dry_run=False):
|
def backup_volumes(backup_paths, apps, dry_run=False):
|
||||||
result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
|
result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
|
||||||
print(result)
|
print(result)
|
||||||
logging.info(result)
|
logging.info(result)
|
||||||
|
|
||||||
|
|
||||||
@cli.command()
|
@cli.command()
|
||||||
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
|
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
|
||||||
@click.option('target', '--target', '-t', envvar='TARGET', default='/')
|
@click.option('target', '--target', '-t', envvar='TARGET', default='/')
|
||||||
|
@ -136,7 +149,8 @@ def restore(snapshot, target):
|
||||||
if SERVICE:
|
if SERVICE:
|
||||||
service_paths = service_paths + f'{SERVICE}_*'
|
service_paths = service_paths + f'{SERVICE}_*'
|
||||||
print(f"restoring Snapshot {snapshot} of {service_paths} at {target}")
|
print(f"restoring Snapshot {snapshot} of {service_paths} at {target}")
|
||||||
result = restic.restore(snapshot_id=snapshot, include=service_paths, target_dir=target)
|
result = restic.restore(snapshot_id=snapshot,
|
||||||
|
include=service_paths, target_dir=target)
|
||||||
logging.debug(result)
|
logging.debug(result)
|
||||||
|
|
||||||
|
|
||||||
|
@ -144,9 +158,10 @@ def restore(snapshot, target):
|
||||||
def snapshots():
|
def snapshots():
|
||||||
snapshots = restic.snapshots()
|
snapshots = restic.snapshots()
|
||||||
for snap in snapshots:
|
for snap in snapshots:
|
||||||
if not SERVICE or (tags:= snap.get('tags')) and SERVICE in tags:
|
if not SERVICE or (tags := snap.get('tags')) and SERVICE in tags:
|
||||||
print(snap['time'], snap['id'])
|
print(snap['time'], snap['id'])
|
||||||
|
|
||||||
|
|
||||||
@cli.command()
|
@cli.command()
|
||||||
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
|
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
|
||||||
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
|
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
|
||||||
|
@ -156,6 +171,7 @@ def ls(snapshot, path):
|
||||||
if r.get('path'):
|
if r.get('path'):
|
||||||
print(f"{r['ctime']}\t{r['path']}")
|
print(f"{r['ctime']}\t{r['path']}")
|
||||||
|
|
||||||
|
|
||||||
def list_files(snapshot, path):
|
def list_files(snapshot, path):
|
||||||
cmd = restic.cat.base_command() + ['ls', snapshot]
|
cmd = restic.cat.base_command() + ['ls', snapshot]
|
||||||
if path:
|
if path:
|
||||||
|
@ -165,6 +181,7 @@ def list_files(snapshot, path):
|
||||||
results = list(map(json.loads, output.split('|')))
|
results = list(map(json.loads, output.split('|')))
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@cli.command()
|
@cli.command()
|
||||||
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
|
@click.option('snapshot', '--snapshot', '-s', envvar='SNAPSHOT', default='latest')
|
||||||
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
|
@click.option('path', '--path', '-p', envvar='INCLUDE_PATH')
|
||||||
|
|
Loading…
Reference in New Issue