diff --git a/backupbot.py b/backupbot.py index 1c3e49b..afa0c17 100755 --- a/backupbot.py +++ b/backupbot.py @@ -18,7 +18,7 @@ from shutil import copyfile, rmtree VOLUME_PATH = "/var/lib/docker/volumes/" SECRET_PATH = '/secrets/' -SERVICE = None +SERVICE = 'ALL' logger = logging.getLogger("backupbot") logging.addLevelName(55, 'SUMMARY') @@ -100,7 +100,7 @@ def create(retries): app_settings = parse_backup_labels() pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings) copy_secrets(apps) - backup_paths.append(SECRET_PATH) + backup_paths.append(Path(SECRET_PATH)) run_commands(pre_commands) backup_volumes(backup_paths, apps, int(retries)) run_commands(post_commands) @@ -114,10 +114,10 @@ def create(retries): @click.option('container', '--container', '-c', envvar='CONTAINER', multiple=True) def restore(snapshot, target, noninteractive, volumes, container): app_settings = parse_backup_labels('restore', container) - if SERVICE: + if SERVICE != 'ALL': app_settings = {SERVICE: app_settings[SERVICE]} pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings, volumes) - snapshots = get_snapshots(snapshot_id=snapshot, app=SERVICE) + snapshots = get_snapshots(snapshot_id=snapshot) if not snapshot: logger.error("No Snapshots with ID {snapshots} for {apps} found.") exit(1) @@ -136,7 +136,6 @@ def restore(snapshot, target, noninteractive, volumes, container): logger.error("Restore aborted") exit(1) print(f"Restoring Snapshot {snapshot} at {target}") - # TODO: use tags if no snapshot is selected, to use a snapshot including SERVICE run_commands(pre_commands) result = restic_restore(snapshot_id=snapshot, include=backup_paths, target_dir=target) run_commands(post_commands) @@ -152,16 +151,15 @@ def restic_restore(snapshot_id='latest', include=[], target_dir=None): return restic.internal.command_executor.execute(cmd) -def get_snapshots(snapshot_id=None, app=None): +def get_snapshots(snapshot_id=None): if snapshot_id and snapshot_id != 'latest': snapshots = restic.snapshots(snapshot_id=snapshot_id) - if app and app not in snapshots[0]['tags']: - logger.error(f'Snapshot with ID {snapshot_id} does not contain {app}') + if SERVICE not in snapshots[0]['tags']: + logger.error(f'Snapshot with ID {snapshot_id} does not contain {SERVICE}') exit(1) else: snapshots = restic.snapshots() - if app: - snapshots = list(filter(lambda x: app in x.get('tags'), snapshots)) + snapshots = list(filter(lambda x: x.get('tags') and SERVICE in x.get('tags'), snapshots)) if snapshot_id == 'latest': return snapshots[-1:] else: @@ -214,10 +212,8 @@ def get_backup_details(app_settings, volumes=[]): post_hooks = {} for app, settings in app_settings.items(): if settings.get('enabled'): - # Uncomment these lines to backup only a specific service - # This will unfortenately decrease restice performance - # if SERVICE and SERVICE != app: - # continue + if SERVICE != 'ALL' and SERVICE != app: + continue backup_apps.append(app) add_backup_paths(backup_paths, settings, app, volumes) if hooks:= settings.get('pre_hooks'): @@ -309,7 +305,7 @@ def copy_secrets(apps): f"For the secret {sec['SecretName']} the file {src} does not exist for {s.name}") continue dst = SECRET_PATH + sec['SecretName'] - logger.debug("Copy Secret {sec['SecretName']}") + logger.debug(f"Copy Secret {sec['SecretName']}") copyfile(src, dst) @@ -337,10 +333,17 @@ def run_commands(commands): def backup_volumes(backup_paths, apps, retries, dry_run=False): while True: try: - logger.info("Start volume backup") - logger.debug(backup_paths) + logger.info("Backup these paths:") + logger.debug("\n".join(map(str, backup_paths))) backup_paths = list(filter(path_exists, backup_paths)) - result = restic.backup(backup_paths, dry_run=dry_run, tags=apps) + cmd = restic.cat.base_command() + parent = get_snapshots('latest') + if parent: + # https://restic.readthedocs.io/en/stable/040_backup.html#file-change-detection + cmd.extend(['--parent', parent[0]['short_id']]) + tags = set(apps + [SERVICE]) + logger.info("Start volume backup") + result = restic.internal.backup.run(cmd, backup_paths, dry_run=dry_run, tags=tags) logger.summary("backup finished", extra=result) return except ResticFailedError as error: @@ -361,12 +364,12 @@ def path_exists(path): @cli.command() def snapshots(): - snapshots = get_snapshots(app=SERVICE) + snapshots = get_snapshots() for snap in snapshots: print(snap['time'], snap['id']) if not snapshots: err_msg = "No Snapshots found" - if SERVICE: + if SERVICE != 'ALL': service_name = SERVICE.replace('_', '.') err_msg += f' for app {service_name}' logger.warning(err_msg) @@ -384,8 +387,7 @@ def ls(snapshot, path): def list_files(snapshot, path): cmd = restic.cat.base_command() + ['ls'] - if SERVICE: - cmd = cmd + ['--tag', SERVICE] + cmd = cmd + ['--tag', SERVICE] cmd.append(snapshot) if path: cmd.append(path) @@ -394,7 +396,7 @@ def list_files(snapshot, path): except ResticFailedError as error: if 'no snapshot found' in str(error): err_msg = f'There is no snapshot "{snapshot}"' - if SERVICE: + if SERVICE != 'ALL': err_msg += f' for the app "{SERVICE}"' logger.error(err_msg) exit(1) @@ -426,7 +428,7 @@ def download(snapshot, path, volumes, secrets): tarinfo.size = len(binary_output) file_dumps.append((binary_output, tarinfo)) if volumes: - if not SERVICE: + if SERVICE == 'ALL': logger.error("Please specify '--host' when using '--volumes'") exit(1) files = list_files(snapshot, VOLUME_PATH) @@ -439,7 +441,7 @@ def download(snapshot, path, volumes, secrets): tarinfo.size = len(binary_output) file_dumps.append((binary_output, tarinfo)) if secrets: - if not SERVICE: + if SERVICE == 'ALL': logger.error("Please specify '--host' when using '--secrets'") exit(1) filename = f"{SERVICE}.json" @@ -476,8 +478,7 @@ def get_formatted_size(file_path): def dump(snapshot, path): cmd = restic.cat.base_command() + ['dump'] - if SERVICE: - cmd = cmd + ['--tag', SERVICE] + cmd = cmd + ['--tag', SERVICE] cmd = cmd + [snapshot, path] print(f"Dumping {path} from snapshot '{snapshot}'") output = subprocess.run(cmd, capture_output=True)