add host while preserving filechange detection #53

backupbot.py

@@ -18,7 +18,7 @@ from shutil import copyfile, rmtree
 
 VOLUME_PATH = "/var/lib/docker/volumes/"
 SECRET_PATH = '/secrets/'
-SERVICE = None
+SERVICE = 'ALL'
 
 logger = logging.getLogger("backupbot")
 logging.addLevelName(55, 'SUMMARY')
@@ -100,7 +100,7 @@ def create(retries):
     app_settings = parse_backup_labels()
     pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings)
     copy_secrets(apps)
-    backup_paths.append(SECRET_PATH)
+    backup_paths.append(Path(SECRET_PATH))
     run_commands(pre_commands)
     backup_volumes(backup_paths, apps, int(retries))
     run_commands(post_commands)
@@ -114,10 +114,10 @@ def create(retries):
 @click.option('container', '--container', '-c', envvar='CONTAINER', multiple=True)
 def restore(snapshot, target, noninteractive, volumes, container):
     app_settings = parse_backup_labels('restore', container)
-    if SERVICE:
+    if SERVICE != 'ALL':
         app_settings = {SERVICE: app_settings[SERVICE]}
     pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings, volumes)
-    snapshots = get_snapshots(snapshot_id=snapshot, app=SERVICE)
+    snapshots = get_snapshots(snapshot_id=snapshot)
     if not snapshot:
         logger.error("No Snapshots with ID {snapshots} for {apps} found.")
         exit(1)
@@ -136,7 +136,6 @@ def restore(snapshot, target, noninteractive, volumes, container):
             logger.error("Restore aborted")
             exit(1)
     print(f"Restoring Snapshot {snapshot} at {target}")
-    # TODO: use tags if no snapshot is selected, to use a snapshot including SERVICE
     run_commands(pre_commands)
     result = restic_restore(snapshot_id=snapshot, include=backup_paths, target_dir=target)
     run_commands(post_commands)
@@ -152,16 +151,15 @@ def restic_restore(snapshot_id='latest', include=[], target_dir=None):
     return restic.internal.command_executor.execute(cmd)
 
 
-def get_snapshots(snapshot_id=None, app=None):
+def get_snapshots(snapshot_id=None):
     if snapshot_id and snapshot_id != 'latest':
         snapshots = restic.snapshots(snapshot_id=snapshot_id)
-        if app and app not in snapshots[0]['tags']:
+        if SERVICE not in snapshots[0]['tags']:
-            logger.error(f'Snapshot with ID {snapshot_id} does not contain {app}')
+            logger.error(f'Snapshot with ID {snapshot_id} does not contain {SERVICE}')
             exit(1)
     else:
         snapshots = restic.snapshots()
-        if app:
+        snapshots = list(filter(lambda x: x.get('tags') and SERVICE in x.get('tags'), snapshots))
-            snapshots = list(filter(lambda x: app in x.get('tags'), snapshots))
     if snapshot_id == 'latest':
         return snapshots[-1:]
     else:
@@ -214,10 +212,8 @@ def get_backup_details(app_settings, volumes=[]):
     post_hooks = {}
     for app, settings in app_settings.items():
         if settings.get('enabled'):
-            # Uncomment these lines to backup only a specific service
+            if SERVICE != 'ALL' and SERVICE != app:
-            # This will unfortenately decrease restice performance
+                continue
-            # if SERVICE and SERVICE != app:
-            #     continue
             backup_apps.append(app)
             add_backup_paths(backup_paths, settings, app, volumes)
             if hooks:= settings.get('pre_hooks'):
@@ -309,7 +305,7 @@ def copy_secrets(apps):
                         f"For the secret {sec['SecretName']} the file {src} does not exist for {s.name}")
                     continue
                 dst = SECRET_PATH + sec['SecretName']
-                logger.debug("Copy Secret {sec['SecretName']}")
+                logger.debug(f"Copy Secret {sec['SecretName']}")
                 copyfile(src, dst)
 
 
@@ -337,10 +333,17 @@ def run_commands(commands):
 def backup_volumes(backup_paths, apps, retries, dry_run=False):
     while True:
         try:
-            logger.info("Start volume backup")
+            logger.info("Backup these paths:")
-            logger.debug(backup_paths)
+            logger.debug("\n".join(map(str, backup_paths)))
             backup_paths = list(filter(path_exists, backup_paths))
-            result = restic.backup(backup_paths, dry_run=dry_run, tags=apps)
+            cmd = restic.cat.base_command()
+            parent = get_snapshots('latest') 
+            if parent:
+                # https://restic.readthedocs.io/en/stable/040_backup.html#file-change-detection
+                cmd.extend(['--parent', parent[0]['short_id']])
+            tags = set(apps + [SERVICE])
+            logger.info("Start volume backup")
+            result = restic.internal.backup.run(cmd, backup_paths, dry_run=dry_run, tags=tags)
             logger.summary("backup finished", extra=result)
             return
         except ResticFailedError as error:
@@ -361,12 +364,12 @@ def path_exists(path):
 
 @cli.command()
 def snapshots():
-    snapshots = get_snapshots(app=SERVICE)
+    snapshots = get_snapshots()
     for snap in snapshots:
         print(snap['time'], snap['id'])
     if not snapshots:
         err_msg = "No Snapshots found"
-        if SERVICE:
+        if SERVICE != 'ALL':
             service_name = SERVICE.replace('_', '.')
             err_msg += f' for app {service_name}'
         logger.warning(err_msg)
@@ -384,8 +387,7 @@ def ls(snapshot, path):
 
 def list_files(snapshot, path):
     cmd = restic.cat.base_command() + ['ls']
-    if SERVICE:
+    cmd = cmd + ['--tag', SERVICE]
-        cmd = cmd + ['--tag', SERVICE]
     cmd.append(snapshot)
     if path:
         cmd.append(path)
@@ -394,7 +396,7 @@ def list_files(snapshot, path):
     except ResticFailedError as error:
         if 'no snapshot found' in str(error):
             err_msg = f'There is no snapshot "{snapshot}"'
-            if SERVICE:
+            if SERVICE != 'ALL':
                 err_msg += f' for the app "{SERVICE}"'
             logger.error(err_msg)
             exit(1)
@@ -426,7 +428,7 @@ def download(snapshot, path, volumes, secrets):
         tarinfo.size = len(binary_output)
         file_dumps.append((binary_output, tarinfo))
     if volumes:
-        if not SERVICE:
+        if SERVICE == 'ALL':
             logger.error("Please specify '--host' when using '--volumes'")
             exit(1)
         files = list_files(snapshot, VOLUME_PATH)
@@ -439,7 +441,7 @@ def download(snapshot, path, volumes, secrets):
                 tarinfo.size = len(binary_output)
                 file_dumps.append((binary_output, tarinfo))
     if secrets:
-        if not SERVICE:
+        if SERVICE == 'ALL':
             logger.error("Please specify '--host' when using '--secrets'")
             exit(1)
         filename = f"{SERVICE}.json"
@@ -476,8 +478,7 @@ def get_formatted_size(file_path):
 
 def dump(snapshot, path):
     cmd = restic.cat.base_command() + ['dump']
-    if SERVICE:
+    cmd = cmd + ['--tag', SERVICE]
-        cmd = cmd + ['--tag', SERVICE]
     cmd = cmd + [snapshot, path]
     print(f"Dumping {path} from snapshot '{snapshot}'")
     output = subprocess.run(cmd, capture_output=True)