implement volume excludes and path includes

backupbot.py

@@ -97,7 +97,8 @@ def export_secrets():
 @cli.command()
 @click.option('retries', '--retries', '-r', envvar='RETRIES', default=1)
 def create(retries):
-    pre_commands, post_commands, backup_paths, apps = get_backup_cmds()
+    app_settings = parse_backup_labels()
+    pre_commands, post_commands, backup_paths, apps = get_backup_details(app_settings)
     copy_secrets(apps)
     backup_paths.append(SECRET_PATH)
     run_commands(pre_commands)
@@ -105,6 +106,94 @@ def create(retries):
     run_commands(post_commands)
 
 
+def parse_backup_labels():
+    client = docker.from_env()
+    container_by_service = {
+        c.labels.get('com.docker.swarm.service.name'): c for c in client.containers.list()}
+    services = client.services.list()
+    app_settings = {}
+    for s in services:
+        specs = s.attrs['Spec']
+        labels = specs['Labels']
+        stack_name = labels['com.docker.stack.namespace']
+        app_settings[stack_name] = settings = app_settings.get(stack_name) or {}
+        if mounts:= specs['TaskTemplate']['ContainerSpec'].get('Mounts'):
+            volumes = parse_volumes(stack_name, mounts)
+            excluded_volumes, included_paths = parse_excludes_includes(volumes, labels)
+            settings['excluded_volumes'] = excluded_volumes.union(settings.get('excluded_volumes') or set())
+            settings['included_paths'] = included_paths.union(settings.get('included_paths') or set())
+            settings['volume_paths'] = set(volumes.values()).union(settings.get('volume_paths') or set())
+        if (backup := labels.get('backupbot.backup')) and bool(backup):
+            settings['enabled'] = True
+        if container := container_by_service.get(s.name):
+            if command := labels.get('backupbot.backup.pre-hook'):
+                if not (pre_hooks:= settings.get('pre_hooks')):
+                    pre_hooks = settings['pre_hooks'] = {}
+                pre_hooks[container] = command
+            if command := labels.get('backupbot.backup.post-hook'):
+                if not (post_hooks:= settings.get('post_hooks')):
+                    post_hooks = settings['post_hooks'] = {}
+                post_hooks[container] = command
+        else:
+            logger.error(
+                f"Container {s.name} is not running, hooks can not be executed")
+    return app_settings
+
+
+def get_backup_details(app_settings):
+    backup_paths = set()
+    backup_apps = []
+    pre_hooks= {}
+    post_hooks = {}
+    for app, settings in app_settings.items():
+        if settings.get('enabled'):
+            # Uncomment these lines to backup only a specific service
+            # This will unfortenately decrease restice performance
+            # if SERVICE and SERVICE != app:
+            #     continue
+            backup_apps.append(app)
+            paths = (settings['volume_paths'] - settings['excluded_volumes']).union(settings['included_paths'])
+            backup_paths = backup_paths.union(paths)
+            if hooks:= settings.get('pre_hooks'):
+                pre_hooks.update(hooks)
+            if hooks:= settings.get('post_hooks'):
+                post_hooks.update(hooks)
+    return pre_hooks, post_hooks, list(backup_paths), backup_apps
+
+
+def parse_volumes(stack_name, mounts):
+    volumes = {}
+    for m in mounts:
+        if m['Type'] != 'volume':
+            continue
+        relative_path = m['Source']
+        name = relative_path.removeprefix(stack_name + '_')
+        absolute_path = Path(f"{VOLUME_PATH}{relative_path}/_data/")
+        #TODO: if absolute_path.exists():
+        volumes[name] = absolute_path
+    return volumes
+
+
+def parse_excludes_includes(volumes, labels):
+    excluded_volumes_paths = set()
+    included_paths = set()
+    for label, value in labels.items():
+        if label.startswith('backupbot.backup.volumes.'):
+            volume_name = label.removeprefix('backupbot.backup.volumes.')
+            if not (volume_path:= volumes.get(volume_name)):
+                logger.error(f'Can not find volume with the name {volume_name}')
+            elif label.endswith('path'): 
+                relative_paths = value.split(',')
+                for p in relative_paths:
+                    absolute_path = Path(f"{volume_path}/{p}")
+                    #TODO: if absolute_path.exists():
+                    included_paths.add(absolute_path)
+                excluded_volumes_paths.add(volume_path)
+            elif bool(value):
+                excluded_volumes_paths.add(volume_path)
+    return excluded_volumes_paths, included_paths
+
+
 def get_backup_cmds():
     client = docker.from_env()
     container_by_service = {