From f1abf6229f5771b86ae17f0b8d715713621fcb50 Mon Sep 17 00:00:00 2001 From: Danny Groenewegen Date: Tue, 7 Apr 2026 14:16:23 +0200 Subject: [PATCH 1/2] - Support alakazam.yml in CWD - Resolve ABRA_DIR using env var or abra.yml instead of hardcoding ~/.abra - Add ABRA_DIR to exclude_paths when inside root --- alakazam.py | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/alakazam.py b/alakazam.py index 0fbd873..2d85428 100755 --- a/alakazam.py +++ b/alakazam.py @@ -54,7 +54,7 @@ ALL_CONFIGS = {} # ... #} SETTINGS = {} -SETTINGS_PATH = "~/.config/alakazam.yml" +SETTINGS_PATH = "alakazam.yml" if Path("alakazam.yml").exists() else "~/.config/alakazam.yml" class MySafeConstructor(SafeConstructor): @@ -467,8 +467,27 @@ def generate_all_secrets(domain: str) -> None: print(f"\t {gen_sec['name']}: {gen_sec['value']}") +def get_abra_dir() -> Path: + if abra_dir := os.environ.get("ABRA_DIR"): + return Path(abra_dir) + home = Path.home() + current = Path.cwd() + while current != current.parent: + for name in ("abra.yaml", "abra.yml"): + config_file = current / name + if config_file.exists(): + abra_config = read_config(str(config_file)) + if abra_dir := abra_config.get("abraDir"): + p = Path(abra_dir) + return (current / p).resolve() if not p.is_absolute() else p + if current == home: + break + current = current.parent + return home / ".abra" + + def get_env_path(server: str, domain: str) -> Path: - return Path(f"~/.abra/servers/{server}/{domain}.env").expanduser() + return Path(ABRA_DIR / "servers" / server / f"{domain}.env") def exchange_secrets(app1: str, instance_config: Dict[str, Any], apps: Tuple[str]) -> None: @@ -806,7 +825,9 @@ def cli(loglevel: str, group_path: str, exclude:Tuple[str]) -> None: global INSTANCE_CONFIGS global ALL_CONFIGS global SETTINGS + global ABRA_DIR SETTINGS = read_config(SETTINGS_PATH) + ABRA_DIR = get_abra_dir() if not (root_path:= SETTINGS.get('root')): root_path = os.getcwd() logging.warning(f"There is no 'root' path defined in '{SETTINGS_PATH}', use current path '{root_path}'instead") @@ -817,9 +838,11 @@ def cli(loglevel: str, group_path: str, exclude:Tuple[str]) -> None: exit(1) all_group_configs = merge_all_group_configs(_root_path) exclude_paths = list(map(lambda p: str(Path(p).absolute()), exclude)) + if ABRA_DIR.is_relative_to(_root_path) and str(ABRA_DIR) not in exclude_paths: + exclude_paths.append(str(ABRA_DIR)) instance_configs = get_merged_instance_configs(_group_path, all_group_configs, exclude_paths) INSTANCE_CONFIGS = merge_connection_configs(instance_configs) - all_configs = get_merged_instance_configs(_root_path, all_group_configs, []) + all_configs = get_merged_instance_configs(_root_path, all_group_configs, exclude_paths) ALL_CONFIGS = merge_connection_configs(all_configs) if loglevel: numeric_level = getattr(logging, loglevel.upper(), None) @@ -1357,11 +1380,11 @@ def install() -> None: @click.option('-p', '--push', is_flag=True, help='Push all changes to the remote') def git(recipes: Tuple[str], message: str, push: bool = False) -> None: """ - Run multiple git commands on each ~/.abra/servers. Without any specified options each repository will only be pulled. + Run multiple git commands on each {ABRA_DIR}/servers. Without any specified options each repository will only be pulled. """ servers = get_server(recipes) for server in servers: - server_path = Path(f"~/.abra/servers/{server}").expanduser() + server_path = Path(ABRA_DIR / "servers" / server) try: repo = Repo(server_path) except InvalidGitRepositoryError: @@ -1391,12 +1414,12 @@ def git(recipes: Tuple[str], message: str, push: bool = False) -> None: @click.option('recipes', '-r', '--recipe', multiple=True, metavar='', help='Filter for selcted recipes, this option can be specified multiple times.') def diff(recipes: Tuple[str]) -> None: """ - Show the changes in the .env repositories inside ~/.abra/servers. + Show the changes in the .env repositories inside {ABRA_DIR}/servers. """ init(autoreset=True) servers = get_server(recipes) for server in servers: - server_path = Path(f"~/.abra/servers/{server}").expanduser() + server_path = Path(ABRA_DIR / "servers" / server) try: repo = Repo(server_path) except InvalidGitRepositoryError: -- 2.49.0 From 38440692ee33f31b70d7cd7f0be741947809de06 Mon Sep 17 00:00:00 2001 From: Danny Groenewegen Date: Tue, 14 Apr 2026 13:11:08 +0200 Subject: [PATCH 2/2] feat: add --syncvalues flag and improve secret sharing - Add --syncvalues flag to secrets command to update secret values from source to target apps (with redeploy of target app). Without --syncvalues, log a warning when values differ. - Fix bidirectionality: when source is missing its secret but target has one, warn instead of silently copying back and potentially overwriting the source value - Add get_secret_from_host fallback for distroless containers that reads the secret from the Docker host filesystem over SSH - Run secret hooks for all apps before any secret exchange to ensure source secrets are available before syncing to target app --- alakazam.py | 151 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 18 deletions(-) diff --git a/alakazam.py b/alakazam.py index 2d85428..78f38a1 100755 --- a/alakazam.py +++ b/alakazam.py @@ -490,7 +490,7 @@ def get_env_path(server: str, domain: str) -> Path: return Path(ABRA_DIR / "servers" / server / f"{domain}.env") -def exchange_secrets(app1: str, instance_config: Dict[str, Any], apps: Tuple[str]) -> None: +def exchange_secrets(app1: str, instance_config: Dict[str, Any], apps: Tuple[str], syncvalues: bool = False) -> None: """ Facilitates the exchange of shared secrets between apps within the same instance based on the configuration. This function checks for shared secrets configurations and applies them to ensure that secrets are synchronized between apps. @@ -499,6 +499,7 @@ def exchange_secrets(app1: str, instance_config: Dict[str, Any], apps: Tuple[str app1 (str): The first app to participate in the secret exchange. instance_config (dict): Configuration of the instance including all apps. apps (list): List of other apps in the instance to potentially share secrets with. + syncvalues (bool): When True, update target secrets whose values differ from the source. Returns: None @@ -510,24 +511,28 @@ def exchange_secrets(app1: str, instance_config: Dict[str, Any], apps: Tuple[str app2_domain = app2_config['app_domain'] if app1_shared_secrets := get_value(app1_config, "shared_secrets", app2): logging.info(f'share secrets between {app1_domain} and {app2_domain}') - share_secrets(app1_domain, app2_domain, app1_shared_secrets) + share_secrets(app1_domain, app2_domain, app1_shared_secrets, syncvalues=syncvalues) if app2_shared_secrets := get_value(app2_config, "shared_secrets", app1): logging.info(f'share secrets between {app1_domain} and {app2_domain}') - share_secrets(app2_domain, app1_domain, app2_shared_secrets) + share_secrets(app2_domain, app1_domain, app2_shared_secrets, syncvalues=syncvalues) def str2bool(value: str) -> bool: return value.lower() in ("yes", "true", "t", "1") -def share_secrets(app1_domain: str, app2_domain: str, secrets: Dict[str, str]) -> None: +def share_secrets(app1_domain: str, app2_domain: str, secrets: Dict[str, str], syncvalues: bool = False) -> None: """ Facilitates the sharing of secrets between two applications. - This function checks and transfers secrets from one applications to another if one applications possesses a secret that the other lacks, ensuring both applications maintain synchronized secret configurations. + app1 is treated as the source; app2 is the target. + This function checks and transfers secrets from app1 to app2 if app2 lacks the secret, + or compares values when both have the secret (with --syncvalues to update the target). Args: - app1_domain (str): The domain of the first application. - app2_domain (str): The domain of the second application. + app1_domain (str): The domain of the source application. + app2_domain (str): The domain of the target application. secrets (dict): A dictionary mapping secret names in app2 to their corresponding names in app1. + syncvalues (bool): When True, update app2's secret if its value differs from app1's. + When False, only log a warning on mismatch. """ app1_stored_secrets = abra("app", "secret", "ls", app1_domain, machine_output=True) app1_stored_secrets = {x['name']: str2bool(x['created on server']) for x in app1_stored_secrets} @@ -547,7 +552,6 @@ def share_secrets(app1_domain: str, app2_domain: str, secrets: Dict[str, str]) - app2_container = [] for app2_secret in secrets: app1_secret = secrets[app2_secret] - # TODO: test if both apps have the secret available try: app1_secret_is_stored = app1_stored_secrets[app1_secret] except KeyError: @@ -559,21 +563,55 @@ def share_secrets(app1_domain: str, app2_domain: str, secrets: Dict[str, str]) - logging.error(f"{app2_domain} does not contain secret {app2_secret}") continue if app1_secret_is_stored and not app2_secret_is_stored: + # Source has the secret, target doesn't if not app1_container: - logging.error(f"{app1_domain} already contains the secrets {app1_secret} but it's not running, so it's not possible to export the secret.") + logging.error(f"{app1_domain} has {app1_secret} but is not running; cannot export the secret.") + continue secret = get_secret(app1_domain, app1_secret, app1_container) insert_secret(app2_domain, app2_secret, secret) logging.info(f"Shared secret {app1_secret} from {app1_domain} to {app2_domain} as {app2_secret}") elif app2_secret_is_stored and not app1_secret_is_stored: - if not app2_container: - logging.error(f"{app2_domain} already contains the secrets {app2_secret} but it's not running, so it's not possible to export the secret.") - secret = get_secret(app2_domain, app2_secret, app2_container) - insert_secret(app1_domain, app1_secret, secret) - logging.info(f"Shared secret {app2_secret} from {app2_domain} to {app1_domain} as {app1_secret}") + # Target has secret but source doesn't, warn instead of copying back. + # Copying target to source risks overwriting the correct source value. + logging.warning( + f"{app2_domain} has {app2_secret} but {app1_domain} (source) is missing {app1_secret}. " + f"The source secret should be restored manually." + ) elif not any([app1_secret_is_stored, app2_secret_is_stored]): + # Neither has it — generate new and insert to both secret = generate_secret(app1_domain, app1_secret) insert_secret(app2_domain, app2_secret, secret) - logging.info(f"Generated secrets exachanged secret {app1_secret} and {app2_secret} in {app1_domain} and {app2_domain}") + logging.info(f"Generated secrets exchanged {app1_secret} and {app2_secret} in {app1_domain} and {app2_domain}") + else: + # Both apps have the secret — compare values (app1 is source, app2 is target) + if not app1_container: + logging.warning( + f"{app1_domain} has {app1_secret} but is not running; cannot compare values" + ) + continue + try: + app1_value = get_secret(app1_domain, app1_secret, app1_container) + app2_value = get_secret(app2_domain, app2_secret, app2_container) if app2_container else None + except RuntimeError as e: + logging.warning(f"Could not read secret for comparison ({app1_domain}/{app2_domain}): {e}") + continue + if app2_value is None or app1_value != app2_value: + if syncvalues: + logging.info( + f"Syncing {app1_secret} from {app1_domain} to {app2_secret} in {app2_domain}" + ) + update_secret(app2_domain, app2_secret, app1_value, app2_container) + else: + logging.warning( + f"Secret mismatch: {app1_domain}/{app1_secret} differs from " + f"{app2_domain}/{app2_secret}. " + f"Run 'alakazam GROUP_PATH secrets --syncvalues' to synchronize." + ) + else: + logging.info( + f"Secrets {app1_secret}/{app2_secret} already in sync " + f"between {app1_domain} and {app2_domain}" + ) def get_secret(domain: str, secret_name: str, containers: List) -> str: @@ -594,7 +632,60 @@ def get_secret(domain: str, secret_name: str, containers: List) -> str: return str(secret) except RuntimeError: continue - raise RuntimeError(f"{secret_name} not found for {domain}") + + logging.info(f"{secret_name} not found for {domain} in a running container." + f"Trying to read from host filesystem via SSH for distroless containers") + return get_secret_from_host(domain, secret_name) + + +def get_secret_from_host(domain: str, secret_name: str) -> str: + """ + Read a Docker Swarm secret value by SSHing to the host and reading from the + container's mount path. Works for distroless containers that have no shell. + """ + server = None + for instance_apps in INSTANCE_CONFIGS.values(): + for app_config in instance_apps.values(): + if app_config.get('app_domain') == domain: + server = app_config['server'] + break + if server: + break + if not server: + raise RuntimeError(f"Could not determine server for {domain}") + + # Stack name convention: dots replaced with underscores (e.g. foo.example.com → foo_example_com) + stack_name = domain.replace(".", "_") + + # Step 1: find SECRET_ID and container ID for this secret + find_cmd = ( + f"docker stack services {stack_name} --format '{{{{.Name}}}}' | while read svc; do " + f" CID=$(docker ps --no-trunc -q --filter \"name=$svc\" | head -1); " + f" [ -z \"$CID\" ] && continue; " + f" docker service inspect \"$svc\" --format " + f" '{{{{json .Spec.TaskTemplate.ContainerSpec.Secrets}}}}' " + f" | jq -r --arg cid \"$CID\" " + f" '.[]? | .SecretID + \" \" + $cid + \" \" + .SecretName'; " + f"done | grep ' {stack_name}_{secret_name}_' | head -1" + ) + result = subprocess.run(["ssh", server, find_cmd], capture_output=True, text=True) + match_line = result.stdout.strip() + if not match_line: + raise RuntimeError(f"Secret '{secret_name}' not found in stack '{stack_name}' on {server}") + + parts = match_line.split() + secret_id, container_id = parts[0], parts[1] + + # Step 2: read the secret file from the host filesystem + read_cmd = ( + f"cat /var/lib/docker/containers/{container_id}/mounts/secrets/{secret_id} 2>/dev/null " + f"|| sudo cat /var/lib/docker/containers/{container_id}/mounts/secrets/{secret_id}" + ) + read_result = subprocess.run(["ssh", server, read_cmd], capture_output=True, text=True) + value = read_result.stdout # no strip — trailing newline could be part of the secret + if not value: + raise RuntimeError(f"Could not read value for secret '{secret_name}' on {server}") + return value def generate_secret(domain: str, secret_name: str) -> str: """ @@ -684,6 +775,23 @@ def insert_secret(domain: str, secret_name: str, secret: str) -> None: abra("app", "secret", "insert", domain, secret_name, "v1", secret) +def update_secret(domain: str, secret_name: str, secret: str, containers: List) -> None: + """ + Updates an existing secret by removing and reinserting it. + If the app was deployed (containers non-empty), undeploys first and redeploys after. + If the app was not deployed, only replaces the secret in Docker Swarm. + """ + secret = unquote_strings(secret) + was_deployed = bool(containers) + logging.info(f"Updating secret {secret_name} in {domain} (deployed: {was_deployed})") + if was_deployed: + abra("--no-input", "app", "undeploy", domain, ignore_error=True) + abra("app", "secret", "remove", domain, secret_name, ignore_error=True) + abra("app", "secret", "insert", domain, secret_name, "v1", secret) + if was_deployed: + abra("--no-input", "app", "deploy", domain, ignore_error=True) + + def uncomment(keys: List[str], path: str, match_all: bool = False) -> None: """ Uncomments lines in a configuration file that contain specified keys. @@ -897,7 +1005,8 @@ def config(recipes: Tuple[str]) -> None: @cli.command() @click.option('recipes', '-r', '--recipe', multiple=True, metavar='', help='Filter for selcted recipes, this option can be specified multiple times.') -def secrets(recipes: Tuple[str]) -> None: +@click.option('--syncvalues', is_flag=True, default=False, help='Synchronize secret values from source to target when they differ. Without this flag, value mismatches are only reported as warnings.') +def secrets(recipes: Tuple[str], syncvalues: bool) -> None: """ Generates and inserts secrets for specified apps. This function handles the generation of new secrets, the syncronisation of shared secrets and the insertion of existing secrets from the configuration into the appropriate locations for each application. @@ -908,13 +1017,19 @@ def secrets(recipes: Tuple[str]) -> None: selected_apps = [app for app in recipes if app in instance_config.keys()] else: selected_apps = instance_config.keys() + # Pass 1: insert config secrets and run secret hooks for all apps first, + # so that source secrets are stored before any exchange happens. for app in selected_apps: app_config = instance_config[app] domain = app_config['app_domain'] print(f"Create secrets for {domain}") insert_secrets_from_conf(domain, app_config) run_secret_hooks(domain, app_config) - exchange_secrets(app, instance_config, instance_apps) + # Pass 2: exchange secrets between apps, then generate any remaining missing secrets. + for app in selected_apps: + app_config = instance_config[app] + domain = app_config['app_domain'] + exchange_secrets(app, instance_config, instance_apps, syncvalues=syncvalues) generate_all_secrets(domain) -- 2.49.0