build: add argon2 for recipes that hash secrets locally

Dockerfile

@@ -26,8 +26,10 @@ RUN python -m venv /opt/alakazam \
   && /opt/alakazam/bin/pip install --no-cache-dir -r /tmp/requirements.txt
 
 FROM python:3.11-slim
+# git: alakazam syncs the recipe repos itself. openssh-client: abra reaches the servers over
+# ssh. argon2: recipes hash secrets locally in their abra.sh, vaultwarden's admin token does
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends git openssh-client make \
+  && apt-get install -y --no-install-recommends git openssh-client make argon2 \
   && rm -rf /var/lib/apt/lists/* \
   && git config --global --add safe.directory '*'
 

README.md

@@ -185,6 +185,14 @@ Without `-s` all secrets of the recipes selected by `-r` are purged.
 alakazam example.com.yml ps --wait --timeout 300
 ```
 
+### Excluding Recipes
+
+`-er`/`--exclude-recipe` leaves single recipes out of a run entirely:
+
+```
+alakazam -er traefik example.com.yml clean-deploy -n
+```
+
 ### Run CMDs
 
 Escaping can be akward:

alakazam.py

@@ -57,6 +57,7 @@ ALL_CONFIGS = {}
 SETTINGS = {}
 SETTINGS_PATH = ""  # path to the alakazam settings file
 GROUP_PATH = None  # path this run was invoked with, either an instance file or a group directory
+HIDE_SECRETS = False  # keep secret values out of the output, for runs whose logs are archived
 ABRA_DIR = None  # path to the abra data directory
 FETCH_MAX_AGE = 600 # seconds; recipe repos are refetched at most once per hour
 ABRA_RETRIES = 6 # number of attempts per abra command that failed to reach the server
@@ -75,6 +76,8 @@ CONNECTION_ERRORS = (
 )
 # marker in the abra output that indicates a secret which is not stored on the server
 MISSING_SECRET_ERROR = "doesn't exist on server"
+# abra subcommands that take a secret as an argument or return one
+SECRET_COMMANDS = ("insert", "generate")
 # markers in the abra output for a secret run that had nothing left to do, which abra exits 1 on
 NO_SECRETS_GENERATED = "no secrets generated"
 NO_SECRETS_TO_REMOVE = "no secrets to remove"
@@ -548,6 +551,32 @@ def get_merged_instance_configs(config_path: Path, group_configs: Dict[str, Any]
     return instances
 
 
+def exclude_from_configs(configs: Dict[str, Dict[str, Any]], recipes: Tuple[str]) -> Dict[str, Dict[str, Any]]:
+    """
+    Removes the given recipes from every instance configuration.
+
+    An excluded recipe is not merely left undeployed, it is invisible for the whole run: it is
+    neither listed, nor configured, nor purged, nor waited for. That is what keeps a rebuild from
+    touching an app whose state has to survive it, such as the certificates of a reverse proxy.
+
+    Args:
+        configs (dict): Instances as keys and their app configurations as values
+        recipes (tuple): Recipe names to leave out, all recipes are kept if empty
+
+    Returns:
+        dict: The configurations without those recipes
+    """
+    if not recipes:
+        return configs
+    for recipe in recipes:
+        if not any(recipe in apps for apps in configs.values()):
+            logging.warning(f"'{recipe}' is excluded but not configured for this path")
+    return {
+        instance: {name: config for name, config in apps.items() if name not in recipes}
+        for instance, apps in configs.items()
+    }
+
+
 def merge_connection_configs(configs: Dict[str, Any]) -> Dict[str, Any]:
     """
     Merge connection configurations from the 'combine.yml' to extend instance configurations with inter-app secrets and settings.
@@ -584,6 +613,19 @@ def extend_shared_secrets(connection_config: Dict[str, Any]) -> None:
                 target_conf['shared_secrets'] = {source_app: shared_secrets}
 
 
+def is_secret_command(args: Tuple[str, ...]) -> bool:
+    """
+    Checks whether an abra command carries a secret value in its arguments or in its output.
+
+    Args:
+        args (tuple): The arguments passed to abra()
+
+    Returns:
+        bool: True for the commands that insert or generate a secret
+    """
+    return args[:2] == ("app", "secret") and len(args) > 2 and args[2] in SECRET_COMMANDS
+
+
 def is_connection_error(output: str) -> bool:
     """
     Checks whether the output of an abra command indicates that the server could not be reached.
@@ -643,7 +685,11 @@ def abra(*args: str, machine_output: bool = False, ignore_error: bool = False, s
     command = [arg for arg in command if arg]
     if machine_output:
         command.append("-m")
-    logging.debug(f"run command: {' '.join(command)}")
+    # the command line of 'app secret insert' ends in the secret itself, and the output of
+    # 'app secret generate' contains the generated values
+    quiet = HIDE_SECRETS and is_secret_command(args)
+    if not quiet:
+        logging.debug(f"run command: {' '.join(command)}")
     for attempt in range(1, ABRA_RETRIES + 1):
         process = run_streamed(command) if stream else subprocess.run(command, capture_output=True)
         if not process.returncode or attempt == ABRA_RETRIES:
@@ -657,7 +703,7 @@ def abra(*args: str, machine_output: bool = False, ignore_error: bool = False, s
         sleep(delay)
     if process.stderr and ignore_error:
         logging.warning(process.stderr.decode())
-    if process.stdout and not stream:
+    if process.stdout and not stream and not quiet:
         logging.debug(process.stdout.decode())
     if process.returncode and not ignore_error:
         #breakpoint()
@@ -858,7 +904,8 @@ def generate_all_secrets(domain: str) -> None:
             return
         print(f"secrets for {domain} generated")
         for gen_sec in generated_secrets:
-            print(f"\t {gen_sec['name']}: {gen_sec['value']}")
+            value = "[hidden]" if HIDE_SECRETS else gen_sec['value']
+            print(f"\t {gen_sec['name']}: {value}")
 
 
 def resolve_path(path_str: str, base: Optional[Path] = None) -> Path:
@@ -1267,7 +1314,10 @@ def insert_secret(domain: str, secret_name: str, secret: str) -> None:
     """
     # Fix extra quotes around secrets
     secret = unquote_strings(secret)
-    logging.debug(f"Insert secret {secret_name}: {secret} into {domain}")
+    if HIDE_SECRETS:
+        logging.debug(f"Insert secret {secret_name} into {domain}")
+    else:
+        logging.debug(f"Insert secret {secret_name}: {secret} into {domain}")
     abra("app", "secret", "insert", domain, secret_name, "v1", secret)
 
 
@@ -1502,8 +1552,10 @@ def execute_cmds(app_config: Dict[str, Any], commands: Tuple[str] = tuple(), ini
 @click.group(context_settings={"help_option_names": ['-h', '--help']})
 @click.option('-l', '--log', 'loglevel', help='Desired logging level ("debug", "info", "warning", "error", "critical")')
 @click.option('-e', '--exclude', help='Path to a directory that contains a group of instance configurations to be excluded.', multiple=True, type=click.Path(exists=True))
+@click.option('exclude_recipes', '-er', '--exclude-recipe', multiple=True, metavar='<RecipeName>', help='Leave these recipes untouched, this option can be specified multiple times.')
+@click.option('hide_secrets', '--hide-secrets', is_flag=True, help='Keep secret values out of the output, for runs whose logs are kept.')
 @click.argument('group_path', type=click.Path(exists=True))
-def cli(loglevel: str, group_path: str, exclude:Tuple[str]) -> None:
+def cli(loglevel: str, group_path: str, exclude: Tuple[str], exclude_recipes: Tuple[str], hide_secrets: bool) -> None:
     """
     Alakazam is a meta-configuration app-connector and an abra wrapper, designed as a proof-of-concept to simplify the management of environment configuration files across multiple instances.
 
@@ -1514,8 +1566,10 @@ def cli(loglevel: str, group_path: str, exclude:Tuple[str]) -> None:
     global SETTINGS
     global SETTINGS_PATH
     global GROUP_PATH
+    global HIDE_SECRETS
     global ABRA_DIR
     global ROOT_PATH
+    HIDE_SECRETS = hide_secrets
     if loglevel:
         numeric_level = getattr(logging, loglevel.upper(), None)
         if not isinstance(numeric_level, int):
@@ -1542,6 +1596,10 @@ def cli(loglevel: str, group_path: str, exclude:Tuple[str]) -> None:
     config_sets = read_config(str(ROOT_PATH / "config-sets.yml"))
     instance_configs = get_merged_instance_configs(_group_path, all_group_configs, exclude_paths, config_sets)
     INSTANCE_CONFIGS = merge_connection_configs(instance_configs)
+    # dropping the recipes here rather than at every filter is what makes the exclusion complete:
+    # get_apps(), create_secrets(), configure_apps() and the rest all read INSTANCE_CONFIGS.
+    # ALL_CONFIGS stays whole, it is the cross-instance view that backup looks up its bot in
+    INSTANCE_CONFIGS = exclude_from_configs(INSTANCE_CONFIGS, exclude_recipes)
     all_configs = get_merged_instance_configs(ROOT_PATH, all_group_configs, exclude_paths, config_sets)
     ALL_CONFIGS = merge_connection_configs(all_configs)
     fetch_recipes(INSTANCE_CONFIGS)

tests/test_exclude_recipes.py

@@ -0,0 +1,89 @@
+"""Tests for leaving single recipes out of a run."""
+
+import logging
+import os
+import sys
+
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import exclude_from_configs
+
+CONFIGS = {
+    "a.org": {"authentik": {"app_domain": "login.a.org"}, "traefik": {"app_domain": "a.org"}},
+    "b.org": {"traefik": {"app_domain": "b.org"}, "nextcloud": {"app_domain": "cloud.b.org"}},
+}
+
+
+class TestExcludeFromConfigs:
+    def test_nothing_excluded_returns_the_configs_unchanged(self):
+        assert exclude_from_configs(CONFIGS, ()) is CONFIGS
+
+    def test_the_recipe_is_gone_from_every_instance(self):
+        result = exclude_from_configs(CONFIGS, ("traefik",))
+        assert sorted(result["a.org"]) == ["authentik"]
+        assert sorted(result["b.org"]) == ["nextcloud"]
+
+    def test_several_recipes(self):
+        result = exclude_from_configs(CONFIGS, ("traefik", "nextcloud"))
+        assert result["b.org"] == {}
+
+    def test_the_original_is_not_modified(self):
+        exclude_from_configs(CONFIGS, ("traefik",))
+        assert "traefik" in CONFIGS["a.org"]
+
+    def test_an_unknown_recipe_is_reported(self, caplog):
+        """A typo would otherwise exclude nothing and look like it worked."""
+        with caplog.at_level(logging.WARNING):
+            exclude_from_configs(CONFIGS, ("treafik",))
+        assert "'treafik' is excluded but not configured" in caplog.text
+
+    def test_a_known_recipe_is_not_reported(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            exclude_from_configs(CONFIGS, ("traefik",))
+        assert caplog.text == ""
+
+
+class TestExcludeReachesTheCommands:
+    """One filter in cli() has to cover every consumer of INSTANCE_CONFIGS."""
+
+    @pytest.fixture
+    def env(self, tmp_path, monkeypatch):
+        root = tmp_path / "root"
+        (root / "group").mkdir(parents=True)
+        (root / "alaka.yml").write_text(
+            "authentik:\n    version: 1.0.0\ntraefik:\n    version: 2.0.0\n")
+        (root / "group" / "example.com.yml").write_text("authentik:\ntraefik:\n")
+        (root / "alakazam.yml").write_text(f"root: {root}\n")
+        monkeypatch.setattr(alakazam, "get_settings_path", lambda: str(root / "alakazam.yml"))
+        monkeypatch.setattr(alakazam, "get_abra_dir", lambda: root / ".abra")
+        monkeypatch.setattr(alakazam, "fetch_recipes", lambda *a, **k: None)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        return root
+
+    def invoke(self, root, args):
+        return CliRunner().invoke(alakazam.cli, args + [str(root / "group"), "ls"])
+
+    def test_without_the_flag_both_recipes_are_listed(self, env):
+        result = self.invoke(env, [])
+        assert result.exit_code == 0, result.output
+        assert "authentik" in result.output
+        assert "traefik" in result.output
+
+    def test_the_excluded_recipe_disappears(self, env):
+        result = self.invoke(env, ["-er", "traefik"])
+        assert result.exit_code == 0, result.output
+        assert "authentik" in result.output
+        assert "traefik" not in result.output
+
+    def test_the_long_option_works_too(self, env):
+        result = self.invoke(env, ["--exclude-recipe", "traefik"])
+        assert "traefik" not in result.output
+
+    def test_it_can_be_given_more_than_once(self, env):
+        result = self.invoke(env, ["-er", "traefik", "-er", "authentik"])
+        assert "traefik" not in result.output
+        assert "authentik" not in result.output

tests/test_hide_secrets.py

@@ -0,0 +1,100 @@
+"""Tests for keeping secret values out of the output of a run whose logs are kept."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import generate_all_secrets, insert_secret, is_secret_command
+
+VALUE = "hunter2-do-not-log-me"
+
+
+@pytest.fixture(params=[False, True], ids=["visible", "hidden"])
+def hide(request, monkeypatch):
+    monkeypatch.setattr(alakazam, "HIDE_SECRETS", request.param)
+    return request.param
+
+
+class TestIsSecretCommand:
+    @pytest.mark.parametrize("args", [
+        ("app", "secret", "insert", "login.a.org", "db_password", "v1", VALUE),
+        ("app", "secret", "generate", "-a", "login.a.org"),
+    ])
+    def test_commands_carrying_a_value(self, args):
+        assert is_secret_command(args)
+
+    @pytest.mark.parametrize("args", [
+        ("app", "secret", "ls", "login.a.org"),
+        ("app", "secret", "rm", "login.a.org", "db_password"),
+        ("app", "ls"),
+        ("app", "secret"),
+    ])
+    def test_commands_without_a_value(self, args):
+        assert not is_secret_command(args)
+
+
+class TestGeneratedValues:
+    def install(self, monkeypatch, created=False):
+        def abra(*args, **kwargs):
+            if args[:3] == ("app", "secret", "ls"):
+                return [{"name": "db_password", "created on server": str(created).lower()}]
+            if args[:3] == ("app", "secret", "generate"):
+                return [{"name": "db_password", "value": VALUE}]
+            raise AssertionError(f"unexpected: {args}")
+        monkeypatch.setattr(alakazam, "abra", abra)
+
+    def test_the_name_is_always_reported(self, hide, monkeypatch, capsys):
+        self.install(monkeypatch)
+        generate_all_secrets("login.a.org")
+        assert "db_password" in capsys.readouterr().out
+
+    def test_the_value_follows_the_switch(self, hide, monkeypatch, capsys):
+        self.install(monkeypatch)
+        generate_all_secrets("login.a.org")
+        out = capsys.readouterr().out
+        assert (VALUE in out) is not hide
+        assert ("[hidden]" in out) is hide
+
+
+class TestDebugLog:
+    """-l debug is the level a pipeline reaches for when something breaks."""
+
+    def run_abra(self, monkeypatch, args):
+        class Process:
+            returncode = 0
+            stdout = f'[{{"name":"db_password","value":"{VALUE}"}}]'.encode()
+            stderr = b""
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda cmd, capture_output: Process())
+        return alakazam.abra(*args)
+
+    def test_the_command_line_is_not_logged_when_hidden(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "secret", "insert", "login.a.org", "db_password", "v1", VALUE))
+        assert VALUE not in caplog.text
+
+    def test_the_generated_output_is_not_logged_when_hidden(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "secret", "generate", "-a", "login.a.org"))
+        assert VALUE not in caplog.text
+
+    def test_other_commands_are_still_logged(self, monkeypatch, caplog):
+        """Hiding secrets must not turn the debug log off altogether."""
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "ls"))
+        assert "run command" in caplog.text
+
+    def test_insert_logs_the_name_but_not_the_value(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        with caplog.at_level(logging.DEBUG):
+            insert_secret("login.a.org", "db_password", VALUE)
+        assert "db_password" in caplog.text
+        assert VALUE not in caplog.text