feat(secrets): add --hide-secrets to keep values out of the output

Reviewed-on: #13

.dockerignore

@@ -0,0 +1,10 @@
+.git
+# the submodule's git dir is the only source for the abra version, the builder stage reads it
+!.git/modules/abra
+abra/.git
+.venv
+.pipeline
+__pycache__
+tests
+*.png
+*.drawio

.drone.yml

@@ -0,0 +1,45 @@
+---
+kind: pipeline
+type: docker
+name: tests
+
+steps:
+  - name: pytest
+    image: python:3.11
+    commands:
+      # not the slim image: MarkupSafe 1.1.1 has no wheel here and compiles a C extension
+      - pip install --quiet -r requirements-dev.txt
+      - pytest -q
+
+---
+kind: pipeline
+type: docker
+name: publish image
+
+depends_on:
+  - tests
+
+# the image builds abra from the submodule, and the default clone step leaves it empty
+clone:
+  disable: true
+
+steps:
+  - name: clone
+    image: plugins/git
+    settings:
+      recursive: true
+
+  - name: publish image
+    image: plugins/docker
+    settings:
+      registry: git.coopcloud.tech
+      repo: git.coopcloud.tech/toolshed/alakazam
+      username:
+        from_secret: git_coopcloud_tech_user
+      password:
+        from_secret: git_coopcloud_tech_token
+      auto_tag: true
+
+trigger:
+  event:
+    - tag

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+.venv/
+.pipeline/

Dockerfile

@@ -0,0 +1,49 @@
+# Builds an image that runs alakazam together with the abra version pinned by the submodule.
+
+# keep in step with GOVERSION in abra/Makefile, a FROM cannot read it from there
+ARG GO_VERSION=1.26
+
+FROM golang:${GO_VERSION} AS abra
+WORKDIR /src
+COPY abra/ ./
+COPY .git/modules/abra /gitdir
+# abra ships its dependencies in vendor/, so the build itself needs no network
+RUN set -eu; \
+  sed -i '/worktree =/d' /gitdir/config; \
+  version="$(git --git-dir=/gitdir describe --tags)"; \
+  commit="$(git --git-dir=/gitdir rev-parse HEAD)"; \
+  echo "building abra $version"; \
+  go build -mod=vendor -trimpath \
+  -ldflags "-s -w -X 'main.Version=$version' -X 'main.Commit=$commit'" \
+  -o /out/abra ./cmd/abra
+
+FROM python:3.11-slim AS deps
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends gcc libc6-dev \
+  && rm -rf /var/lib/apt/lists/*
+COPY requirements.txt /tmp/requirements.txt
+RUN python -m venv /opt/alakazam \
+  && /opt/alakazam/bin/pip install --no-cache-dir -r /tmp/requirements.txt
+
+FROM python:3.11-slim
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends git openssh-client make \
+  && rm -rf /var/lib/apt/lists/* \
+  && git config --global --add safe.directory '*'
+
+COPY --from=abra /out/abra /usr/local/bin/abra
+COPY --from=deps /opt/alakazam /opt/alakazam
+COPY alakazam.py combine.yml /opt/alakazam/
+RUN printf '#!/bin/sh\nexec /opt/alakazam/bin/python /opt/alakazam/alakazam.py "$@"\n' \
+  > /usr/local/bin/alakazam \
+  && chmod +x /usr/local/bin/alakazam
+
+# PYTHONUNBUFFERED so the streamed hook output arrives while a command is still running.
+# LANG pins abra to its untranslated messages, which alakazam matches on to tell a missing
+# secret or an empty generate run apart from a real failure.
+ENV PYTHONUNBUFFERED=1 \
+  LANG=C.UTF-8 \
+  ABRA_DIR=/root/.abra
+
+WORKDIR /config
+CMD ["alakazam", "--help"]

README.md

@@ -52,7 +52,7 @@ sudo apt install curl git python3.11-venv
 curl -sS https://webi.sh/golang | sh; \
 source ~/.config/envman/PATH.env
 # clone alakazam
-git clone --recursive https://git.coopcloud.tech/moritz/alakazam.git
+git clone --recursive https://git.coopcloud.tech/toolshed/alakazam.git
 # install alakazam
 cd alakazam
 ln -s $PWD/alakazam.sh ~/.local/bin/alakazam
@@ -62,6 +62,34 @@ alakazam install
 
 Create a global `~/.config/alakazam.yml` that contains at least the `root` path, see [Global Settings](#global-settings)
 
+### Run in Docker
+
+The image contains alakazam together with the `abra` version the submodule points at, so nothing has to be installed on the host:
+
+```bash
+docker build -t alakazam .
+```
+
+```bash
+docker run --rm -it \
+    -v /path/to/instance/configs:/config \
+    -v ~/.config/alakazam.yml:/root/.config/alakazam.yml:ro \
+    -v ~/.abra:/root/.abra \
+    -v ~/.ssh:/root/.ssh \
+    alakazam alakazam /config/example.com.yml ls
+```
+
+```bash
+docker run --rm -it alakazam abra --version
+docker run --rm -it alakazam bash
+```
+
+- **`/config`** holds the instance configurations. Set `root: /config` in `alakazam.yml`, since the path is resolved inside the container.
+- **`/root/.abra`** keeps the recipes, the catalogue and the generated app `.env` files across runs. Without it every run starts by cloning every recipe again. The path comes from `ABRA_DIR`; in a CI where each step is a fresh container, point it at the shared workspace instead, so that a server added in one step still exists in the next.
+- **`/root/.ssh`** provides the keys `abra` connects with. Do not mount it read-only if you use `ControlMaster`, as ssh needs to create its socket there.
+
+The image sets `git config --global safe.directory '*'`, because a mounted `~/.abra` belongs to the host user and git would otherwise refuse to touch the recipe repositories. It also pins `LANG=C.UTF-8`: alakazam recognises some `abra` results by their message, and a translated `abra` would not match.
+
 ### Create a new instance:
 
 To set up a new instance with Alakazam, begin by specifying the required applications in `example.com.yml`. The name of this file determines the domain for the instance.
@@ -91,6 +119,28 @@ Steps to initialize the instance:
 3. **Deploy Applications**: `alakazam example.com.yml deploy -e` 
     - the `-e` flag executes post-deployment hooks after each deployment
 
+Or let [`setup`](#setup-and-clean-deploy) do all of it in one command.
+
+### Setup and Clean Deploy
+
+`setup` builds a whole environment in one go:
+
+```
+alakazam example.com.yml setup -n
+```
+
+It writes the `.env` files, then walks the apps level by level in [dependency](#dependencies) order and runs secrets, deployment and [readiness hooks](#readiness-hooks) for each level, before waiting for every service to become healthy and running the `initial-hooks`. Apps of the same level do not wait for each other. Only a dependency makes an app wait, which is what removes the need to deploy an identity provider ahead of everything else by hand. Every failure along the way ends the run with a non-zero exit code, which is what makes it usable from a CI pipeline.
+
+`setup` runs on an **empty environment** only. 
+
+`clean-deploy` is the destructive variant — `undeploy`, `purge`, then `setup`:
+
+```
+alakazam example.com.yml clean-deploy -n
+```
+
+It deletes all data of the selected apps, so it has two safeguards: it insists on a single instance configuration file and refuses a group directory, and it asks for confirmation unless `-n` is given.
+
 ### Updating All Instances
 
 To update all instances:
@@ -98,6 +148,43 @@ To update all instances:
 1. **Update Environment Files**: `alakazam example.com.yml config` to refresh .env files. It's a good practice to keep these files under version control (e.g., in a Git repository at `~/.abra/example.com`) and review changes with `git diff` before proceeding.
 2. **Upgrade Applications**: `alakazam example.com.yml upgrade` to update all applications.
 
+### Rotate Secrets
+
+To replace secrets with freshly generated ones:
+
+```
+alakazam example.com.yml reinsert-secrets -s 'authentik:email_pass,nextcloud:smtp_password'
+```
+
+Secrets are addressed as `<RecipeName>:<SecretName>` pairs, comma separated, and `-s` can be specified multiple times.
+The command undeploys the affected apps, removes the listed secrets, creates them again the same way `secrets` does and redeploys the apps that were deployed before.
+
+- the `-e` flag executes post-deployment hooks after each deployment
+- a secret that is shared between apps has to be listed for every app holding a copy of it, otherwise the old value is copied back from the app that still has it
+
+Secrets are created in four variants: `conf` (values from the `secrets` configuration), `secret-hooks` (local `abra.sh` commands), `exchange` (`shared_secrets` between apps) and `generate` (everything left over, generated by abra).
+By default all four run; `-o`/`--only` limits them, both for `reinsert-secrets` and for `secrets`:
+
+```
+alakazam example.com.yml secrets --only conf,generate
+```
+
+To remove secrets without recreating them, `purge-secrets` uses the same notation:
+
+```
+alakazam example.com.yml purge-secrets -s 'authentik:email_pass'
+```
+
+Without `-s` all secrets of the recipes selected by `-r` are purged.
+
+### Waiting for Health
+
+`ps` shows the container status of the deployed apps. With `--wait` it blocks until every service has settled, which is what an automated deployment gates on:
+
+```
+alakazam example.com.yml ps --wait --timeout 300
+```
+
 ### Run CMDs
 
 Escaping can be akward:
@@ -106,6 +193,15 @@ alakazam example.com.yml cmd -c 'app run_occ ''"config:app:get sociallogin custo
 ```
 
 
+### Tests
+
+The test suite covers the pure logic and needs no `abra` installation:
+
+```
+pip install -r requirements-dev.txt
+pytest
+```
+
 ## Configuration
 
 ### Instance Configuration
@@ -154,12 +250,14 @@ For each app/recipe the following `<app_configurations>` can be used:
     - it matches against parts of the line (i.E. `compose.smtp.yml`)
     - this is useful for env variables that are used multiple times like `COMPOSE_FILE`
 - **`env`**: Sets values for environment variables.
-- **`*-hooks`**: Specifies `abra.sh` commands to run at specific stages.
+- **`*-hooks`**: Specifies `abra.sh` commands or local scripts to run at specific stages.
     - **`initial-hooks`**: commands for initialisation
     - **`deploy-hooks`**: commands that should be run after each deployment
     - **`upgrade-hooks`**: commands that should be run after each upgrade
+- **`readiness-hooks`**: Commands that decide whether an app is usable yet, repeated until one succeeds. See [Readiness Hooks](#readiness-hooks).
+- **`dependency`**: Names the apps that have to be set up before this one. See [Dependencies](#dependencies).
 - **`secrets`**: Inserts specific values (i.E. smtp passwords) into secrets; future updates will support encrypted file usage.
-- **`secret-hooks`**: Run `abra.sh` commands locally for secrets that need to be generated.
+- **`secret-hooks`**: Run `abra.sh` commands locally or local scripts for secrets that need to be generated.
 - **`subdomain`**: Specifies the subdomain scheme for individual recipes and apps. (not available in `combine.yml`/`alaconnect.yml`)
     - i.e. `cloud.example.com` for nextcloud
 - **`version`**: Controls the recipe version to deploy; if unspecified, the latest version is used. (not available in `combine.yml`/`alaconnect.yml`)
@@ -169,6 +267,52 @@ The `combine.yml`/`alaconnect.yml` configuration additionally contains:
 - **`shared_secrets`**: Specifies secret sharing between apps.
      - `<source_secret_name>:<target_secret_name>`
 
+### Dependencies
+
+An app can name the apps it needs in place before itself:
+
+```yaml
+nextcloud:
+    dependency: [authentik]
+```
+
+Alakazam builds a directed graph from those entries and processes the apps in topological order. For the configuration, the secrets and the deployment alike. Apps unrelated by a dependency keep their configuration order. [`setup`](#setup-and-clean-deploy) additionally groups them into levels: everything without a dependency forms the first level, apps depending only on those the second, and so on.
+
+### Readiness Hooks
+
+An app can be deployed and healthy while still not being usable — authentik accepts connections long before it has applied its blueprints. A readiness hook is an `abra.sh` command of the recipe that answers that question and exits zero once the app is ready:
+
+```yaml
+authentik:
+    readiness-hooks:
+        - cmd: app check_blueprints
+          retries: 20
+          interval: 120
+          initial-delay: 60
+```
+
+`cmd` follows the same notation as the other hooks: the first token is the container to run in, or `local`. The first attempt starts after `initial-delay` seconds, further attempts follow every `interval` seconds, at most `retries` times. If none of them succeeds, the run aborts with a non-zero exit code instead of deploying on top of a half-ready app. The defaults are 20 retries, 60 seconds apart, with no initial delay.
+
+Readiness hooks run as part of [`setup`](#setup-and-clean-deploy) only, right after the app they belong to is deployed and before anything that depends on it. `deploy` does not run them, so it keeps returning as soon as the deployment is through.
+
+### \*-Hooks Command Formats
+
+**Abra command** — runs an abra.sh command inside a container:
+
+```yaml
+initial-hooks:
+  - app set_default_quota
+```
+
+**Local script** — runs a script on the local machine (for custom actions that aren't generic enough to be implemented as abra.sh commands):
+
+```yaml
+initial-hooks:
+  - script ./scripts/script.sh arg1
+```
+
+Relative paths resolve from the `root` path. The script receives `ALAKAZAM_APP_DOMAIN`, `ALAKAZAM_APP_SERVER`, and `ALAKAZAM_INSTANCE_DOMAIN` as environment variables.
+
 ### Configuration Structure
 
 Configuration can be simplified into a single `example.com.yml` or expanded into multiple layered `alaka.yml`/`alaka-*.yml` files for complex deployments. This allows for easy maintenance of multiple instances or groups.
@@ -230,6 +374,16 @@ Alakazam supports a flexible inheritance model for configuration management, all
 - **Grouped Instances**: For more complex setups, instances can be organized into folders, each with its own `alaka.yml`/`alaka-*.yml`. This structure allows for group-specific configurations, making it easier to manage settings at different levels of your infrastructure hierarchy.
 - **Inherited Configurations**: In even more complex scenarios, each group can inherit configurations from the levels above it. This cascading setup ensures that changes at a higher level can be automatically applied to all subordinate groups and instances, maintaining consistency and ease of updates across your entire environment.
 
+### Merge Order
+
+Within one directory the configuration files are merged in a fixed order, from weakest to strongest:
+
+1. the merged configuration inherited from the parent directory
+2. `alaka.yml`
+3. the `alaka-*.yml` files in alphabetical order
+
+So a key set in `alaka-versions.yml` overrides the same key in `alaka.yml`, and both override the parent directory. Lists are the exception: they are not overridden but concatenated in merge order.
+
 This hierarchical approach to configuration is designed to scale with your system’s complexity and can be visualized through the following example diagram:
 
 ![File Structure](./config_file_structure.png)

combine.yml

@@ -49,6 +49,7 @@ authentik:
             - SECRET_KIMAI_ID_VERSION
             - SECRET_KIMAI_SECRET_VERSION
             - kimai_logo.png
+            - KIMAI_GROUP
     zammad:
         uncomment:
             - compose.zammad.yml
@@ -123,6 +124,7 @@ kimai:
             - SSO_LOGOUT_URL
         secret_hooks:
             - insert_authentik_certificate
+        dependency: [authentik]
 zammad:
     authentik:
         uncomment:
@@ -131,6 +133,7 @@ zammad:
             - IDP_SLO_SERVICE_URL
         initial-hooks:
             - local enable_authentik_sso
+        dependency: [authentik]
 nextcloud:
     authentik:
         uncomment:
@@ -225,6 +228,7 @@ matrix-synapse:
             - SECRET_KEYCLOAK_CLIENT_SECRET_VERSION
         shared_secrets:
             matrix_secret: keycloak_client_secret
+        dependency: [authentik]
 traefik:
     matrix-synapse:
         uncomment:
@@ -281,6 +285,7 @@ hedgedoc:
             - SECRET_OAUTH_KEY_VERSION
         shared_secrets:
             hedgedoc_secret: oauth_key
+        dependency: [authentik]
 mila:
     authentik:
         env:

requirements-dev.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest==9.1.1

tests/test_backup.py

@@ -0,0 +1,22 @@
+"""Tests for the backupbot lookup of the backup command."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from click.testing import CliRunner
+
+
+class TestBackupBotLookup:
+    def test_a_server_without_a_backupbot_is_reported_not_crashed(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "get_server_apps",
+                            lambda apps=None, all=False: {} if all else {"a.org": ["login.a.org"]})
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        from click.testing import CliRunner
+        result = CliRunner().invoke(alakazam.backup, [], standalone_mode=False)
+        assert result.exit_code == 0, result.exception

tests/test_dependencies.py

@@ -0,0 +1,150 @@
+"""Tests for the dependency ordering that replaces the special casing of authentik."""
+
+import logging
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_apps, group_apps_by_dependency, sort_apps_by_dependency
+
+
+def instance(**apps):
+    """Builds an instance configuration, values are the 'dependency' lists."""
+    return {name: {"app_domain": f"{name}.a.org", "dependency": list(dependencies)} for name, dependencies in apps.items()}
+
+
+class TestSortAppsByDependency:
+    def test_a_dependency_comes_first(self):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        assert sort_apps_by_dependency(["nextcloud", "authentik"], config) == ["authentik", "nextcloud"]
+
+    def test_order_is_independent_of_the_input_order(self):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        assert sort_apps_by_dependency(["authentik", "nextcloud"], config) == ["authentik", "nextcloud"]
+
+    def test_several_apps_share_one_dependency(self):
+        config = instance(nextcloud=["authentik"], wekan=["authentik"], authentik=[])
+        ordered = sort_apps_by_dependency(["nextcloud", "wekan", "authentik"], config)
+        assert ordered.index("authentik") == 0
+
+    def test_transitive_dependencies(self):
+        config = instance(c=["b"], b=["a"], a=[])
+        assert sort_apps_by_dependency(["c", "b", "a"], config) == ["a", "b", "c"]
+
+    def test_unrelated_apps_keep_their_configuration_order(self):
+        config = instance(traefik=[], authentik=[], nextcloud=[])
+        apps = ["traefik", "authentik", "nextcloud"]
+        assert sort_apps_by_dependency(apps, config) == apps
+
+    def test_apps_without_a_dependency_key(self):
+        config = {"authentik": {"app_domain": "login.a.org"}, "nextcloud": {"app_domain": "cloud.a.org"}}
+        assert sort_apps_by_dependency(["nextcloud", "authentik"], config) == ["nextcloud", "authentik"]
+
+    def test_every_app_is_returned_exactly_once(self):
+        config = instance(d=["a", "b"], c=["a"], b=["a"], a=[])
+        ordered = sort_apps_by_dependency(["d", "c", "b", "a"], config)
+        assert sorted(ordered) == ["a", "b", "c", "d"]
+        assert len(ordered) == 4
+
+    def test_empty_selection(self):
+        assert sort_apps_by_dependency([], instance(a=[])) == []
+
+
+class TestGroupAppsByDependency:
+    """Levels express what a flat order cannot: which apps may proceed without waiting."""
+
+    def test_apps_sharing_one_dependency_end_up_in_one_level(self):
+        config = instance(zammad=["authentik"], kimai=["authentik"], hedgedoc=["authentik"],
+                          authentik=[], traefik=[])
+        levels = group_apps_by_dependency(["zammad", "kimai", "hedgedoc", "authentik", "traefik"], config)
+        assert levels == [["authentik", "traefik"], ["zammad", "kimai", "hedgedoc"]]
+
+    def test_unrelated_apps_are_all_in_the_first_level(self):
+        config = instance(traefik=[], authentik=[], nextcloud=[])
+        assert group_apps_by_dependency(["traefik", "authentik", "nextcloud"], config) == [
+            ["traefik", "authentik", "nextcloud"]
+        ]
+
+    def test_a_transitive_chain_yields_one_level_each(self):
+        config = instance(c=["b"], b=["a"], a=[])
+        assert group_apps_by_dependency(["c", "b", "a"], config) == [["a"], ["b"], ["c"]]
+
+    def test_the_deepest_dependency_decides_the_level(self):
+        """An app waits for all of its dependencies, so it lands behind the latest of them."""
+        config = instance(d=["a", "c"], c=["b"], b=["a"], a=[])
+        assert group_apps_by_dependency(["d", "c", "b", "a"], config) == [["a"], ["b"], ["c"], ["d"]]
+
+    def test_configuration_order_is_kept_within_a_level(self):
+        config = instance(wekan=["authentik"], nextcloud=["authentik"], authentik=[])
+        assert group_apps_by_dependency(["wekan", "nextcloud", "authentik"], config)[1] == ["wekan", "nextcloud"]
+
+    def test_a_dependency_outside_the_selection_does_not_add_a_level(self):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        assert group_apps_by_dependency(["nextcloud"], config) == [["nextcloud"]]
+
+    def test_empty_selection(self):
+        assert group_apps_by_dependency([], instance(a=[])) == []
+
+    def test_every_app_appears_exactly_once(self):
+        config = instance(d=["a", "b"], c=["a"], b=["a"], a=[])
+        levels = group_apps_by_dependency(["d", "c", "b", "a"], config)
+        assert sorted(app for level in levels for app in level) == ["a", "b", "c", "d"]
+
+    def test_a_cycle_is_rejected(self):
+        config = instance(nextcloud=["authentik"], authentik=["nextcloud"])
+        with pytest.raises(click.ClickException):
+            group_apps_by_dependency(["nextcloud", "authentik"], config)
+
+
+class TestCycles:
+    def test_a_cycle_is_rejected(self):
+        config = instance(nextcloud=["authentik"], authentik=["nextcloud"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["nextcloud", "authentik"], config)
+        assert excinfo.value.exit_code != 0
+
+    def test_the_cycle_is_named(self):
+        config = instance(nextcloud=["authentik"], authentik=["nextcloud"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["nextcloud", "authentik"], config)
+        assert "nextcloud -> authentik -> nextcloud" in excinfo.value.message
+
+    def test_a_longer_cycle_is_named(self):
+        config = instance(a=["c"], b=["a"], c=["b"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["a", "b", "c"], config)
+        assert "a -> c -> b -> a" in excinfo.value.message
+
+    def test_an_app_depending_on_itself(self):
+        config = instance(a=["a"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["a"], config)
+        assert "a -> a" in excinfo.value.message
+
+
+class TestDependenciesOutsideTheSelection:
+    def test_a_dependency_not_selected_is_skipped(self, caplog):
+        """Filtering a run down to a few recipes must not pull in anything else."""
+        config = instance(nextcloud=["authentik"], authentik=[])
+        with caplog.at_level(logging.WARNING):
+            assert sort_apps_by_dependency(["nextcloud"], config) == ["nextcloud"]
+        assert caplog.text == ""
+
+class TestGetAppsOrdering:
+    """The order has to reach the commands, not just the helper."""
+
+    def test_get_apps_returns_dependencies_first(self, monkeypatch):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": config})
+        assert [app for app, _ in get_apps()["a.org"]] == ["authentik", "nextcloud"]
+
+    def test_an_explicit_recipe_filter_is_ordered_too(self, monkeypatch):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": config})
+        apps = get_apps(("nextcloud", "authentik"))["a.org"]
+        assert [app for app, _ in apps] == ["authentik", "nextcloud"]

tests/test_fetch.py

@@ -0,0 +1,148 @@
+"""Tests for the recipe refresh that keeps the offline abra calls working on current recipes."""
+
+import os
+import sys
+from pathlib import Path
+
+import pytest
+from git import Repo
+from git.exc import GitCommandError
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import fetch_recipes, sync_repo
+
+INSTANCE_CONFIGS = {"a.org": {"authentik": {}, "nextcloud": {}}}
+
+
+def commit(repo, name, content="x"):
+    """Adds a file and commits it, returning the new commit."""
+    path = Path(repo.working_dir) / name
+    path.write_text(content)
+    repo.index.add([name])
+    return repo.index.commit(f"add {name}")
+
+
+@pytest.fixture
+def remote_and_clone(tmp_path):
+    """A bare-ish upstream repository plus a clone of it, as abra would leave one behind."""
+    upstream = Repo.init(tmp_path / "upstream", initial_branch="main")
+    upstream.config_writer().set_value("user", "name", "test").release()
+    upstream.config_writer().set_value("user", "email", "test@example.org").release()
+    commit(upstream, "compose.yml")
+    clone = Repo.clone_from(str(tmp_path / "upstream"), str(tmp_path / "clone"))
+    clone.config_writer().set_value("user", "name", "test").release()
+    clone.config_writer().set_value("user", "email", "test@example.org").release()
+    return upstream, clone
+
+
+class TestSyncRepo:
+    def test_fast_forwards_to_the_remote(self, remote_and_clone):
+        upstream, clone = remote_and_clone
+        commit(upstream, "abra.sh")
+        sync_repo(Path(clone.working_dir))
+        assert (Path(clone.working_dir) / "abra.sh").exists()
+
+    def test_fetches_tags(self, remote_and_clone):
+        """A pinned recipe version is a tag, so abra needs them even without a fast-forward."""
+        upstream, clone = remote_and_clone
+        upstream.create_tag("1.2.3")
+        sync_repo(Path(clone.working_dir))
+        assert "1.2.3" in [tag.name for tag in clone.tags]
+
+    def test_writes_the_fetch_head_marker(self, remote_and_clone):
+        _, clone = remote_and_clone
+        marker = Path(clone.working_dir) / ".git" / "FETCH_HEAD"
+        marker.unlink(missing_ok=True)
+        sync_repo(Path(clone.working_dir))
+        assert marker.exists()
+
+    def test_a_checked_out_version_is_left_alone(self, remote_and_clone):
+        """abra checks a pinned version out itself, moving it here would fight with that."""
+        upstream, clone = remote_and_clone
+        upstream.create_tag("1.2.3")
+        sync_repo(Path(clone.working_dir))
+        clone.git.checkout("1.2.3")
+        commit(upstream, "abra.sh")
+        sync_repo(Path(clone.working_dir))
+        assert clone.head.is_detached
+        assert not (Path(clone.working_dir) / "abra.sh").exists()
+
+    def test_a_dirty_repository_is_left_alone(self, remote_and_clone, caplog):
+        upstream, clone = remote_and_clone
+        (Path(clone.working_dir) / "compose.yml").write_text("local change")
+        commit(upstream, "abra.sh")
+        sync_repo(Path(clone.working_dir))
+        assert (Path(clone.working_dir) / "compose.yml").read_text() == "local change"
+        assert "uncommitted changes" in caplog.text
+
+    def test_a_repository_without_a_remote_is_skipped(self, tmp_path):
+        repo = Repo.init(tmp_path / "solo", initial_branch="main")
+        repo.config_writer().set_value("user", "name", "test").release()
+        repo.config_writer().set_value("user", "email", "test@example.org").release()
+        commit(repo, "compose.yml")
+        sync_repo(tmp_path / "solo")
+
+    def test_an_unreachable_remote_raises(self, tmp_path, remote_and_clone):
+        """fetch_recipes has to catch this, a broken forge must not end the run."""
+        _, clone = remote_and_clone
+        clone.remotes[0].set_url(str(tmp_path / "gone"))
+        with pytest.raises(GitCommandError):
+            sync_repo(Path(clone.working_dir))
+
+
+class TestFetchRecipes:
+    @pytest.fixture
+    def abra_dir(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(alakazam, "ABRA_DIR", tmp_path)
+        synced = []
+        monkeypatch.setattr(alakazam, "sync_repo", synced.append)
+        for name in ("catalogue", "recipes/authentik", "recipes/nextcloud"):
+            (tmp_path / name / ".git").mkdir(parents=True)
+        return synced
+
+    def test_syncs_the_catalogue_and_every_used_recipe(self, tmp_path, abra_dir):
+        """The catalogue feeds 'abra recipe versions', leaving it out freezes version lookups."""
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert [p.name for p in abra_dir] == ["catalogue", "authentik", "nextcloud"]
+
+    def test_a_recently_fetched_repository_is_skipped(self, tmp_path, abra_dir):
+        (tmp_path / "recipes/authentik/.git/FETCH_HEAD").touch()
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert "authentik" not in [p.name for p in abra_dir]
+
+    def test_a_stale_marker_does_not_skip(self, tmp_path, abra_dir):
+        marker = tmp_path / "recipes/authentik/.git/FETCH_HEAD"
+        marker.touch()
+        os.utime(marker, (0, 0))
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert "authentik" in [p.name for p in abra_dir]
+
+    def test_the_marker_is_written_even_without_a_remote(self, tmp_path, abra_dir):
+        """A fetch writes FETCH_HEAD itself, this is the fallback for the paths that do not fetch."""
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert (tmp_path / "recipes/authentik/.git/FETCH_HEAD").exists()
+
+    def test_a_failing_repository_does_not_stop_the_others(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(alakazam, "ABRA_DIR", tmp_path)
+        synced = []
+
+        def failing(path):
+            synced.append(path)
+            if path.name == "authentik":
+                raise GitCommandError("git fetch", 128, b"could not resolve host")
+
+        monkeypatch.setattr(alakazam, "sync_repo", failing)
+        for name in ("catalogue", "recipes/authentik", "recipes/nextcloud"):
+            (tmp_path / name / ".git").mkdir(parents=True)
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert [p.name for p in synced] == ["catalogue", "authentik", "nextcloud"]
+
+    def test_a_recipe_that_was_never_cloned_is_tolerated(self, tmp_path, monkeypatch):
+        """abra clones a missing recipe itself, EnsureExists runs before its offline check."""
+        monkeypatch.setattr(alakazam, "ABRA_DIR", tmp_path)
+        monkeypatch.setattr(alakazam, "sync_repo", alakazam.sync_repo)
+        (tmp_path / "catalogue" / ".git").mkdir(parents=True)
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert not (tmp_path / "recipes/authentik").exists()

tests/test_group_configs.py

@@ -0,0 +1,194 @@
+"""Tests for the group configuration merge order and its collision reporting."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from alakazam import (
+    deduplicate,
+    iter_leaf_paths,
+    merge_all_group_configs,
+    sort_group_config_files,
+    warn_on_key_collisions,
+)
+
+
+def write_config(directory, name, content):
+    """Writes a group configuration file and returns its path."""
+    path = directory / name
+    path.write_text(content)
+    return path
+
+
+class TestSortGroupConfigFiles:
+    def test_selects_only_group_configs(self):
+        files = ["alaka.yml", "example.com.yml", "config-sets.yml", "alaka-smtp.yaml", "notes.md"]
+        assert sort_group_config_files(files) == ["alaka.yml", "alaka-smtp.yaml"]
+
+    def test_base_config_comes_first(self):
+        files = ["alaka-versions.yml", "alaka.yml", "alaka-defaults.yml"]
+        assert sort_group_config_files(files) == ["alaka.yml", "alaka-defaults.yml", "alaka-versions.yml"]
+
+    def test_order_is_independent_of_the_input_order(self):
+        files = ["alaka.yml", "alaka-a.yml", "alaka-b.yml"]
+        assert sort_group_config_files(files) == sort_group_config_files(list(reversed(files)))
+
+    def test_empty_listing(self):
+        assert sort_group_config_files([]) == []
+
+
+class TestIterLeafPaths:
+    def test_nested_dicts_become_key_paths(self):
+        leaves = dict(iter_leaf_paths({"authentik": {"env": {"SMTP_HOST": "mail.example.com"}}}))
+        assert leaves == {("authentik", "env", "SMTP_HOST"): "mail.example.com"}
+
+    def test_lists_are_leaves(self):
+        leaves = dict(iter_leaf_paths({"authentik": {"initial-hooks": ["app one", "app two"]}}))
+        assert leaves == {("authentik", "initial-hooks"): ["app one", "app two"]}
+
+
+class TestDeduplicate:
+    def test_keeps_the_first_occurrence(self):
+        assert deduplicate(["a", "b", "a", "c", "b"]) == ["a", "b", "c"]
+
+    def test_lists_of_mappings(self):
+        """A readiness-hooks entry is a mapping, which is unhashable and needs equality."""
+        hooks = [{"cmd": "app one"}, {"cmd": "app two"}, {"cmd": "app one"}]
+        assert deduplicate(hooks) == [{"cmd": "app one"}, {"cmd": "app two"}]
+
+    def test_mappings_differing_in_one_field_are_kept(self):
+        hooks = [{"cmd": "app one", "retries": 1}, {"cmd": "app one", "retries": 2}]
+        assert deduplicate(hooks) == hooks
+
+    def test_empty_list(self):
+        assert deduplicate([]) == []
+
+
+class TestMergeOrder:
+    def test_specialisations_override_the_base_config(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n")
+        write_config(tmp_path, "alaka-versions.yml", "authentik:\n    version: 2.0.0\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(tmp_path)]["authentik"]["version"] == "2.0.0"
+
+    def test_later_specialisation_wins_alphabetically(self, tmp_path):
+        write_config(tmp_path, "alaka-a.yml", "authentik:\n    version: 1.0.0\n")
+        write_config(tmp_path, "alaka-b.yml", "authentik:\n    version: 2.0.0\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(tmp_path)]["authentik"]["version"] == "2.0.0"
+
+    def test_result_is_stable_across_directory_orderings(self, tmp_path):
+        """The same content laid down in a different creation order must merge identically."""
+        results = []
+        for order in (("alaka.yml", "alaka-smtp.yml"), ("alaka-smtp.yml", "alaka.yml")):
+            directory = tmp_path / "_".join(order)
+            directory.mkdir()
+            contents = {
+                "alaka.yml": "authentik:\n    version: 1.0.0\n    env:\n        SMTP_HOST: base\n",
+                "alaka-smtp.yml": "authentik:\n    env:\n        SMTP_HOST: special\n",
+            }
+            for name in order:
+                write_config(directory, name, contents[name])
+            results.append(merge_all_group_configs(directory)[str(directory)])
+        assert results[0] == results[1]
+        assert results[0]["authentik"]["env"]["SMTP_HOST"] == "special"
+
+    def test_subdirectory_inherits_and_overrides_the_parent(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n    server: parent\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml", "authentik:\n    version: 2.0.0\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)]["authentik"] == {"version": "2.0.0", "server": "parent"}
+
+    def test_directory_without_group_config_inherits_the_parent(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)] == merged[str(tmp_path)]
+
+    def test_lists_of_sibling_files_are_concatenated_in_merge_order(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    initial-hooks:\n        - app first\n")
+        write_config(tmp_path, "alaka-extra.yml", "authentik:\n    initial-hooks:\n        - app second\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(tmp_path)]["authentik"]["initial-hooks"] == ["app first", "app second"]
+
+
+class TestMergingMappingLists:
+    """'readiness-hooks' is the first configuration key whose list holds mappings."""
+
+    def test_a_child_directory_adds_to_the_parents_hooks(self, tmp_path):
+        write_config(tmp_path, "alaka.yml",
+                     "authentik:\n    readiness-hooks:\n        - cmd: app check_blueprints\n          retries: 20\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml",
+                     "authentik:\n    readiness-hooks:\n        - cmd: app check_users\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)]["authentik"]["readiness-hooks"] == [
+            {"cmd": "app check_blueprints", "retries": 20},
+            {"cmd": "app check_users"},
+        ]
+
+    def test_an_identical_hook_is_not_repeated(self, tmp_path):
+        hook = "authentik:\n    readiness-hooks:\n        - cmd: app check_blueprints\n"
+        write_config(tmp_path, "alaka.yml", hook)
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml", hook)
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)]["authentik"]["readiness-hooks"] == [{"cmd": "app check_blueprints"}]
+
+
+class TestKeyCollisions:
+    def test_conflicting_scalar_is_reported(self, caplog):
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"version": "2.0.0"}}, defined_by, "alaka-versions.yml", "/root")
+        assert "authentik.version" in caplog.text
+        assert "alaka-versions.yml takes precedence" in caplog.text
+
+    def test_concatenated_lists_are_not_reported(self, caplog):
+        """Composing one list from several files is intentional, not an override."""
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"initial-hooks": ["app a"]}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"initial-hooks": ["app b"]}}, defined_by, "alaka-extra.yml", "/root")
+        assert caplog.text == ""
+
+    def test_a_list_replacing_a_scalar_is_reported(self, caplog):
+        """merge_dict only concatenates when both sides are lists, otherwise the value is replaced."""
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"version": ["2.0.0"]}}, defined_by, "alaka-versions.yml", "/root")
+        assert "authentik.version" in caplog.text
+
+    def test_identical_values_are_not_reported(self, caplog):
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka-versions.yml", "/root")
+        assert caplog.text == ""
+
+    def test_distinct_keys_are_not_reported(self, caplog):
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"nextcloud": {"version": "2.0.0"}}, defined_by, "alaka-nextcloud.yml", "/root")
+        assert caplog.text == ""
+
+    def test_overriding_a_parent_directory_is_not_reported(self, tmp_path, caplog):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml", "authentik:\n    version: 2.0.0\n")
+        with caplog.at_level(logging.WARNING):
+            merge_all_group_configs(tmp_path)
+        assert caplog.text == ""

tests/test_hide_secrets.py

@@ -0,0 +1,100 @@
+"""Tests for keeping secret values out of the output of a run whose logs are kept."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import generate_all_secrets, insert_secret, is_secret_command
+
+VALUE = "hunter2-do-not-log-me"
+
+
+@pytest.fixture(params=[False, True], ids=["visible", "hidden"])
+def hide(request, monkeypatch):
+    monkeypatch.setattr(alakazam, "HIDE_SECRETS", request.param)
+    return request.param
+
+
+class TestIsSecretCommand:
+    @pytest.mark.parametrize("args", [
+        ("app", "secret", "insert", "login.a.org", "db_password", "v1", VALUE),
+        ("app", "secret", "generate", "-a", "login.a.org"),
+    ])
+    def test_commands_carrying_a_value(self, args):
+        assert is_secret_command(args)
+
+    @pytest.mark.parametrize("args", [
+        ("app", "secret", "ls", "login.a.org"),
+        ("app", "secret", "rm", "login.a.org", "db_password"),
+        ("app", "ls"),
+        ("app", "secret"),
+    ])
+    def test_commands_without_a_value(self, args):
+        assert not is_secret_command(args)
+
+
+class TestGeneratedValues:
+    def install(self, monkeypatch, created=False):
+        def abra(*args, **kwargs):
+            if args[:3] == ("app", "secret", "ls"):
+                return [{"name": "db_password", "created on server": str(created).lower()}]
+            if args[:3] == ("app", "secret", "generate"):
+                return [{"name": "db_password", "value": VALUE}]
+            raise AssertionError(f"unexpected: {args}")
+        monkeypatch.setattr(alakazam, "abra", abra)
+
+    def test_the_name_is_always_reported(self, hide, monkeypatch, capsys):
+        self.install(monkeypatch)
+        generate_all_secrets("login.a.org")
+        assert "db_password" in capsys.readouterr().out
+
+    def test_the_value_follows_the_switch(self, hide, monkeypatch, capsys):
+        self.install(monkeypatch)
+        generate_all_secrets("login.a.org")
+        out = capsys.readouterr().out
+        assert (VALUE in out) is not hide
+        assert ("[hidden]" in out) is hide
+
+
+class TestDebugLog:
+    """-l debug is the level a pipeline reaches for when something breaks."""
+
+    def run_abra(self, monkeypatch, args):
+        class Process:
+            returncode = 0
+            stdout = f'[{{"name":"db_password","value":"{VALUE}"}}]'.encode()
+            stderr = b""
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda cmd, capture_output: Process())
+        return alakazam.abra(*args)
+
+    def test_the_command_line_is_not_logged_when_hidden(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "secret", "insert", "login.a.org", "db_password", "v1", VALUE))
+        assert VALUE not in caplog.text
+
+    def test_the_generated_output_is_not_logged_when_hidden(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "secret", "generate", "-a", "login.a.org"))
+        assert VALUE not in caplog.text
+
+    def test_other_commands_are_still_logged(self, monkeypatch, caplog):
+        """Hiding secrets must not turn the debug log off altogether."""
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "ls"))
+        assert "run command" in caplog.text
+
+    def test_insert_logs_the_name_but_not_the_value(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        with caplog.at_level(logging.DEBUG):
+            insert_secret("login.a.org", "db_password", VALUE)
+        assert "db_password" in caplog.text
+        assert VALUE not in caplog.text

tests/test_local_scripts.py

@@ -0,0 +1,184 @@
+"""Tests for local script functionality in hooks."""
+
+import logging
+import os
+import stat
+import sys
+
+import click
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import (create_secrets, deploy_apps, execute_cmds, get_abra_dir,
+                       resolve_path, run_local_script, run_secret_hooks)
+
+
+class FakeProcess:
+    def __init__(self, returncode):
+        self.returncode = returncode
+
+
+def make_script(tmp_path, name="script.sh", executable=True):
+    """Write a script file under tmp_path, executable by default."""
+    path = tmp_path / name
+    path.write_text("#!/bin/sh\nexit 0\n")
+    if executable:
+        path.chmod(path.stat().st_mode | stat.S_IXUSR)
+    return path
+
+
+class TestResolvePath:
+    def test_an_absolute_path_passes_through_unchanged(self, tmp_path):
+        absolute = tmp_path / "script.sh"
+        assert resolve_path(str(absolute)) == absolute
+
+    def test_a_tilde_path_expands_against_home(self, monkeypatch, tmp_path):
+        monkeypatch.setenv("HOME", str(tmp_path))
+        assert resolve_path("~/script.sh") == tmp_path / "script.sh"
+
+    def test_a_relative_path_resolves_against_the_given_base(self, tmp_path):
+        base = tmp_path / "instance"
+        assert resolve_path("scripts/script.sh", base) == base / "scripts/script.sh"
+
+    def test_a_relative_path_resolves_against_root_path_by_default(self, monkeypatch, tmp_path):
+        monkeypatch.setattr(alakazam, "ROOT_PATH", tmp_path)
+        assert resolve_path("scripts/script.sh") == tmp_path / "scripts/script.sh"
+
+
+class TestRunLocalScript:
+    def test_the_script_receives_its_env_vars_and_arguments(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        calls = []
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda cmd, env: calls.append((cmd, env)) or FakeProcess(0))
+        run_local_script(["script", str(script), "arg1"], "login.a.org", "a.org", "a.org")
+        [(cmd, env)] = calls
+        assert cmd == [str(script), "arg1"]
+        assert env["ALAKAZAM_APP_DOMAIN"] == "login.a.org"
+        assert env["ALAKAZAM_APP_SERVER"] == "a.org"
+        assert env["ALAKAZAM_INSTANCE_DOMAIN"] == "a.org"
+
+    def test_a_non_executable_script_is_not_run(self, monkeypatch, tmp_path, caplog):
+        script = make_script(tmp_path, executable=False)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda *a, **k: pytest.fail("a non-executable script must not run"))
+        with caplog.at_level(logging.ERROR):
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org")
+        assert "not executable" in caplog.text
+
+    def test_dry_run_does_not_execute_the_script(self, monkeypatch, tmp_path, capsys):
+        script = make_script(tmp_path)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda *a, **k: pytest.fail("dry_run must not execute anything"))
+        run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org", dry_run=True)
+        assert "Run local script" in capsys.readouterr().out
+
+    def test_a_non_zero_exit_only_warns_by_default(self, monkeypatch, tmp_path, caplog):
+        script = make_script(tmp_path)
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda *a, **k: FakeProcess(1))
+        with caplog.at_level(logging.WARNING):
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org")
+        assert "exited with code 1" in caplog.text
+
+    def test_a_non_zero_exit_aborts_in_strict_mode(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda *a, **k: FakeProcess(1))
+        with pytest.raises(click.ClickException) as excinfo:
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org", strict=True)
+        assert "exited with code 1" in excinfo.value.message
+
+    def test_a_non_executable_script_aborts_in_strict_mode(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path, executable=False)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda *a, **k: pytest.fail("a non-executable script must not run"))
+        with pytest.raises(click.ClickException) as excinfo:
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org", strict=True)
+        assert "not executable" in excinfo.value.message
+
+
+class TestScriptHookDispatch:
+    def test_a_script_secret_hook_dispatches_to_run_local_script(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        calls = []
+        monkeypatch.setattr(alakazam, "run_local_script", lambda *a, **k: calls.append((a, k)))
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: pytest.fail("should not call abra"))
+        run_secret_hooks("login.a.org", {"secret_hooks": [f"script {script}"], "server": "a.org"}, "a.org")
+        [(args, kwargs)] = calls
+        assert args == (["script", str(script)], "login.a.org", "a.org", "a.org")
+
+    def test_a_normal_secret_hook_goes_through_abra(self, monkeypatch):
+        calls = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append((a, k)) or "")
+        run_secret_hooks("login.a.org", {"secret_hooks": ["insert_cert"], "server": "a.org"})
+        assert calls, "a non-script hook must still run through abra"
+
+    def test_a_script_command_dispatches_to_run_local_script(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        calls = []
+        monkeypatch.setattr(alakazam, "run_local_script", lambda *a, **k: calls.append((a, k)))
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: pytest.fail("should not call abra"))
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org",
+                       "initial-hooks": [f"script {script} arg1"]}, initial=True)
+        [(args, kwargs)] = calls
+        assert args == (["script", str(script), "arg1"], "login.a.org", "a.org", "", False, False)
+
+    def test_a_normal_command_goes_through_abra(self, monkeypatch):
+        calls = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append((a, k)) or "")
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org",
+                       "initial-hooks": ["app set_default_quota"]}, initial=True)
+        assert calls, "a non-script command must still run through abra"
+
+    def test_a_script_token_with_a_missing_file_falls_back_to_abra(self, monkeypatch, tmp_path):
+        """A 'script' hook whose path does not exist is not a script hook at all, just an abra
+        command whose container happens to be named 'script'."""
+        missing = tmp_path / "does-not-exist.sh"
+        calls = []
+        monkeypatch.setattr(alakazam, "run_local_script",
+                            lambda *a, **k: pytest.fail("should not run a missing script"))
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append((a, k)) or "")
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org",
+                       "initial-hooks": [f"script {missing}"]}, initial=True)
+        assert calls, "a missing script path must fall back to a normal abra command"
+
+
+class TestInstanceDomainThreading:
+    """create_secrets(), deploy_apps() and the 'cmd' CLI command each thread instance_domain
+    into run_secret_hooks()/execute_cmds() for script hooks."""
+
+    @pytest.fixture
+    def app_config(self, monkeypatch):
+        """A single 'authentik' app on instance 'a.org', shared by tests below."""
+        config = {"app_domain": "login.a.org", "server": "a.org"}
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": {"authentik": config}})
+        return config
+
+    def test_create_secrets_threads_the_instance_domain(self, monkeypatch, app_config):
+        calls = []
+        monkeypatch.setattr(alakazam, "run_secret_hooks", lambda *a, **k: calls.append((a, k)))
+        create_secrets(recipes=(), variants={"secret-hooks"})
+        [(args, kwargs)] = calls
+        assert args == ("login.a.org", app_config, "a.org")
+
+    def test_deploy_apps_threads_the_instance_domain(self, monkeypatch, app_config):
+        app_config["version"] = "1.0.0"
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        calls = []
+        monkeypatch.setattr(alakazam, "execute_cmds", lambda app_config, **k: calls.append(k))
+        deploy_apps({"a.org": [["authentik", "login.a.org"]]}, execute_hooks=True)
+        [kwargs] = calls
+        assert kwargs["deploy"] is True
+        assert kwargs["instance_domain"] == "a.org"
+
+    def test_cmd_threads_the_instance_domain(self, monkeypatch, app_config):
+        monkeypatch.setattr(alakazam, "get_deployed_apps", lambda apps: {"login.a.org": "1.0.0"})
+        calls = []
+        monkeypatch.setattr(alakazam, "execute_cmds", lambda app_config, **k: calls.append(k))
+        result = CliRunner().invoke(alakazam.cmd, ["-i"], standalone_mode=False)
+        assert result.exception is None
+        [kwargs] = calls
+        assert kwargs["initial"] is True
+        assert kwargs["instance_domain"] == "a.org"

tests/test_preflight.py

@@ -0,0 +1,226 @@
+"""Tests for the configuration preflight that runs before any abra call."""
+
+import os
+import sys
+
+import click
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import check_config_readable, get_relevant_config_paths, preflight_configs
+
+# a git-crypt encrypted file starts with this marker and is not valid UTF-8
+GIT_CRYPT_HEADER = b"\x00GITCRYPT\x00\xeb\x1c\x9a\x7f\x3d\x00\xff\xfe"
+
+
+@pytest.fixture
+def config_root(tmp_path):
+    """A root path holding one group with one instance, mirroring the documented layout."""
+    root = tmp_path / "root"
+    group = root / "group"
+    group.mkdir(parents=True)
+    (root / "alaka.yml").write_text("authentik:\n    version: 1.0.0\n")
+    (root / "config-sets.yml").write_text("bbb:\n    authentik:\n        env:\n            A: b\n")
+    (group / "alaka-versions.yml").write_text("authentik:\n    version: 2.0.0\n")
+    (group / "example.com.yml").write_text("authentik:\n    server: example.com\n")
+    return root
+
+
+class TestCheckConfigReadable:
+    def test_valid_config(self, tmp_path):
+        path = tmp_path / "alaka.yml"
+        path.write_text("authentik:\n    version: 1.0.0\n")
+        assert check_config_readable(path) is None
+
+    def test_missing_file_is_not_a_failure(self, tmp_path):
+        """A configuration that simply is not there is a legitimate state."""
+        assert check_config_readable(tmp_path / "absent.yml") is None
+
+    def test_empty_file_is_not_a_failure(self, tmp_path):
+        path = tmp_path / "alaka.yml"
+        path.write_text("")
+        assert check_config_readable(path) is None
+
+    def test_encrypted_file(self, tmp_path):
+        path = tmp_path / "alaka-secrets.yml"
+        path.write_bytes(GIT_CRYPT_HEADER)
+        assert "codec can't decode" in check_config_readable(path)
+
+    def test_malformed_yaml(self, tmp_path):
+        path = tmp_path / "alaka.yml"
+        path.write_text("authentik:\n  - version: 1.0.0\n    broken: [\n")
+        assert check_config_readable(path)
+
+    def test_reason_is_a_single_line(self, tmp_path):
+        """Multi-line parser output would break up the failure report."""
+        path = tmp_path / "alaka.yml"
+        path.write_text("authentik:\n  - version: 1.0.0\n    broken: [\n")
+        assert "\n" not in check_config_readable(path)
+
+
+class TestReadConfigSeverity:
+    """An unreadable file that reaches read_config is out of scope by construction."""
+
+    def test_an_unreadable_config_warns_rather_than_errors(self, config_root, caplog):
+        import logging
+        broken = config_root / "foreign" / "alaka-secrets.yml"
+        broken.parent.mkdir()
+        broken.write_bytes(GIT_CRYPT_HEADER)
+        with caplog.at_level(logging.DEBUG):
+            assert alakazam.read_config(str(broken)) == {}
+        assert [r.levelname for r in caplog.records] == ["WARNING"]
+        assert "is skipped" in caplog.text
+
+
+class TestPreflightConfigs:
+    def test_passes_for_readable_configs(self, config_root):
+        preflight_configs([config_root / "alaka.yml", config_root / "config-sets.yml"])
+
+    def test_aborts_on_an_unreadable_config(self, config_root):
+        broken = config_root / "alaka-secrets.yml"
+        broken.write_bytes(GIT_CRYPT_HEADER)
+        with pytest.raises(click.ClickException) as excinfo:
+            preflight_configs([config_root / "alaka.yml", broken])
+        assert str(broken) in excinfo.value.message
+        assert "git crypt unlock" in excinfo.value.message
+
+    def test_reports_every_failure_at_once(self, config_root):
+        """A CI run should see all broken files, not just the first one."""
+        for name in ("alaka-a.yml", "alaka-b.yml"):
+            (config_root / name).write_bytes(GIT_CRYPT_HEADER)
+        with pytest.raises(click.ClickException) as excinfo:
+            preflight_configs([config_root / "alaka-a.yml", config_root / "alaka-b.yml"])
+        assert "alaka-a.yml" in excinfo.value.message
+        assert "alaka-b.yml" in excinfo.value.message
+        assert excinfo.value.message.startswith("2 ")
+
+    def test_exit_code_is_non_zero(self):
+        assert click.ClickException("").exit_code != 0
+
+
+class TestRelevantConfigPaths:
+    def test_covers_the_inheritance_chain_and_the_instances(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert config_root / "alaka.yml" in paths
+        assert config_root / "config-sets.yml" in paths
+        assert config_root / "group" / "alaka-versions.yml" in paths
+        assert config_root / "group" / "example.com.yml" in paths
+
+    def test_covers_the_connection_configuration(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert alakazam.Path(alakazam.COMBINE_PATH) in paths
+
+    def test_a_single_instance_file_pulls_in_its_ancestors(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group" / "example.com.yml", [])
+        assert config_root / "alaka.yml" in paths
+        assert config_root / "group" / "alaka-versions.yml" in paths
+        assert config_root / "group" / "example.com.yml" in paths
+
+    def test_leaves_out_foreign_groups(self, config_root):
+        """A group that cannot change this run's result stays out of the strict check."""
+        foreign = config_root / "foreign"
+        foreign.mkdir()
+        (foreign / "alaka.yml").write_text("nextcloud:\n    version: 1.0.0\n")
+        (foreign / "other.com.yml").write_text("nextcloud:\n    server: other.com\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert foreign / "alaka.yml" not in paths
+        assert foreign / "other.com.yml" not in paths
+
+    def test_leaves_out_paths_above_the_root(self, config_root):
+        (config_root.parent / "alaka.yml").write_text("authentik:\n    version: 0.0.1\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert config_root.parent / "alaka.yml" not in paths
+
+    def test_honours_exclude_paths(self, config_root):
+        excluded = config_root / "group" / "excluded"
+        excluded.mkdir()
+        (excluded / "skip.com.yml").write_text("authentik:\n    server: skip.com\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [str(excluded)])
+        assert excluded / "skip.com.yml" not in paths
+
+    def test_has_no_duplicates(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert len(paths) == len(set(paths))
+
+
+class TestPreflightInCli:
+    """The preflight has to abort before the first abra call, not halfway through a run."""
+
+    @pytest.fixture
+    def cli_env(self, config_root, monkeypatch):
+        calls = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append(a) or "")
+        monkeypatch.setattr(alakazam, "fetch_recipes", lambda *a, **k: None)
+        monkeypatch.setattr(alakazam, "get_settings_path", lambda: str(config_root / "alakazam.yml"))
+        monkeypatch.setattr(alakazam, "get_abra_dir", lambda: config_root / ".abra")
+        (config_root / "alakazam.yml").write_text(f"root: {config_root}\n")
+        return calls
+
+    def test_clean_configs_run_through(self, config_root, cli_env):
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code == 0, result.output
+
+    def test_encrypted_config_aborts_without_calling_abra(self, config_root, cli_env):
+        (config_root / "group" / "alaka-secrets.yml").write_bytes(GIT_CRYPT_HEADER)
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code != 0
+        assert "alaka-secrets.yml" in result.output
+        assert "git crypt unlock" in result.output
+        assert cli_env == []
+
+    def test_encrypted_config_of_a_foreign_group_does_not_abort(self, config_root, cli_env):
+        foreign = config_root / "foreign"
+        foreign.mkdir()
+        (foreign / "alaka-secrets.yml").write_bytes(GIT_CRYPT_HEADER)
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code == 0, result.output
+
+    def test_broken_settings_file_aborts(self, config_root, cli_env):
+        (config_root / "alakazam.yml").write_bytes(GIT_CRYPT_HEADER)
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code != 0
+        assert "alakazam.yml" in result.output
+
+
+class TestLogLevel:
+    """-l has to win over the handler that an early logging.warning() installs by itself."""
+
+    @pytest.fixture
+    def probe(self, config_root, monkeypatch):
+        import logging
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        monkeypatch.setattr(alakazam, "fetch_recipes", lambda *a, **k: None)
+        monkeypatch.setattr(alakazam, "get_settings_path", lambda: str(config_root / "alakazam.yml"))
+        monkeypatch.setattr(alakazam, "get_abra_dir", lambda: config_root / ".abra")
+        (config_root / "alakazam.yml").write_text(f"root: {config_root}\n")
+        # a missing config-sets.yml makes read_config warn before cli() is through
+        (config_root / "config-sets.yml").unlink()
+        levels = []
+
+        @alakazam.cli.command()
+        def probe():
+            levels.append(logging.getLogger().getEffectiveLevel())
+
+        monkeypatch.setattr(alakazam, "GROUP_PATH", None)
+        return levels
+
+    def run(self, config_root, level):
+        return CliRunner().invoke(alakazam.cli, ["-l", level, str(config_root / "group"), "probe"])
+
+    def test_debug_reaches_the_root_logger(self, config_root, probe):
+        import logging
+        result = self.run(config_root, "DEBUG")
+        assert result.exit_code == 0, result.output
+        assert probe == [logging.DEBUG]
+
+    def test_info_reaches_the_root_logger(self, config_root, probe):
+        import logging
+        self.run(config_root, "INFO")
+        assert probe == [logging.INFO]
+
+    def test_an_invalid_level_is_rejected(self, config_root, probe):
+        result = self.run(config_root, "LOUD")
+        assert result.exit_code != 0

tests/test_ps_wait.py

@@ -0,0 +1,154 @@
+"""Tests for the health gate of 'ps --wait'."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_pending_containers, wait_for_healthy_apps
+
+
+def container(service, status, version="1.0.0"):
+    return {"service": service, "status": status, "version": version, "chaos": "false", "state": "running", "image": "img"}
+
+
+class TestPendingContainers:
+    def test_healthy_containers_are_not_pending(self):
+        containers = [("a.org", container("app", "healthy")), ("a.org", container("db", "running"))]
+        assert get_pending_containers(containers) == []
+
+    @pytest.mark.parametrize("status", ["starting", "unknown", "unhealthy", "starting (health: starting)"])
+    def test_unsettled_statuses_are_pending(self, status):
+        containers = [("a.org", container("app", status))]
+        assert len(get_pending_containers(containers)) == 1
+
+    def test_init_containers_are_exempt(self):
+        """Init containers run once and exit, they never report a healthy status."""
+        containers = [("a.org", container("authentik-init", "unknown")), ("a.org", container("app", "healthy"))]
+        assert get_pending_containers(containers) == []
+
+    def test_an_init_container_does_not_hide_a_real_one(self):
+        containers = [("a.org", container("authentik-init", "unknown")), ("a.org", container("app", "starting"))]
+        pending = get_pending_containers(containers)
+        assert [c["service"] for _, c in pending] == ["app"]
+
+
+class FakeCluster:
+    """Serves a scripted sequence of container states, one per health check."""
+
+    def __init__(self, rounds, deployed=True):
+        self.rounds = list(rounds)
+        self.deployed = deployed
+        self.checks = 0
+        self.slept = 0
+
+    def install(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "get_apps_by_deployment", self.get_apps_by_deployment)
+        monkeypatch.setattr(alakazam, "get_app_containers", self.get_app_containers)
+        monkeypatch.setattr(alakazam, "sleep", self.sleep)
+        return self
+
+    def get_apps_by_deployment(self, recipes, deployed=True):
+        return {"a.org": [["authentik", "login.a.org", "1.0.0"]]} if self.deployed else {}
+
+    def get_app_containers(self, instance_apps):
+        self.checks += 1
+        return self.rounds[min(self.checks - 1, len(self.rounds) - 1)]
+
+    def sleep(self, seconds):
+        self.slept += seconds
+
+
+HEALTHY = [("login.a.org", container("app", "healthy"))]
+STARTING = [("login.a.org", container("app", "starting"))]
+
+
+class TestWaitForHealthyApps:
+    def test_returns_immediately_when_everything_is_healthy(self, monkeypatch, capsys):
+        cluster = FakeCluster([HEALTHY]).install(monkeypatch)
+        wait_for_healthy_apps((), timeout=300)
+        assert cluster.checks == 1
+        assert cluster.slept == 0
+        assert "All services are healthy" in capsys.readouterr().out
+
+    def test_waits_until_the_services_settle(self, monkeypatch):
+        cluster = FakeCluster([STARTING, STARTING, HEALTHY]).install(monkeypatch)
+        wait_for_healthy_apps((), timeout=300)
+        assert cluster.checks == 3
+        assert cluster.slept == 2 * alakazam.PS_WAIT_INTERVAL
+
+    def test_timeout_aborts_and_names_the_services(self, monkeypatch):
+        cluster = FakeCluster([STARTING]).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            wait_for_healthy_apps((), timeout=0)
+        assert "login.a.org app: starting" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+    def test_timeout_checks_at_least_once(self, monkeypatch):
+        """A zero timeout still has to look, otherwise it could not report anything."""
+        cluster = FakeCluster([STARTING]).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            wait_for_healthy_apps((), timeout=0)
+        assert cluster.checks == 1
+
+    def test_nothing_deployed_is_a_failure(self, monkeypatch):
+        """Reporting the health of an environment that does not exist would turn a broken deploy green."""
+        FakeCluster([HEALTHY], deployed=False).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            wait_for_healthy_apps((), timeout=300)
+        assert "nothing to wait for" in excinfo.value.message
+
+    def test_an_init_container_alone_does_not_block(self, monkeypatch):
+        rounds = [[("login.a.org", container("authentik-init", "unknown")), ("login.a.org", container("app", "healthy"))]]
+        cluster = FakeCluster(rounds).install(monkeypatch)
+        wait_for_healthy_apps((), timeout=300)
+        assert cluster.checks == 1
+
+
+class TestPsCommand:
+    """Without --wait the command must behave exactly as it did before the flag existed."""
+
+    @pytest.fixture
+    def cluster(self, monkeypatch):
+        cluster = FakeCluster([STARTING]).install(monkeypatch)
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": {"authentik": {"app_domain": "login.a.org"}}})
+        return cluster
+
+    def invoke(self, args):
+        from click.testing import CliRunner
+        return CliRunner().invoke(alakazam.ps, args, standalone_mode=False)
+
+    def test_plain_ps_checks_once_and_does_not_wait(self, cluster):
+        result = self.invoke([])
+        assert result.exit_code == 0, result.output
+        assert cluster.checks == 1
+        assert cluster.slept == 0
+
+    def test_plain_ps_prints_the_table(self, cluster):
+        result = self.invoke([])
+        assert "Service" in result.output
+        assert "login.a.org" in result.output
+        assert "starting" in result.output
+
+    def test_plain_ps_succeeds_on_unhealthy_services(self, cluster):
+        """Reporting a status is not the same as gating on it."""
+        assert self.invoke([]).exit_code == 0
+
+    def test_plain_ps_without_deployed_apps(self, cluster, monkeypatch):
+        cluster.deployed = False
+        result = self.invoke([])
+        assert result.exit_code == 0
+        assert "No apps deployed" in result.output
+
+    def test_wait_and_watch_are_mutually_exclusive(self, cluster):
+        result = self.invoke(["--wait", "--watch"])
+        assert isinstance(result.exception, click.UsageError)
+
+    def test_wait_gates_on_the_status(self, cluster):
+        result = self.invoke(["--wait", "--timeout", "0"])
+        assert isinstance(result.exception, click.ClickException)
+        assert "still not healthy after 0s" in result.exception.message

tests/test_purge.py

@@ -0,0 +1,92 @@
+"""Tests for the removal of apps and everything they left on their servers."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import purge_app_secrets, purge_apps
+
+INSTANCE_APPS = {"a.org": [["authentik", "login.a.org"]]}
+CONFIG = {"a.org": {"authentik": {"app_domain": "login.a.org", "server": "a.org"}}}
+
+
+@pytest.fixture
+def env_file(tmp_path, monkeypatch):
+    """Places a local app configuration where purge_apps looks for it."""
+    path = tmp_path / "login.a.org.env"
+    path.write_text("")
+    monkeypatch.setattr(alakazam, "get_env_path", lambda server, domain: path)
+    monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+    return path
+
+
+
+class TestPurgeApps:
+    def test_a_failing_removal_aborts(self, env_file, monkeypatch):
+        """Reporting a purge that did not happen would make a later setup build on leftovers."""
+        def failing(*args, **kwargs):
+            raise RuntimeError("FATA login.a.org is still deployed")
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            purge_apps(INSTANCE_APPS)
+        assert "could not purge login.a.org" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+    def test_a_successful_removal_reports_it(self, env_file, monkeypatch, capsys):
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "removed")
+        purge_apps(INSTANCE_APPS)
+        assert "login.a.org purged" in capsys.readouterr().out
+
+    def test_an_app_without_a_local_config_is_skipped(self, tmp_path, monkeypatch):
+        """abra addresses secrets and volumes through the .env, without it there is nothing to remove."""
+        calls = []
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+        monkeypatch.setattr(alakazam, "get_env_path", lambda server, domain: tmp_path / "absent.env")
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append(a) or "")
+        purge_apps(INSTANCE_APPS)
+        assert calls == []
+
+
+class TestPurgeAppSecrets:
+    @pytest.fixture
+    def config(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+
+    def install(self, monkeypatch, error=None):
+        calls = []
+
+        def abra(*args, **kwargs):
+            calls.append(args)
+            if error:
+                raise RuntimeError(error)
+            return ""
+
+        monkeypatch.setattr(alakazam, "abra", abra)
+        return calls
+
+    def test_all_secrets_of_a_recipe_are_removed(self, config, monkeypatch, capsys):
+        calls = self.install(monkeypatch)
+        purge_app_secrets({"authentik": []})
+        assert calls == [("app", "secret", "rm", "-a", "login.a.org")]
+        assert "purged" in capsys.readouterr().out
+
+    def test_an_app_without_secrets_is_not_a_failure(self, config, monkeypatch, capsys):
+        """abra exits non-zero when it found nothing to remove, which says the job is already done."""
+        self.install(monkeypatch, error="FATA no secrets to remove?")
+        purge_app_secrets({"authentik": []})
+        assert "has no secrets on its server" in capsys.readouterr().out
+
+    def test_a_real_failure_still_propagates(self, config, monkeypatch):
+        self.install(monkeypatch, error="FATA error during connect: no route to host")
+        with pytest.raises(RuntimeError):
+            purge_app_secrets({"authentik": []})
+
+    def test_a_named_secret_that_is_absent_is_skipped(self, config, monkeypatch, capsys):
+        self.install(monkeypatch, error="FATA email_pass doesn't exist on server?")
+        purge_app_secrets({"authentik": ["email_pass"]})
+        assert "is not stored on the server" in capsys.readouterr().out

tests/test_readiness_hooks.py

@@ -0,0 +1,160 @@
+"""Tests for the readiness hooks that gate dependent apps on an app becoming usable."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+import logging
+
+from alakazam import abra, build_app_cmd, run_readiness_hook, run_readiness_hooks, run_streamed
+
+
+class FakeAbra:
+    """Fails a scripted number of times before succeeding, recording every call."""
+
+    def __init__(self, failures=0, always_fail=False):
+        self.failures = failures
+        self.always_fail = always_fail
+        self.calls = []
+        self.slept = []
+
+    def install(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "abra", self)
+        monkeypatch.setattr(alakazam, "sleep", self.slept.append)
+        return self
+
+    def __call__(self, *args, **kwargs):
+        self.calls.append(args)
+        if self.always_fail or len(self.calls) <= self.failures:
+            raise RuntimeError("FATA blueprint not applied")
+        return "ok"
+
+
+class TestBuildAppCmd:
+    def test_container_command(self):
+        assert build_app_cmd("login.a.org", "app check_blueprints") == [
+            "app", "cmd", "-T", "", "login.a.org", "app", "--", "check_blueprints",
+        ]
+
+    def test_local_command(self):
+        assert build_app_cmd("login.a.org", "local enable_sso") == [
+            "app", "cmd", "--local", "", "login.a.org", "--", "enable_sso",
+        ]
+
+    def test_arguments_are_passed_through(self):
+        assert build_app_cmd("login.a.org", "app set_flag on")[-2:] == ["set_flag", "on"]
+
+    def test_chaos_flag(self):
+        assert "-C" in build_app_cmd("login.a.org", "app check_blueprints", "-C")
+
+
+class TestRunReadinessHook:
+    def test_succeeds_on_the_first_attempt(self, monkeypatch, capsys):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints"})
+        assert len(abra.calls) == 1
+        assert abra.slept == []
+        assert "is ready" in capsys.readouterr().out
+
+    def test_retries_until_it_succeeds(self, monkeypatch):
+        abra = FakeAbra(failures=2).install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "interval": 5})
+        assert len(abra.calls) == 3
+        assert abra.slept == [5, 5]
+
+    def test_gives_up_after_the_configured_retries(self, monkeypatch):
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "retries": 3, "interval": 5})
+        assert len(abra.calls) == 3
+        assert "failed 3 times" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+    def test_does_not_sleep_after_the_last_attempt(self, monkeypatch):
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "retries": 3, "interval": 5})
+        assert abra.slept == [5, 5]
+
+    def test_initial_delay_precedes_the_first_attempt(self, monkeypatch):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "initial-delay": 60})
+        assert abra.slept == [60]
+
+    def test_defaults_apply_when_only_a_command_is_given(self, monkeypatch):
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints"})
+        assert len(abra.calls) == alakazam.READINESS_RETRIES
+        assert abra.slept == [alakazam.READINESS_INTERVAL] * (alakazam.READINESS_RETRIES - 1)
+
+    def test_a_hook_without_a_command_is_rejected(self, monkeypatch):
+        FakeAbra().install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_readiness_hook("login.a.org", {"retries": 3})
+        assert "no 'cmd'" in excinfo.value.message
+
+    def test_the_last_failure_is_reported(self, monkeypatch):
+        FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "retries": 1})
+        assert "blueprint not applied" in excinfo.value.message
+
+
+class TestVisibility:
+    """A hook can block for half an hour, so silence is indistinguishable from a hang."""
+
+    def test_the_command_output_is_streamed(self):
+        result = run_streamed(["bash", "-c", "echo first; echo second >&2; exit 3"])
+        assert result.returncode == 3
+        assert result.stdout.decode().splitlines() == ["first", "second"]
+
+    def test_the_hook_output_is_not_buffered(self, monkeypatch):
+        """abra is asked to stream, otherwise nothing appears until the command has finished."""
+        streamed = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **kw: streamed.append(kw.get("stream")) or "ok")
+        monkeypatch.setattr(alakazam, "sleep", lambda seconds: None)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints"})
+        assert streamed == [True]
+
+    def test_progress_is_printed(self, monkeypatch, capsys):
+        """Progress is what the run is doing, not a diagnostic, and logging.info needs -l to show."""
+        FakeAbra(failures=1).install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "interval": 5})
+        out = capsys.readouterr().out
+        assert "attempt 1/20" in out
+        assert "retry in 5s" in out
+
+    def test_the_initial_delay_is_announced(self, monkeypatch, capsys):
+        FakeAbra().install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "initial-delay": 60})
+        assert "Wait 60s" in capsys.readouterr().out
+
+    def test_streaming_and_machine_output_are_incompatible(self):
+        """Streaming merges abra's log lines into stdout, which would break the JSON parsing."""
+        with pytest.raises(ValueError):
+            abra("app", "ls", machine_output=True, stream=True)
+
+
+class TestRunReadinessHooks:
+    def test_no_hooks_configured_is_a_no_op(self, monkeypatch):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hooks("login.a.org", {"version": "1.0.0"})
+        assert abra.calls == []
+
+    def test_hooks_run_in_configuration_order(self, monkeypatch):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hooks("login.a.org", {"readiness-hooks": [{"cmd": "app first"}, {"cmd": "app second"}]})
+        assert [call[-1] for call in abra.calls] == ["first", "second"]
+
+    def test_a_failing_hook_stops_the_following_ones(self, monkeypatch):
+        """Dependent apps must not proceed on a half ready app."""
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            run_readiness_hooks("login.a.org", {"readiness-hooks": [{"cmd": "app first", "retries": 1}, {"cmd": "app second"}]})
+        assert all(call[-1] == "first" for call in abra.calls)

tests/test_retry.py

@@ -0,0 +1,103 @@
+"""Tests for which abra failures may be retried, and which configuration keys are ignored."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import abra, is_connection_error, update_configs
+
+
+class FakeProcess:
+    def __init__(self, returncode, stderr=b""):
+        self.returncode = returncode
+        self.stdout = b""
+        self.stderr = stderr
+
+
+UNREACHABLE = b"FATA error during connect: kex_exchange_identification: Connection closed by remote host"
+REJECTED = b"FATA login.a.org is still deployed"
+
+
+class TestRetryLoop:
+    """A server that drops the ssh handshake is usually throttling, so the retries have to spread out."""
+
+    @pytest.fixture
+    def runs(self, monkeypatch):
+        calls = {"results": [], "slept": []}
+        monkeypatch.setattr(alakazam, "sleep", calls["slept"].append)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda cmd, capture_output: calls["results"].pop(0))
+        return calls
+
+    def test_a_successful_command_does_not_retry(self, runs):
+        runs["results"] = [FakeProcess(0)]
+        abra("app", "ls")
+        assert runs["slept"] == []
+
+    def test_the_delay_doubles_between_attempts(self, runs):
+        runs["results"] = [FakeProcess(1, UNREACHABLE)] * alakazam.ABRA_RETRIES
+        with pytest.raises(RuntimeError):
+            abra("app", "ls")
+        expected = [alakazam.ABRA_RETRY_DELAY * 2 ** n for n in range(alakazam.ABRA_RETRIES - 1)]
+        assert runs["slept"] == expected
+
+    def test_it_stops_retrying_once_the_command_succeeds(self, runs):
+        runs["results"] = [FakeProcess(1, UNREACHABLE), FakeProcess(1, UNREACHABLE), FakeProcess(0)]
+        abra("app", "ls")
+        assert runs["slept"] == [alakazam.ABRA_RETRY_DELAY, alakazam.ABRA_RETRY_DELAY * 2]
+
+    def test_a_rejected_operation_is_not_repeated(self, runs):
+        """Repeating a command the server answered would only waste the whole backoff."""
+        runs["results"] = [FakeProcess(1, REJECTED)]
+        with pytest.raises(RuntimeError):
+            abra("app", "ls")
+        assert runs["slept"] == []
+
+
+class TestConnectionErrors:
+    """Only unreachable servers may be retried, a rejected operation must not be repeated."""
+
+    @pytest.mark.parametrize("output", [
+        'error during connect: Get "http://docker.example.com/v1.51/info"',
+        "kex_exchange_identification: Connection closed by remote host",
+        "Connection closed by 152.53.133.16 port 22",
+        "connection reset by peer",
+        "ssh: connect to host example.com port 22: Connection refused",
+        "Connection timed out",
+        "No route to host",
+        "write: broken pipe",
+    ])
+    def test_unreachable_server(self, output):
+        assert is_connection_error(output)
+
+    @pytest.mark.parametrize("output", [
+        "FATA email_pass doesn't exist on server?",
+        "FATA login.a.org is still deployed",
+        "FATA secret is in use",
+        "",
+    ])
+    def test_operational_failures_are_not_retried(self, output):
+        assert not is_connection_error(output)
+
+
+class TestUnknownConfigKeys:
+    def test_an_unknown_key_never_reaches_the_env_file(self, tmp_path):
+        """A promotion mechanism sets 'hold: <reason>' on a recipe, which must pass through untouched."""
+        env = tmp_path / "app.env"
+        env.write_text("#SMTP_HOST=mail.example.com\n")
+        update_configs(env, {"hold": "waiting for the upstream fix", "env": {"SMTP_HOST": "mail.a.org"}})
+        content = env.read_text()
+        assert "SMTP_HOST=mail.a.org" in content
+        assert "hold" not in content
+        assert "waiting for the upstream fix" not in content
+
+    def test_an_unknown_key_alone_changes_nothing(self, tmp_path):
+        env = tmp_path / "app.env"
+        env.write_text("#SMTP_HOST=mail.example.com\n")
+        update_configs(env, {"hold": "waiting for the upstream fix"})
+        assert env.read_text() == "#SMTP_HOST=mail.example.com\n"

tests/test_secrets.py

@@ -0,0 +1,67 @@
+"""Tests for the secret generation, in particular its behaviour on a repeated run."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import generate_all_secrets
+
+# what 'abra app secret generate -a' reports when every secret is already on the server, exiting 1
+NOTHING_LEFT = RuntimeError(
+    "abra -o app secret generate -a login.a.org -m \n STDOUT: \n \n STDERR: "
+    "WARN login_a_org_db_password_v1 already exists\nWARN no secrets generated"
+)
+
+
+class FakeAbra:
+    def __init__(self, stored, generate_error=None):
+        self.stored = stored
+        self.generate_error = generate_error
+        self.generated = False
+
+    def __call__(self, *args, **kwargs):
+        if args[:3] == ("app", "secret", "ls"):
+            return [{"name": name, "created on server": str(created).lower()}
+                    for name, created in self.stored.items()]
+        if args[:3] == ("app", "secret", "generate"):
+            self.generated = True
+            if self.generate_error:
+                raise self.generate_error
+            return [{"name": "db_password", "value": "s3cret"}]
+        raise AssertionError(f"unexpected abra call: {args}")
+
+
+class TestGenerateAllSecrets:
+    def test_nothing_runs_when_every_secret_exists(self, monkeypatch):
+        abra = FakeAbra({"db_password": True, "email_pass": True})
+        monkeypatch.setattr(alakazam, "abra", abra)
+        generate_all_secrets("login.a.org")
+        assert not abra.generated
+
+    def test_missing_secrets_are_generated(self, monkeypatch, capsys):
+        abra = FakeAbra({"db_password": False})
+        monkeypatch.setattr(alakazam, "abra", abra)
+        generate_all_secrets("login.a.org")
+        assert abra.generated
+        assert "db_password: s3cret" in capsys.readouterr().out
+
+    def test_an_empty_generate_run_is_not_a_failure(self, monkeypatch, caplog):
+        """A retry after a half finished run finds every secret in place, abra still exits 1."""
+        abra = FakeAbra({"db_password": False}, generate_error=NOTHING_LEFT)
+        monkeypatch.setattr(alakazam, "abra", abra)
+        with caplog.at_level(logging.WARNING):
+            generate_all_secrets("login.a.org")
+        assert "generated no secrets" in caplog.text
+
+    def test_a_real_failure_still_propagates(self, monkeypatch):
+        """Only the 'nothing to do' case is tolerated, an unreachable server is not."""
+        abra = FakeAbra({"db_password": False},
+                        generate_error=RuntimeError("STDERR: error during connect: no route to host"))
+        monkeypatch.setattr(alakazam, "abra", abra)
+        with pytest.raises(RuntimeError):
+            generate_all_secrets("login.a.org")

tests/test_setup.py

@@ -0,0 +1,256 @@
+"""Tests for 'setup' and 'clean-deploy', the commands that replace the external deploy script."""
+
+import os
+import sys
+from pathlib import Path
+
+import click
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_storage_traces, require_empty_environment, setup_environment
+
+
+class FakeServer:
+    """Answers the abra calls of a setup run and records the order of the steps."""
+
+    def __init__(self, deployed=(), secrets=(), volumes=()):
+        self.deployed = set(deployed)
+        self.secrets = dict(secrets)
+        self.volumes = set(volumes)
+        self.steps = []
+
+    def install(self, monkeypatch, config=None):
+        if config is None:
+            config = {
+                    "authentik": {"app_domain": "login.a.org", "server": "a.org", "dependency": []},
+                "nextcloud": {"app_domain": "cloud.a.org", "server": "a.org", "dependency": ["authentik"],
+                              "readiness-hooks": [{"cmd": "app ready", "retries": 1}]},
+            }
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": config})
+        monkeypatch.setattr(alakazam, "abra", self.abra)
+        monkeypatch.setattr(alakazam, "sleep", lambda seconds: None)
+        monkeypatch.setattr(alakazam, "configure_apps", lambda recipes: self.steps.append("config"))
+        monkeypatch.setattr(alakazam, "create_secrets", self.create_secrets)
+        monkeypatch.setattr(alakazam, "deploy_apps", self.deploy_apps)
+        monkeypatch.setattr(alakazam, "undeploy_apps", self.undeploy_apps)
+        monkeypatch.setattr(alakazam, "purge_apps", self.purge_apps)
+        monkeypatch.setattr(alakazam, "execute_cmds", lambda app_config, **kw: self.steps.append(f"init:{app_config['app_domain']}"))
+        return self
+
+    def create_secrets(self, recipes, variants=None, **kwargs):
+        label = "conf" if variants == {"conf"} else "secrets"
+        self.steps.append(f"{label}:{','.join(recipes) if recipes else 'all'}")
+
+    def deploy_apps(self, instance_apps, **kwargs):
+        for _, apps in instance_apps.items():
+            for app, domain, *_ in apps:
+                self.steps.append(f"deploy:{app}")
+                self.deployed.add(domain)
+
+    def undeploy_apps(self, instance_apps):
+        self.steps.append("undeploy")
+        self.deployed.clear()
+
+    def purge_apps(self, instance_apps):
+        self.steps.append("purge")
+        self.secrets.clear()
+        self.volumes.clear()
+
+    def abra(self, *args, machine_output=False, **kwargs):
+        if args[:3] == ("app", "secret", "ls"):
+            return [{"name": name, "created on server": "true"} for name in self.secrets.get(args[3], [])]
+        if args[:3] == ("app", "volume", "ls"):
+            return "NAME  ON SERVER\ndata  /var/lib\n" if args[4] in self.volumes else ""
+        if args[:2] == ("app", "ls"):
+            apps = [{"appName": d, "status": "deployed", "chaos": "false", "version": "1.0.0"} for d in sorted(self.deployed)]
+            return {"a.org": {"apps": apps}}
+        if args[:2] == ("app", "ps"):
+            self.steps.append(f"ps:{args[-1]}")
+            return {"app": {"service": "app", "status": "healthy", "version": "1.0.0",
+                            "chaos": "false", "state": "running", "image": "img"}}
+        if args[:3] == ("app", "cmd", "-T"):
+            self.steps.append(f"ready:{args[4]}")
+            return "ok"
+        raise AssertionError(f"unexpected abra call: {args}")
+
+
+class TestStorageTraces:
+    def test_an_empty_environment_leaves_no_traces(self, monkeypatch):
+        FakeServer().install(monkeypatch)
+        assert get_storage_traces({"a.org": [["authentik", "login.a.org"]]}) == []
+
+    def test_leftover_secrets_are_found(self, monkeypatch):
+        FakeServer(secrets={"login.a.org": ["email_pass", "admin_pass"]}).install(monkeypatch)
+        traces = get_storage_traces({"a.org": [["authentik", "login.a.org"]]})
+        assert "2 secret(s)" in traces[0]
+        assert "admin_pass, email_pass" in traces[0]
+
+    def test_leftover_volumes_are_found(self, monkeypatch):
+        FakeServer(volumes=["login.a.org"]).install(monkeypatch)
+        traces = get_storage_traces({"a.org": [["authentik", "login.a.org"]]})
+        assert traces == ["login.a.org still has volumes on its server"]
+
+    def test_an_unreadable_volume_list_aborts(self, monkeypatch):
+        """Guessing that a failing read means "no volumes" would deploy on top of existing data."""
+        server = FakeServer().install(monkeypatch)
+        original = server.abra
+
+        def failing(*args, **kwargs):
+            if args[:3] == ("app", "volume", "ls"):
+                raise RuntimeError("abra -o app volume ls -d login.a.org\n STDERR: Error: no such server")
+            return original(*args, **kwargs)
+
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            get_storage_traces({"a.org": [["authentik", "login.a.org"]]})
+        assert "could not read the volumes of login.a.org" in excinfo.value.message
+
+    def test_uncreated_secrets_do_not_count(self, monkeypatch):
+        """A secret the recipe declares but the server does not hold is not a leftover."""
+        server = FakeServer().install(monkeypatch)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: [{"name": "x", "created on server": "false"}]
+                            if a[:3] == ("app", "secret", "ls") else "")
+        assert get_storage_traces({"a.org": [["authentik", "login.a.org"]]}) == []
+
+
+class TestRequireEmptyEnvironment:
+    def test_no_traces_passes(self):
+        require_empty_environment([])
+
+    def test_traces_abort_with_a_hint(self):
+        with pytest.raises(click.ClickException) as excinfo:
+            require_empty_environment(["login.a.org is deployed"])
+        assert "login.a.org is deployed" in excinfo.value.message
+        assert "clean-deploy" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+
+class TestSetupEnvironment:
+    def test_runs_every_phase_per_dependency_level(self, monkeypatch):
+        config = {
+            "authentik": {"app_domain": "login.a.org", "server": "a.org", "dependency": []},
+            "traefik": {"app_domain": "a.org", "server": "a.org", "dependency": []},
+            "nextcloud": {"app_domain": "cloud.a.org", "server": "a.org", "dependency": ["authentik"],
+                          "readiness-hooks": [{"cmd": "app ready", "retries": 1}]},
+            "wekan": {"app_domain": "boards.a.org", "server": "a.org", "dependency": ["authentik"]},
+        }
+        server = FakeServer().install(monkeypatch, config=config)
+        setup_environment(())
+        assert server.steps[:8] == [
+            "config",
+            "conf:all",
+            "secrets:authentik,traefik", "deploy:authentik", "deploy:traefik",
+            "secrets:nextcloud,wekan", "deploy:nextcloud", "deploy:wekan",
+        ]
+
+    def test_apps_of_one_level_are_deployed_before_that_level_is_awaited(self, monkeypatch):
+        """Nothing in a level waits for a sibling, the readiness hooks close the level."""
+        server = FakeServer().install(monkeypatch)
+        setup_environment(())
+        assert server.steps.index("deploy:nextcloud") < server.steps.index("ready:cloud.a.org")
+
+    def test_configured_secrets_are_inserted_before_any_exchange(self, monkeypatch):
+        """Otherwise an exchange generates a random value that shadows the configured one."""
+        server = FakeServer().install(monkeypatch)
+        setup_environment(())
+        assert server.steps[1] == "conf:all"
+        assert all(step != "conf:all" for step in server.steps[2:])
+
+    def test_initial_hooks_run_after_the_health_check(self, monkeypatch):
+        server = FakeServer().install(monkeypatch)
+        setup_environment(())
+        assert server.steps[-2:] == ["init:login.a.org", "init:cloud.a.org"]
+        assert server.steps.index("ps:login.a.org") < server.steps.index("init:login.a.org")
+
+    def test_a_deployed_app_aborts_before_deploying(self, monkeypatch):
+        server = FakeServer(deployed=["login.a.org"]).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "login.a.org is deployed" in excinfo.value.message
+        assert server.steps == ["config"]
+
+    def test_leftover_secrets_abort_the_run(self, monkeypatch):
+        """An undeploy without a purge must not look like an empty environment."""
+        server = FakeServer(secrets={"login.a.org": ["email_pass"]}).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "email_pass" in excinfo.value.message
+        assert server.steps == ["config"]
+
+    def test_leftover_volumes_abort_the_run(self, monkeypatch):
+        server = FakeServer(volumes=["cloud.a.org"]).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            setup_environment(())
+        assert "deploy:authentik" not in server.steps
+
+    def test_nothing_configured_is_a_failure(self, monkeypatch):
+        FakeServer().install(monkeypatch, config={})
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "nothing to set up" in excinfo.value.message
+
+    def test_a_failing_readiness_hook_stops_the_run(self, monkeypatch):
+        server = FakeServer().install(monkeypatch)
+        original = server.abra
+
+        def failing(*args, **kwargs):
+            if args[:3] == ("app", "cmd", "-T"):
+                raise RuntimeError("FATA blueprints missing")
+            return original(*args, **kwargs)
+
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "is not ready" in excinfo.value.message
+        assert "ps:login.a.org" not in server.steps
+
+
+class TestCleanDeployCommand:
+    def invoke(self, args, group_path, monkeypatch):
+        monkeypatch.setattr(alakazam, "GROUP_PATH", Path(group_path))
+        return CliRunner().invoke(alakazam.clean_deploy, args, standalone_mode=False)
+
+    def test_rebuilds_an_instance(self, monkeypatch, tmp_path):
+        server = FakeServer(deployed=["login.a.org"], secrets={"login.a.org": ["email_pass"]}).install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        result = self.invoke(["-n"], instance, monkeypatch)
+        assert result.exit_code == 0, result.exception
+        assert server.steps[:4] == ["config", "undeploy", "purge", "config"]
+
+    def test_the_apps_are_configured_before_they_are_purged(self, monkeypatch, tmp_path):
+        """purge_apps() skips an app without a local .env, so a fresh checkout would purge nothing."""
+        server = FakeServer(deployed=["login.a.org"]).install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        self.invoke(["-n"], instance, monkeypatch)
+        assert server.steps.index("config") < server.steps.index("purge")
+
+    def test_the_apps_are_configured_again_after_the_purge(self, monkeypatch, tmp_path):
+        """'abra app rm' deletes the local .env as well, so setup has to write it a second time."""
+        server = FakeServer(deployed=["login.a.org"]).install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        self.invoke(["-n"], instance, monkeypatch)
+        assert server.steps.count("config") == 2
+        assert server.steps.index("purge") < len(server.steps) - 1 - server.steps[::-1].index("config")
+
+    def test_a_group_directory_is_rejected(self, monkeypatch, tmp_path):
+        server = FakeServer().install(monkeypatch)
+        result = self.invoke(["-n"], tmp_path, monkeypatch)
+        assert isinstance(result.exception, click.UsageError)
+        assert "single instance configuration file" in str(result.exception)
+        assert server.steps == []
+
+    def test_the_confirmation_can_be_declined(self, monkeypatch, tmp_path):
+        server = FakeServer().install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        monkeypatch.setattr(alakazam, "GROUP_PATH", instance)
+        result = CliRunner().invoke(alakazam.clean_deploy, [], input="no\n", standalone_mode=False)
+        assert result.exit_code == 0
+        assert server.steps == []

tests/test_strict_hooks.py

@@ -0,0 +1,74 @@
+"""Tests for the strict hook mode that lets a scratch build fail on a broken hook."""
+
+import os
+import stat
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import execute_cmds, run_secret_hooks
+
+
+def failing_script(tmp_path):
+    """A tiny local script that exits non-zero, for testing 'script' hook failure handling."""
+    script = tmp_path / "fail.sh"
+    script.write_text("#!/bin/sh\nexit 1\n")
+    script.chmod(script.stat().st_mode | stat.S_IXUSR)
+    return script
+
+
+class TestStrictHooks:
+    def test_a_secret_hook_failure_is_tolerated_by_default(self, monkeypatch):
+        """'secrets' has always continued past a failing hook, that must not change."""
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        run_secret_hooks("login.a.org", {"secret_hooks": ["insert_cert"]})
+
+    def test_a_secret_hook_failure_aborts_in_strict_mode(self, monkeypatch):
+        def failing(*args, ignore_error=False, **kwargs):
+            assert not ignore_error
+            assert kwargs.get("stream"), "a hook can run for minutes, its output must not be buffered"
+            raise RuntimeError("FATA no such command")
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_secret_hooks("login.a.org", {"secret_hooks": ["insert_cert"]}, strict=True)
+        assert "secret hook 'insert_cert' failed" in excinfo.value.message
+
+    def test_a_command_failure_is_tolerated_by_default(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": ["app init"]}, initial=True)
+
+    def test_a_command_failure_aborts_in_strict_mode(self, monkeypatch):
+        def failing(*args, ignore_error=False, **kwargs):
+            raise RuntimeError("FATA container not found")
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": ["app init"]}, initial=True, strict=True)
+        assert "command 'app init' failed" in excinfo.value.message
+
+    def test_a_script_secret_hook_failure_is_tolerated_by_default(self, tmp_path):
+        """A failing local script must be tolerated by default too, same as a failing abra.sh hook."""
+        script = failing_script(tmp_path)
+        run_secret_hooks("login.a.org", {"secret_hooks": [f"script {script}"], "server": "a.org"})
+
+    def test_a_script_secret_hook_failure_aborts_in_strict_mode(self, tmp_path):
+        script = failing_script(tmp_path)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_secret_hooks("login.a.org", {"secret_hooks": [f"script {script}"], "server": "a.org"}, strict=True)
+        assert str(script) in excinfo.value.message
+        assert "failed" in excinfo.value.message
+
+    def test_a_script_command_failure_is_tolerated_by_default(self, tmp_path):
+        script = failing_script(tmp_path)
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": [f"script {script}"]}, initial=True)
+
+    def test_a_script_command_failure_aborts_in_strict_mode(self, tmp_path):
+        script = failing_script(tmp_path)
+        with pytest.raises(click.ClickException) as excinfo:
+            execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": [f"script {script}"]},
+                         initial=True, strict=True)
+        assert str(script) in excinfo.value.message
+        assert "failed" in excinfo.value.message

tests/test_versions.py

@@ -0,0 +1,61 @@
+"""Tests for the classification of recipe version strings."""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from alakazam import classify_version, is_hash, is_version
+
+# a coop cloud version is '<recipe-version>+<upstream-tag>'
+COOP_CLOUD_VERSION = "1.2.3+4.5.6-fpm"
+
+
+class TestIsHash:
+    @pytest.mark.parametrize("value", ["deadbeef", "DEADBEEF", "1234", "a" * 64])
+    def test_hashes(self, value):
+        assert is_hash(value)
+
+    @pytest.mark.parametrize("value", ["1.2.3", COOP_CLOUD_VERSION, "chaos", "latest", None, "", "abc", "g" * 8, "a" * 65])
+    def test_non_hashes(self, value):
+        assert not is_hash(value)
+
+
+class TestIsVersion:
+    @pytest.mark.parametrize("value", ["1.2.3", COOP_CLOUD_VERSION, "1234"])
+    def test_versions(self, value):
+        assert is_version(value)
+
+    @pytest.mark.parametrize("value", ["chaos", "latest", None, "", "deadbeef"])
+    def test_non_versions(self, value):
+        """None must not raise, packaging answers it with a TypeError rather than InvalidVersion."""
+        assert not is_version(value)
+
+
+class TestClassifyVersion:
+    @pytest.mark.parametrize(
+        "value,expected",
+        [
+            ("1.2.3", "version"),
+            (COOP_CLOUD_VERSION, "version"),
+            ("chaos", "chaos"),
+            ("latest", "chaos"),
+            (None, "chaos"),
+            ("deadbeef", "hash"),
+            ("", "unknown"),
+            ("no version at all", "unknown"),
+        ],
+    )
+    def test_classification(self, value, expected):
+        assert classify_version(value) == expected
+
+    def test_hash_takes_precedence_over_version(self):
+        """A purely numeric commit hash must not be mistaken for a release."""
+        assert is_version("1234") and is_hash("1234")
+        assert classify_version("1234") == "hash"
+
+    def test_upstream_tag_does_not_change_the_class(self):
+        """packaging treats everything after '+' as build metadata, the full string still classifies."""
+        assert classify_version("1.2.3") == classify_version(COOP_CLOUD_VERSION)