fix(combine): use monitoring-ng 1.x names for the grafana stack

alakazam.py

@@ -916,6 +916,23 @@ def update_configs(path: Path, config: Dict[str, Any]) -> None:
             dotenv.set_key(path, key, value, quote_mode="never")
 
 
+def get_missing_secrets(domain: str) -> List[str]:
+    """
+    Lists the secrets a recipe declares that the server does not hold.
+
+    This is the same condition abra checks before it deploys, where a missing secret ends the run
+    with "secret not generated". Reading it beforehand is what lets a caller fill the gap instead.
+
+    Args:
+        domain (str): The app domain to check
+
+    Returns:
+        list: The names of the missing secrets, empty when the app is complete
+    """
+    stored_secrets = abra("app", "secret", "ls", domain, machine_output=True)
+    return [s['name'] for s in stored_secrets or [] if not str2bool(s['created on server'])]
+
+
 def generate_all_secrets(domain: str) -> None:
     """
     Generates all secrets for the app specified by its domain using the 'abra' command.
@@ -1896,7 +1913,7 @@ def upgrade(recipes: Tuple[str], execute_hooks: bool, dry_run: bool, redeploy: b
             app_details.append(upgrade_version)
             upgrade_apps.append(app_details)
             logging.info(f'upgrade {app}: {domain} from version {deployed_version} to version "{upgrade_version}"')
-            upgrade_cmds.append((app_config, upgrade_cmd, instance))
+            upgrade_cmds.append((app, app_config, upgrade_cmd, instance))
             if version_type == 'version':
                 release_note_cmd = upgrade_cmd.copy()
                 release_note_cmd.insert(1, '-r')
@@ -1911,8 +1928,13 @@ def upgrade(recipes: Tuple[str], execute_hooks: bool, dry_run: bool, redeploy: b
         print(app)
         print(note)
     if not dry_run and noninteractive or input(f"Do you really want to upgrade these apps? Type YES: ") == "YES":
-        for app_config, upgrade_cmd, instance_domain in upgrade_cmds:
+        for app, app_config, upgrade_cmd, instance_domain in upgrade_cmds:
             app_domain = app_config.get('app_domain')
+            # a recipe can bring a new secret along, which abra refuses to deploy without. The app
+            # is running, so nothing else can be missing: it had all of them when it was deployed
+            if missing := get_missing_secrets(app_domain):
+                print(f"{app_domain} is missing {len(missing)} secret(s): {', '.join(sorted(missing))}")
+                create_secrets((app,), instances=(instance_domain,))
             if redeploy:
                 upgrade_cmd.pop(0)
                 upgrade_cmd.insert(0, 'deploy')

combine.yml

@@ -124,6 +124,7 @@ kimai:
             - SSO_PROVIDER_URL
             - SSO_SAML_URL
             - SSO_LOGOUT_URL
+            - SSO_ADMIN_GROUP_NAME
         secret_hooks:
             - insert_authentik_certificate
         dependency: [authentik]
@@ -317,10 +318,10 @@ monitoring-ng:
             - LOKI_RETENTION_PERIOD
             - LOKI_STORAGE_FILESYSTEM
             - compose.grafana.yml
-            - SECRET_GF_ADMINPASSWD_VERSION
-            - compose.grafana-oidc.yml
             - OIDC_ENABLED
-            - SECRET_GF_OIDC_SECRET_VERSION
+            - SECRET_GRAFANA_ADMIN_PASSWORD_VERSION
+            - SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION
+            - SECRET_GRAFANA_SMTP_PASSWORD_VERSION
         shared_secrets:
-            monitoring_secret: gf_oidc_secret
+            monitoring_secret: grafana_oidc_client_secret
         dependency: [authentik]

tests/test_upgrade_secrets.py

@@ -0,0 +1,85 @@
+"""Tests for filling in secrets a recipe added, before the upgrade that would fail on them."""
+
+import os
+import sys
+
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_missing_secrets
+
+CONFIG = {"a.org": {"nextcloud": {"app_domain": "cloud.a.org", "server": "a.org", "version": "2.0.0"}}}
+
+
+class TestGetMissingSecrets:
+    def install(self, monkeypatch, stored):
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: stored)
+
+    def test_a_complete_app_reports_nothing(self, monkeypatch):
+        self.install(monkeypatch, [{"name": "db_password", "created on server": "true"}])
+        assert get_missing_secrets("cloud.a.org") == []
+
+    def test_the_missing_names_are_returned(self, monkeypatch):
+        self.install(monkeypatch, [
+            {"name": "db_password", "created on server": "true"},
+            {"name": "oidc_secret", "created on server": "false"},
+        ])
+        assert get_missing_secrets("cloud.a.org") == ["oidc_secret"]
+
+    def test_an_app_without_any_secrets(self, monkeypatch):
+        """abra answers with an empty document rather than a list when nothing is stored."""
+        self.install(monkeypatch, {})
+        assert get_missing_secrets("cloud.a.org") == []
+
+
+class TestUpgradeFillsThemIn:
+    @pytest.fixture
+    def steps(self, monkeypatch):
+        steps = []
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+        monkeypatch.setattr(alakazam, "sleep", lambda s: None)
+        monkeypatch.setattr(alakazam, "get_apps_by_deployment",
+                            lambda recipes, deployed=True: {"a.org": [["nextcloud", "cloud.a.org", "1.0.0"]]})
+        monkeypatch.setattr(alakazam, "create_secrets",
+                            lambda recipes, **kw: steps.append(f"secrets:{recipes[0]}"))
+        return steps
+
+    def install_abra(self, monkeypatch, steps, missing):
+        def abra(*args, **kwargs):
+            if args[:3] == ("app", "secret", "ls"):
+                return [{"name": n, "created on server": str(n not in missing).lower()}
+                        for n in ("db_password", "oidc_secret")]
+            steps.append(" ".join(a for a in args if a))
+            return ""
+        monkeypatch.setattr(alakazam, "abra", abra)
+
+    def invoke(self):
+        return CliRunner().invoke(alakazam.upgrade, ["-n"], standalone_mode=False)
+
+    def test_a_missing_secret_is_created_before_the_upgrade(self, monkeypatch, steps):
+        self.install_abra(monkeypatch, steps, missing={"oidc_secret"})
+        result = self.invoke()
+        assert result.exit_code == 0, result.exception
+        # the planning phase fetches release notes with the same command plus -r, skip that one
+        upgrade = next(i for i, s in enumerate(steps)
+                       if s.startswith("app upgrade") and " -r " not in s)
+        assert steps.index("secrets:nextcloud") < upgrade
+
+    def test_the_missing_name_is_reported(self, monkeypatch, steps):
+        """Silently repairing secrets would hide which one the recipe added."""
+        self.install_abra(monkeypatch, steps, missing={"oidc_secret"})
+        assert "missing 1 secret(s): oidc_secret" in self.invoke().output
+
+    def test_a_complete_app_does_not_run_the_secrets(self, monkeypatch, steps):
+        self.install_abra(monkeypatch, steps, missing=set())
+        self.invoke()
+        assert not any(s.startswith("secrets:") for s in steps)
+
+    def test_a_dry_run_creates_nothing(self, monkeypatch, steps):
+        """The check sits behind the confirmation, so --dry-run must not reach it."""
+        self.install_abra(monkeypatch, steps, missing={"oidc_secret"})
+        CliRunner().invoke(alakazam.upgrade, ["-n", "--dry-run"], standalone_mode=False)
+        assert not any(s.startswith("secrets:") for s in steps)