change config structure

README.md

@@ -2,6 +2,15 @@
 
 Proof-of-concept meta-configuration app-connector abra wrapper.
 
+## Problem Statement
+
+- managing a lot of env files can be error prone
+    - copy pasting env files
+- loosing the overview of configuration differences between instances
+- updating env files is a manual process of comparing the env files
+- no way to define default configurations that should always be applied
+- connecting two apps requires manual effort of specific env file changes and sharing secrets
+
 ## Advantages
 
 - Have a global configuration that applies on every instance:
@@ -26,21 +35,19 @@ Proof-of-concept meta-configuration app-connector abra wrapper.
 This concept contains the following configuration files.
 Each configuration can be templated with jinja2 and global variables.
 
-1. `defaults.yml`
+1. `~/.abra/defaults.yml`
     - It contains global configurations for each app that are applied on each instance
-    - i.e. smtp config, language...
+        - i.e. smtp config, language...
-    - This configuration is for the operator to avoid copy pasta errors and reduce manual configurations
-2. `domains.yml`
     - It contains a mapping for each app to a subdomain
-    - i.e. `nextcloud: cloud.example.com`
+        - i.e. `cloud.example.com` for each nextcloud instance
-    - This configuration is for the operator to keep the same naming convention
+    - This configuration is for the operator to avoid copy pasta errors, reduce manual configurations and keep the same naming convention
-3. `config.yml`
+2. `./config.yml`
     - a minimalist configuration per instance
     - contains at least:
         - the domain (and the server, if they differ)
         - the apps to be installed
-    - can optionally contain instance specific configurations for each app
+    - can optionally contain instance specific configurations for each app, overwriting the `defaults.yml` configurations
-4. `combine.yml`
+3. `combine.yml`
     - contains the configuration steps that are required to combine apps with each other
     - This configuration should not be touched by the operator
         - It should be either split into the recipes repositories and maintained by the recipe maintainer or it should be maintained by the alakazam maintainer
@@ -60,6 +67,10 @@ For each app the following `<app_configurations>` steps can be used:
 - `secrets:`
     - insert a specific value into a secret
     - for secrets that can not be generated, i.e. smtp passwords
+- `subdomain`:
+    - contains the subdomain that should be used for a specific app
+    - i.e. `cloud.example.com` for nextcloud
+    - (not available in `combine.yml`)
 
 The `combine.yml` configuration additionally contains
 
@@ -92,13 +103,7 @@ nextcloud:
         <app_configurations>
     ```
 
-2. `domains.yml`
+2. `config.yml`
-
-    ```
-    <app_recipe>: <subdomain>.example.com
-    ```
-
-3. `config.yml`
 
     ```
     domain: <instance_domain>
@@ -108,7 +113,7 @@ nextcloud:
             <app_configurations>
     ```
 
-4. `combine.yml`
+3. `combine.yml`
     - To combine two deployed apps each `<target_app_configurations>` of a specific `<target_app_recipe>` is only applied if the `config.yml` also contains the belonging `<source_app_recipe>`
 
     ```
@@ -131,7 +136,7 @@ Create a new instance:
     3. for each app the `config.yml` configs are applied on the env files and secrets are inserted
     4. for each app the `combine.yml` configs are applied on the env files and secrets are inserted
         - the shared_secrets are generated
-        - `<source_app_recipe>.example.com` is replaced according to `domains.yml` in the `<target_app_recipe>` env file
+        - `<source_app_recipe>.example.com` is replaced in the `<target_app_recipe>` env file according to `subdomain` configuration
     5. all the remaining secrets are generated
 4. `./alakazam.py deploy-apps`
     - each app is deployed

alakazam.py

@@ -1,5 +1,6 @@
 #!/bin/python3
 
+import os
 import json
 import logging
 from pathlib import Path
@@ -10,17 +11,28 @@ import dotenv
 from jinja2 import Environment, FileSystemLoader
 import yaml
 
-def read_config(filename):
+
-    with open(filename) as file:
+def read_config(filepath):
-        DEFAULTS = yaml.safe_load(file)
+    filepath = Path(filepath).expanduser()
-    jinja = Environment(loader = FileSystemLoader('.'), trim_blocks=True, lstrip_blocks=True)
+    if not filepath.exists():
-    template = jinja.get_template(filename, globals=DEFAULTS.get('GLOBALS'))
+        logging.warning(f"config file {filepath} does not exist")
+        return {}
+    with open(filepath) as file:
+        yaml_config = yaml.safe_load(file)
+        if not yaml_config:
+            logging.warning(f"config file {filepath} is empty")
+            return {}
+        globals = yaml_config.get('GLOBALS')
+    jinja = Environment(loader=FileSystemLoader(
+        ['/', '.']), trim_blocks=True, lstrip_blocks=True)
+    template = jinja.get_template(filepath.as_posix(), globals=globals)
     return yaml.safe_load(template.render())
 
-CONFIG = read_config("config.yml")
+
-DOMAINS = read_config("domains.yml")
+INSTANCE = read_config("config.yml")
-DEFAULTS = read_config("defaults.yml")
+DEFAULTS = read_config("~/.abra/defaults.yml")
-COMBINE = read_config("combine.yml")
+COMBINE = read_config(os.path.dirname(
+    os.path.realpath(__file__)) + "/combine.yml")
 
 
 def abra(*args, machine_output=False, ignore_error=False):
@@ -32,7 +44,8 @@ def abra(*args, machine_output=False, ignore_error=False):
     if process.stderr:
         logging.debug(process.stderr.decode())
     if process.returncode and not ignore_error:
-        raise RuntimeError(f'STDOUT: \n {process.stdout.decode()} \n STDERR: {process.stderr.decode()}')
+        raise RuntimeError(
+            f'{" ".join(command)} \n STDOUT: \n {process.stdout.decode()} \n STDERR: {process.stderr.decode()}')
     if process.stderr:
         logging.error(process.stderr.decode())
     if machine_output:
@@ -54,27 +67,30 @@ def write_env_header(path):
 
 
 def new_app(recipe):
-    domain = map_domain(recipe)
+    domain = map_subdomain(recipe)
+    server = INSTANCE["server"] if INSTANCE.get(
+        "server") else INSTANCE["domain"]
+    print(f'create {recipe} config on {server} at {domain}')
     path = get_env_path(recipe)
-    out = abra("app", "new", recipe, "-n", "-s",
+    out = abra("app", "new", recipe, "-n", "-s", server, "-D", domain)
-               CONFIG["server"], "-D", domain)
     if not "app has been created" in out:
         raise RuntimeError(f'App "{recipe}" creation failed')
     else:
         write_env_header(path)
-        print(f'{recipe} created on {CONFIG["server"]} at {domain}')
+        logging.info(f'{recipe} created on {server} at {domain}')
 
 
-def map_domain(recipe):
+def map_subdomain(recipe):
-    if subdomain := DOMAINS.get(recipe):
+    if ((INSTANCE.get('apps') and (app := INSTANCE['apps'].get(recipe)) and (subdomain := app.get('subdomain'))) or
-        domain = subdomain.replace("example.com", CONFIG["domain"])
+            ((app := DEFAULTS.get(recipe)) and (subdomain := app.get('subdomain')))):
+        domain = subdomain.replace("example.com", INSTANCE["domain"])
     else:
-        domain = f"{recipe}.{CONFIG['domain']}"
+        domain = f"{recipe}.{INSTANCE['domain']}"
     return domain
 
 
-def generate_secrets(app):
+def generate_secrets(recipe):
-    domain = map_domain(app)
+    domain = map_subdomain(recipe)
     stored_secrets = abra("app", "secret", "ls", domain).splitlines()
     if any("false" in line for line in stored_secrets):
         abra("app", "secret", "generate", "-a", domain)
@@ -82,14 +98,16 @@ def generate_secrets(app):
 
 
 def get_env_path(recipe):
-    domain = map_domain(recipe)
+    domain = map_subdomain(recipe)
-    return Path(f"~/.abra/servers/{CONFIG['server']}/{domain}.env").expanduser()
+    server = INSTANCE["server"] if INSTANCE.get(
+        "server") else INSTANCE["domain"]
+    return Path(f"~/.abra/servers/{server}/{domain}.env").expanduser()
 
 
 def connect_apps(target_app):
     path = get_env_path(target_app)
     target_conf = COMBINE.get(target_app)
-    for source_app in CONFIG["apps"]:
+    for source_app in INSTANCE["apps"]:
         if target_conf and (configs := target_conf.get(source_app)):
             logging.info(f'connect {target_app} with {source_app}')
             set_configs(target_app, configs)
@@ -110,14 +128,14 @@ def set_configs(recipe, configs):
             logging.debug(f'set {key}={value} in {path}')
             dotenv.set_key(path, key, value, quote_mode="never")
     if secrets := configs.get("secrets"):
-        domain = map_domain(recipe)
+        domain = map_subdomain(recipe)
         for secret_name, secret in secrets.items():
             insert_secret(domain, secret_name, secret)
 
 
 def share_secrets(target, source, secrets):
-    target_domain = map_domain(target)
+    target_domain = map_subdomain(target)
-    source_domain = map_domain(source)
+    source_domain = map_subdomain(source)
     stored_secrets = abra("app", "secret", "ls", target_domain).splitlines()
     for source_secret in secrets:
         target_secret = secrets[source_secret]
@@ -162,7 +180,7 @@ def uncomment(keys, path, match_all=False):
 
 
 def insert_domains(path, app):
-    domain = map_domain(app)
+    domain = map_subdomain(app)
     logging.debug(f'replace all {app}.example.com with {domain} in {path}')
     with open(path, "r") as file:
         content = file.read()
@@ -172,15 +190,15 @@ def insert_domains(path, app):
 
 
 def execute_cmds(app):
-    domain = map_domain(app)
+    domain = map_subdomain(app)
     all_cmds = []
     if (app_conf := DEFAULTS.get(app)) and (cmds := app_conf.get("execute")):
         all_cmds += cmds
     if COMBINE.get(app):
         for target_app, target_conf in COMBINE[app].items():
-            if target_app in CONFIG["apps"] and (cmds := target_conf.get("execute")):
+            if target_app in INSTANCE["apps"] and (cmds := target_conf.get("execute")):
                 all_cmds += cmds
-    if (app_conf := CONFIG["apps"][app]) and (cmds := app_conf.get("execute")):
+    if (app_conf := INSTANCE["apps"][app]) and (cmds := app_conf.get("execute")):
         all_cmds += cmds
     for cmd in all_cmds:
         container = cmd.split()[0]
@@ -190,14 +208,19 @@ def execute_cmds(app):
         else:
             print(abra("app", "cmd", domain, container, cmd, ignore_error=True))
 
+
 @click.group()
-@click.option('-l','--log','loglevel')
+@click.option('-l', '--log', 'loglevel')
-def cli(loglevel):
+@click.option('-c', '--config', 'config')
+def cli(loglevel, config):
+    global INSTANCE
     if loglevel:
         numeric_level = getattr(logging, loglevel.upper(), None)
         if not isinstance(numeric_level, int):
             raise ValueError('Invalid log level: %s' % loglevel)
         logging.basicConfig(level=numeric_level)
+    if config:
+        INSTANCE = read_config(config)
 
 
 @cli.command()
@@ -212,38 +235,39 @@ def init_server():
 def setup_apps(apps):
     """ Configure and connect the apps """
     if not apps:
-        apps = CONFIG["apps"]
+        apps = INSTANCE["apps"]
     for app in apps:
         new_app(app)
         if configs := DEFAULTS.get(app):
             logging.info(f'set defaults for {app}')
             set_configs(app, configs)
-        if configs := CONFIG["apps"].get(app):
+        if configs := INSTANCE["apps"].get(app):
             logging.info(f'set extra configs for {app}')
             set_configs(app, configs)
     for app in apps:
+        print(f'create connections for {app}')
         connect_apps(app)
     for app in apps:
         logging.info(f'generate secrets for {app}')
         generate_secrets(app)
 
+
 def get_deployed_apps():
-    deployed = abra(
+    server = INSTANCE["server"] if INSTANCE.get("server") else INSTANCE["domain"]
-        "app", "ls", "-S", "-s", CONFIG["server"], "-m", machine_output=True
+    deployed = abra("app", "ls", "-S", "-s", server, "-m", machine_output=True)
-    )
     deployed_apps = filter(
-        lambda x: x["status"] == "deployed", deployed[CONFIG["server"]]["apps"]
+        lambda x: x["status"] == "deployed", deployed[server]["apps"])
-    )
     deployed_domains = [app["domain"] for app in deployed_apps]
     return deployed_domains
 
+
 @cli.command()
 @click.option('-a', '--apps', multiple=True)
 def deploy_apps(apps):
     """ Deploy all the apps """
     deployed_domains = get_deployed_apps()
-    for app in CONFIG["apps"]:
+    for app in INSTANCE["apps"]:
-        domain = map_domain(app)
+        domain = map_subdomain(app)
         if (apps and app in apps) or domain not in deployed_domains:
             logging.info(f'deploy {domain}')
             print(abra("app", "deploy", "-C", "-n", domain))
@@ -257,40 +281,43 @@ def undeploy_apps(apps):
     """ Undeploy all the apps """
     deployed_domains = get_deployed_apps()
     if not apps:
-        apps = CONFIG["apps"]
+        apps = INSTANCE["apps"]
     for app in apps:
-        domain = map_domain(app)
+        domain = map_subdomain(app)
         logging.info(f'undeploy {domain}')
         if domain in deployed_domains:
             print(abra("app", "undeploy", "-n", domain))
 
+
 @cli.command()
 @click.option('-a', '--apps', multiple=True)
 def cmds(apps):
     """ execute all post deploy cmds """
     deployed_domains = get_deployed_apps()
     if not apps:
-        apps = CONFIG["apps"]
+        apps = INSTANCE["apps"]
     for app in apps:
-        domain = map_domain(app)
+        domain = map_subdomain(app)
         if domain in deployed_domains:
             logging.info(f'execute commands for {domain}')
             execute_cmds(app)
 
+
 @cli.command()
 def purge_apps():
     """ Completely remove all the apps """
-    for recipe in CONFIG["apps"]:
+    for recipe in INSTANCE["apps"]:
-        domain = map_domain(recipe)
+        domain = map_subdomain(recipe)
         logging.info(f'purge {domain}')
         abra("app", "rm", "-n", domain)
         print(f"{domain} purged")
 
+
 @cli.command()
 def purge_secrets():
     """ Remove all the apps secrets """
-    for recipe in CONFIG["apps"]:
+    for recipe in INSTANCE["apps"]:
-        domain = map_domain(recipe)
+        domain = map_subdomain(recipe)
         logging.info(f'purge secrets from {domain}')
         stored_secrets = abra("app", "secret", "ls", domain)
         if "true" in stored_secrets:

combine.yml

@@ -65,7 +65,6 @@ nextcloud:
         uncomment:
             - ONLYOFFICE_URL
             - SECRET_ONLYOFFICE_JWT_VERSION
-            - SECRET_BBB_SECRET_VERSION
         execute:
             - app install_onlyoffice
 onlyoffice:
@@ -110,6 +109,7 @@ matrix-synapse:
             KEYCLOAK_ID: authentik
             KEYCLOAK_NAME: sso
             KEYCLOAK_URL: https://authentik.example.com/application/o/matrix/
+            # TODO: correct client domain?
             KEYCLOAK_CLIENT_DOMAIN: https://element-web.example.com
             KEYCLOAK_ALLOW_EXISTING_USERS: true
             # TODO: set CLIENT_ID as secret

domains.yml

@@ -1,6 +0,0 @@
-authentik: example.com
-nextcloud: cloud.example.com
-wekan: board.example.com
-element-web: chat.example.com
-backup-bot-two: backup.example.com
-matrix-synapse: matrix.example.com

examples/config.yml

@@ -8,7 +8,9 @@ apps:
     nextcloud:
     onlyoffice:
     wordpress:
+        subdomain: "blog.special-domain.com"
     vikunja:
+        subdomain: "todo.example.com"
     matrix-synapse:
     element-web:
         env:

examples/defaults.yml

@@ -4,6 +4,7 @@ GLOBALS:
     smtp_domain: example.com
     smtp_host: mail.example.com
 authentik:
+    subdomain: example.com
     env:
         AUTHENTIK_EMAIL__HOST: "{{smtp_host}}"
         AUTHENTIK_EMAIL__USERNAME: "{{smtp_user}}@{{smtp_domain}}"
@@ -18,6 +19,7 @@ authentik:
     secrets:
         email_pass: "{{smtp_password}}"
 nextcloud:
+    subdomain: cloud.example.com
     env: 
         SMTP_AUTHTYPE: LOGIN
         SMTP_HOST: "{{smtp_host}}"
@@ -27,10 +29,12 @@ nextcloud:
         MAIL_FROM_ADDRESS: "{{smtp_user}}"
         MAIL_DOMAIN: "{{smtp_domain}}"
         APPS: "calendar"
+        BBB_URL: "https://talk.example.com/bigbluebutton/"
     uncomment:
         - compose.smtp.yml
         - SECRET_SMTP_PASSWORD_VERSION
         - compose.apps.yml
+        - SECRET_BBB_SECRET_VERSION
     secrets:
         smtp_password: "{{smtp_password}}"
     execute:
@@ -66,12 +70,14 @@ vikunja:
     secrets:
         smtp_password: "{{smtp_password}}"
 matrix-synapse:
+    subdomain: matrix.example.com
     env: 
         SMTP_APP_NAME: mail
         SMTP_FROM: "{{smtp_user}}@{{smtp_domain}}"
         SMTP_HOST: "{{smtp_host}}"
         SMTP_PORT: 587
         SMTP_USER: "{{smtp_user}}@{{smtp_domain}}"
+        ENCRYPTED_BY_DEFAULT: "off"
     uncomment:
         - POST_DEPLOY_CMDS
         - compose.smtp.yml
@@ -79,10 +85,15 @@ matrix-synapse:
         - SECRET_SMTP_PASSWORD_VERSION
     secrets:
         smtp_password: "{{smtp_password}}"
+element-web:
+    subdomain: chat.example.com
 wekan:
+    subdomain: board.example.com
     env:
         # TODO fix format for smtp_password
         MAIL_URL: "smtp://{{smtp_user}}%40{{smtp_domain}}%3A{{smtp_password}}@{{smtp_host}}:587/"
         MAIL_FROM: "Wekan Notifications <{{smtp_user}}@{{smtp_domain}}>"
     uncomment:
         - PASSWORD_LOGIN_ENABLED
+backup-bot-two:
+    subdomain: backup.example.com