From 21e983a08627825046aeea30167a37f2719a2994 Mon Sep 17 00:00:00 2001 From: Moritz Date: Tue, 25 Jul 2023 16:57:53 +0200 Subject: [PATCH] change config structure --- README.md | 41 +++++---- alakazam.py | 125 ++++++++++++++++---------- combine.yml | 2 +- domains.yml | 6 -- config.yml => examples/config.yml | 2 + defaults.yml => examples/defaults.yml | 11 +++ 6 files changed, 113 insertions(+), 74 deletions(-) delete mode 100644 domains.yml rename config.yml => examples/config.yml (79%) rename defaults.yml => examples/defaults.yml (88%) diff --git a/README.md b/README.md index 59dce43..ab9558d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,15 @@ Proof-of-concept meta-configuration app-connector abra wrapper. +## Problem Statement + +- managing a lot of env files can be error prone + - copy pasting env files +- loosing the overview of configuration differences between instances +- updating env files is a manual process of comparing the env files +- no way to define default configurations that should always be applied +- connecting two apps requires manual effort of specific env file changes and sharing secrets + ## Advantages - Have a global configuration that applies on every instance: @@ -26,21 +35,19 @@ Proof-of-concept meta-configuration app-connector abra wrapper. This concept contains the following configuration files. Each configuration can be templated with jinja2 and global variables. -1. `defaults.yml` +1. `~/.abra/defaults.yml` - It contains global configurations for each app that are applied on each instance - - i.e. smtp config, language... - - This configuration is for the operator to avoid copy pasta errors and reduce manual configurations -2. `domains.yml` + - i.e. smtp config, language... - It contains a mapping for each app to a subdomain - - i.e. `nextcloud: cloud.example.com` - - This configuration is for the operator to keep the same naming convention -3. `config.yml` + - i.e. `cloud.example.com` for each nextcloud instance + - This configuration is for the operator to avoid copy pasta errors, reduce manual configurations and keep the same naming convention +2. `./config.yml` - a minimalist configuration per instance - contains at least: - the domain (and the server, if they differ) - the apps to be installed - - can optionally contain instance specific configurations for each app -4. `combine.yml` + - can optionally contain instance specific configurations for each app, overwriting the `defaults.yml` configurations +3. `combine.yml` - contains the configuration steps that are required to combine apps with each other - This configuration should not be touched by the operator - It should be either split into the recipes repositories and maintained by the recipe maintainer or it should be maintained by the alakazam maintainer @@ -60,6 +67,10 @@ For each app the following `` steps can be used: - `secrets:` - insert a specific value into a secret - for secrets that can not be generated, i.e. smtp passwords +- `subdomain`: + - contains the subdomain that should be used for a specific app + - i.e. `cloud.example.com` for nextcloud + - (not available in `combine.yml`) The `combine.yml` configuration additionally contains @@ -92,13 +103,7 @@ nextcloud: ``` -2. `domains.yml` - - ``` - : .example.com - ``` - -3. `config.yml` +2. `config.yml` ``` domain: @@ -108,7 +113,7 @@ nextcloud: ``` -4. `combine.yml` +3. `combine.yml` - To combine two deployed apps each `` of a specific `` is only applied if the `config.yml` also contains the belonging `` ``` @@ -131,7 +136,7 @@ Create a new instance: 3. for each app the `config.yml` configs are applied on the env files and secrets are inserted 4. for each app the `combine.yml` configs are applied on the env files and secrets are inserted - the shared_secrets are generated - - `.example.com` is replaced according to `domains.yml` in the `` env file + - `.example.com` is replaced in the `` env file according to `subdomain` configuration 5. all the remaining secrets are generated 4. `./alakazam.py deploy-apps` - each app is deployed diff --git a/alakazam.py b/alakazam.py index 504bb94..59d3e6e 100755 --- a/alakazam.py +++ b/alakazam.py @@ -1,5 +1,6 @@ #!/bin/python3 +import os import json import logging from pathlib import Path @@ -10,17 +11,28 @@ import dotenv from jinja2 import Environment, FileSystemLoader import yaml -def read_config(filename): - with open(filename) as file: - DEFAULTS = yaml.safe_load(file) - jinja = Environment(loader = FileSystemLoader('.'), trim_blocks=True, lstrip_blocks=True) - template = jinja.get_template(filename, globals=DEFAULTS.get('GLOBALS')) + +def read_config(filepath): + filepath = Path(filepath).expanduser() + if not filepath.exists(): + logging.warning(f"config file {filepath} does not exist") + return {} + with open(filepath) as file: + yaml_config = yaml.safe_load(file) + if not yaml_config: + logging.warning(f"config file {filepath} is empty") + return {} + globals = yaml_config.get('GLOBALS') + jinja = Environment(loader=FileSystemLoader( + ['/', '.']), trim_blocks=True, lstrip_blocks=True) + template = jinja.get_template(filepath.as_posix(), globals=globals) return yaml.safe_load(template.render()) -CONFIG = read_config("config.yml") -DOMAINS = read_config("domains.yml") -DEFAULTS = read_config("defaults.yml") -COMBINE = read_config("combine.yml") + +INSTANCE = read_config("config.yml") +DEFAULTS = read_config("~/.abra/defaults.yml") +COMBINE = read_config(os.path.dirname( + os.path.realpath(__file__)) + "/combine.yml") def abra(*args, machine_output=False, ignore_error=False): @@ -32,7 +44,8 @@ def abra(*args, machine_output=False, ignore_error=False): if process.stderr: logging.debug(process.stderr.decode()) if process.returncode and not ignore_error: - raise RuntimeError(f'STDOUT: \n {process.stdout.decode()} \n STDERR: {process.stderr.decode()}') + raise RuntimeError( + f'{" ".join(command)} \n STDOUT: \n {process.stdout.decode()} \n STDERR: {process.stderr.decode()}') if process.stderr: logging.error(process.stderr.decode()) if machine_output: @@ -54,27 +67,30 @@ def write_env_header(path): def new_app(recipe): - domain = map_domain(recipe) + domain = map_subdomain(recipe) + server = INSTANCE["server"] if INSTANCE.get( + "server") else INSTANCE["domain"] + print(f'create {recipe} config on {server} at {domain}') path = get_env_path(recipe) - out = abra("app", "new", recipe, "-n", "-s", - CONFIG["server"], "-D", domain) + out = abra("app", "new", recipe, "-n", "-s", server, "-D", domain) if not "app has been created" in out: raise RuntimeError(f'App "{recipe}" creation failed') else: write_env_header(path) - print(f'{recipe} created on {CONFIG["server"]} at {domain}') + logging.info(f'{recipe} created on {server} at {domain}') -def map_domain(recipe): - if subdomain := DOMAINS.get(recipe): - domain = subdomain.replace("example.com", CONFIG["domain"]) +def map_subdomain(recipe): + if ((INSTANCE.get('apps') and (app := INSTANCE['apps'].get(recipe)) and (subdomain := app.get('subdomain'))) or + ((app := DEFAULTS.get(recipe)) and (subdomain := app.get('subdomain')))): + domain = subdomain.replace("example.com", INSTANCE["domain"]) else: - domain = f"{recipe}.{CONFIG['domain']}" + domain = f"{recipe}.{INSTANCE['domain']}" return domain -def generate_secrets(app): - domain = map_domain(app) +def generate_secrets(recipe): + domain = map_subdomain(recipe) stored_secrets = abra("app", "secret", "ls", domain).splitlines() if any("false" in line for line in stored_secrets): abra("app", "secret", "generate", "-a", domain) @@ -82,14 +98,16 @@ def generate_secrets(app): def get_env_path(recipe): - domain = map_domain(recipe) - return Path(f"~/.abra/servers/{CONFIG['server']}/{domain}.env").expanduser() + domain = map_subdomain(recipe) + server = INSTANCE["server"] if INSTANCE.get( + "server") else INSTANCE["domain"] + return Path(f"~/.abra/servers/{server}/{domain}.env").expanduser() def connect_apps(target_app): path = get_env_path(target_app) target_conf = COMBINE.get(target_app) - for source_app in CONFIG["apps"]: + for source_app in INSTANCE["apps"]: if target_conf and (configs := target_conf.get(source_app)): logging.info(f'connect {target_app} with {source_app}') set_configs(target_app, configs) @@ -110,14 +128,14 @@ def set_configs(recipe, configs): logging.debug(f'set {key}={value} in {path}') dotenv.set_key(path, key, value, quote_mode="never") if secrets := configs.get("secrets"): - domain = map_domain(recipe) + domain = map_subdomain(recipe) for secret_name, secret in secrets.items(): insert_secret(domain, secret_name, secret) def share_secrets(target, source, secrets): - target_domain = map_domain(target) - source_domain = map_domain(source) + target_domain = map_subdomain(target) + source_domain = map_subdomain(source) stored_secrets = abra("app", "secret", "ls", target_domain).splitlines() for source_secret in secrets: target_secret = secrets[source_secret] @@ -162,7 +180,7 @@ def uncomment(keys, path, match_all=False): def insert_domains(path, app): - domain = map_domain(app) + domain = map_subdomain(app) logging.debug(f'replace all {app}.example.com with {domain} in {path}') with open(path, "r") as file: content = file.read() @@ -172,15 +190,15 @@ def insert_domains(path, app): def execute_cmds(app): - domain = map_domain(app) + domain = map_subdomain(app) all_cmds = [] if (app_conf := DEFAULTS.get(app)) and (cmds := app_conf.get("execute")): all_cmds += cmds if COMBINE.get(app): for target_app, target_conf in COMBINE[app].items(): - if target_app in CONFIG["apps"] and (cmds := target_conf.get("execute")): + if target_app in INSTANCE["apps"] and (cmds := target_conf.get("execute")): all_cmds += cmds - if (app_conf := CONFIG["apps"][app]) and (cmds := app_conf.get("execute")): + if (app_conf := INSTANCE["apps"][app]) and (cmds := app_conf.get("execute")): all_cmds += cmds for cmd in all_cmds: container = cmd.split()[0] @@ -190,14 +208,19 @@ def execute_cmds(app): else: print(abra("app", "cmd", domain, container, cmd, ignore_error=True)) + @click.group() -@click.option('-l','--log','loglevel') -def cli(loglevel): +@click.option('-l', '--log', 'loglevel') +@click.option('-c', '--config', 'config') +def cli(loglevel, config): + global INSTANCE if loglevel: numeric_level = getattr(logging, loglevel.upper(), None) if not isinstance(numeric_level, int): raise ValueError('Invalid log level: %s' % loglevel) logging.basicConfig(level=numeric_level) + if config: + INSTANCE = read_config(config) @cli.command() @@ -212,38 +235,39 @@ def init_server(): def setup_apps(apps): """ Configure and connect the apps """ if not apps: - apps = CONFIG["apps"] + apps = INSTANCE["apps"] for app in apps: new_app(app) if configs := DEFAULTS.get(app): logging.info(f'set defaults for {app}') set_configs(app, configs) - if configs := CONFIG["apps"].get(app): + if configs := INSTANCE["apps"].get(app): logging.info(f'set extra configs for {app}') set_configs(app, configs) for app in apps: + print(f'create connections for {app}') connect_apps(app) for app in apps: logging.info(f'generate secrets for {app}') generate_secrets(app) + def get_deployed_apps(): - deployed = abra( - "app", "ls", "-S", "-s", CONFIG["server"], "-m", machine_output=True - ) + server = INSTANCE["server"] if INSTANCE.get("server") else INSTANCE["domain"] + deployed = abra("app", "ls", "-S", "-s", server, "-m", machine_output=True) deployed_apps = filter( - lambda x: x["status"] == "deployed", deployed[CONFIG["server"]]["apps"] - ) + lambda x: x["status"] == "deployed", deployed[server]["apps"]) deployed_domains = [app["domain"] for app in deployed_apps] return deployed_domains + @cli.command() @click.option('-a', '--apps', multiple=True) def deploy_apps(apps): """ Deploy all the apps """ deployed_domains = get_deployed_apps() - for app in CONFIG["apps"]: - domain = map_domain(app) + for app in INSTANCE["apps"]: + domain = map_subdomain(app) if (apps and app in apps) or domain not in deployed_domains: logging.info(f'deploy {domain}') print(abra("app", "deploy", "-C", "-n", domain)) @@ -257,40 +281,43 @@ def undeploy_apps(apps): """ Undeploy all the apps """ deployed_domains = get_deployed_apps() if not apps: - apps = CONFIG["apps"] + apps = INSTANCE["apps"] for app in apps: - domain = map_domain(app) + domain = map_subdomain(app) logging.info(f'undeploy {domain}') if domain in deployed_domains: print(abra("app", "undeploy", "-n", domain)) + @cli.command() @click.option('-a', '--apps', multiple=True) def cmds(apps): """ execute all post deploy cmds """ deployed_domains = get_deployed_apps() if not apps: - apps = CONFIG["apps"] + apps = INSTANCE["apps"] for app in apps: - domain = map_domain(app) + domain = map_subdomain(app) if domain in deployed_domains: logging.info(f'execute commands for {domain}') execute_cmds(app) + @cli.command() def purge_apps(): """ Completely remove all the apps """ - for recipe in CONFIG["apps"]: - domain = map_domain(recipe) + for recipe in INSTANCE["apps"]: + domain = map_subdomain(recipe) logging.info(f'purge {domain}') abra("app", "rm", "-n", domain) print(f"{domain} purged") + @cli.command() def purge_secrets(): """ Remove all the apps secrets """ - for recipe in CONFIG["apps"]: - domain = map_domain(recipe) + for recipe in INSTANCE["apps"]: + domain = map_subdomain(recipe) logging.info(f'purge secrets from {domain}') stored_secrets = abra("app", "secret", "ls", domain) if "true" in stored_secrets: diff --git a/combine.yml b/combine.yml index da5124d..e5e1662 100644 --- a/combine.yml +++ b/combine.yml @@ -65,7 +65,6 @@ nextcloud: uncomment: - ONLYOFFICE_URL - SECRET_ONLYOFFICE_JWT_VERSION - - SECRET_BBB_SECRET_VERSION execute: - app install_onlyoffice onlyoffice: @@ -110,6 +109,7 @@ matrix-synapse: KEYCLOAK_ID: authentik KEYCLOAK_NAME: sso KEYCLOAK_URL: https://authentik.example.com/application/o/matrix/ + # TODO: correct client domain? KEYCLOAK_CLIENT_DOMAIN: https://element-web.example.com KEYCLOAK_ALLOW_EXISTING_USERS: true # TODO: set CLIENT_ID as secret diff --git a/domains.yml b/domains.yml deleted file mode 100644 index d1fc886..0000000 --- a/domains.yml +++ /dev/null @@ -1,6 +0,0 @@ -authentik: example.com -nextcloud: cloud.example.com -wekan: board.example.com -element-web: chat.example.com -backup-bot-two: backup.example.com -matrix-synapse: matrix.example.com diff --git a/config.yml b/examples/config.yml similarity index 79% rename from config.yml rename to examples/config.yml index c2ab586..88c1e2f 100644 --- a/config.yml +++ b/examples/config.yml @@ -8,7 +8,9 @@ apps: nextcloud: onlyoffice: wordpress: + subdomain: "blog.special-domain.com" vikunja: + subdomain: "todo.example.com" matrix-synapse: element-web: env: diff --git a/defaults.yml b/examples/defaults.yml similarity index 88% rename from defaults.yml rename to examples/defaults.yml index 434db0d..e83e3b8 100644 --- a/defaults.yml +++ b/examples/defaults.yml @@ -4,6 +4,7 @@ GLOBALS: smtp_domain: example.com smtp_host: mail.example.com authentik: + subdomain: example.com env: AUTHENTIK_EMAIL__HOST: "{{smtp_host}}" AUTHENTIK_EMAIL__USERNAME: "{{smtp_user}}@{{smtp_domain}}" @@ -18,6 +19,7 @@ authentik: secrets: email_pass: "{{smtp_password}}" nextcloud: + subdomain: cloud.example.com env: SMTP_AUTHTYPE: LOGIN SMTP_HOST: "{{smtp_host}}" @@ -27,10 +29,12 @@ nextcloud: MAIL_FROM_ADDRESS: "{{smtp_user}}" MAIL_DOMAIN: "{{smtp_domain}}" APPS: "calendar" + BBB_URL: "https://talk.example.com/bigbluebutton/" uncomment: - compose.smtp.yml - SECRET_SMTP_PASSWORD_VERSION - compose.apps.yml + - SECRET_BBB_SECRET_VERSION secrets: smtp_password: "{{smtp_password}}" execute: @@ -66,12 +70,14 @@ vikunja: secrets: smtp_password: "{{smtp_password}}" matrix-synapse: + subdomain: matrix.example.com env: SMTP_APP_NAME: mail SMTP_FROM: "{{smtp_user}}@{{smtp_domain}}" SMTP_HOST: "{{smtp_host}}" SMTP_PORT: 587 SMTP_USER: "{{smtp_user}}@{{smtp_domain}}" + ENCRYPTED_BY_DEFAULT: "off" uncomment: - POST_DEPLOY_CMDS - compose.smtp.yml @@ -79,10 +85,15 @@ matrix-synapse: - SECRET_SMTP_PASSWORD_VERSION secrets: smtp_password: "{{smtp_password}}" +element-web: + subdomain: chat.example.com wekan: + subdomain: board.example.com env: # TODO fix format for smtp_password MAIL_URL: "smtp://{{smtp_user}}%40{{smtp_domain}}%3A{{smtp_password}}@{{smtp_host}}:587/" MAIL_FROM: "Wekan Notifications <{{smtp_user}}@{{smtp_domain}}>" uncomment: - PASSWORD_LOGIN_ENABLED +backup-bot-two: + subdomain: backup.example.com