substitute global jinja variables in the last step

alakazam.py

@@ -12,7 +12,7 @@ from tabulate import tabulate
 import click
 import dotenv
 from icecream import ic
-from jinja2 import Environment, FileSystemLoader
+from jinja2 import Template
 from ruamel.yaml import YAML
 from ruamel.yaml.constructor import SafeConstructor
 from ruamel.yaml.nodes import ScalarNode
@@ -41,17 +41,17 @@ MySafeConstructor.add_constructor(
     MySafeConstructor.construct_yaml_str)
 
 
-def read_config(filepath: str) -> Dict[str, Any]:
+def read_config(configpath: str) -> Dict[str, Any]:
     """
-    Reads a YAML configuration file from the specified filepath and returns the configuration as a dictionary, including processing any Jinja2 templating specified within the file. It handles the loading of YAML files, processes any Jinja2 templating using the 'GLOBALS' section for variables, and converts the file into a dictionary format.
+    Reads a YAML configuration file from the specified configpath and returns the configuration as a dictionary, including processing any Jinja2 templating specified within the file. It handles the loading of YAML files, processes any Jinja2 templating using the 'GLOBALS' section for variables, and converts the file into a dictionary format.
 
     Args:
-        filepath (str): The path to the YAML configuration file. The file should contain valid YAML and optional Jinja2 templating.
+        configpath (str): The path to the YAML configuration file. The file should contain valid YAML and optional Jinja2 templating.
 
     Returns:
         dict: The configuration data extracted and processed from the YAML file, structured as a dictionary.
     """
-    filepath = Path(filepath).expanduser()
+    filepath = Path(configpath).expanduser()
     if not filepath.exists():
         logging.warning(f"config file {filepath} does not exist")
         return {}
@@ -62,11 +62,12 @@ def read_config(filepath: str) -> Dict[str, Any]:
         if not yaml_config:
             logging.warning(f"config file {filepath} is empty")
             return {}
-        globals = yaml_config.get('GLOBALS')
-    jinja = Environment(loader=FileSystemLoader(
-        ['/', '.']), trim_blocks=True, lstrip_blocks=True)
-    template = jinja.get_template(filepath.as_posix(), globals=globals)
-    return yaml.load(template.render())
+    return yaml_config
+    # globals = yaml_config.get('GLOBALS')
+    # jinja = Environment(loader=FileSystemLoader(
+    #     ['/', '.']), trim_blocks=True, lstrip_blocks=True)
+    # template = jinja.get_template(filepath.as_posix(), globals=globals)
+    # return yaml.load(template.render())
 
 
 def get_value(dictionary: Dict[Any, Any], *keys: str) -> Optional[Any]:
@@ -152,6 +153,24 @@ def merge_all_group_configs(dir_path: str) -> Dict[str, Dict[str, Any]]:
             merged_configs[root] = merged_configs.get(os.path.dirname(root))
     return merged_configs
 
+
+def substitute_jinja_variable(jinja_dict, subs_dict) -> None:
+    """
+    This function recursively traverses the given jinja_dict and wherever it finds a jinja template variable, it replaces it with the corresponding value from the subs_dict.
+
+    Args:
+        jinja_dict (dict): The dictionary which may contain jinja template variables. Can be a nested dictionary.
+        subs_dict (dict): The dictionary containing the substitutions for the jinja template variables. 
+    """
+    for key, value in jinja_dict.items():
+        if isinstance(value, dict):  # If value itself is dictionary
+            substitute_jinja_variable(value, subs_dict)  # Recursive call
+        else:
+            if "{{" in str(value) and "}}" in str(value):  # If value is a jinja template
+                template = Template(str(value))
+                jinja_dict[key] = template.render(subs_dict)
+
+
 def merge_instance_configs(pool_config: Dict[str, Any], instance_domain: str, instance_config: Dict[str, Any]) -> Dict[str, Any]:
     """
     Merge instance-specific configurations with pool configurations to produce a consolidated configuration dictionary.
@@ -180,6 +199,9 @@ def merge_instance_configs(pool_config: Dict[str, Any], instance_domain: str, in
             server = instance_domain
         if not merged_config[app].get('server'):
             merged_config[app]['server'] = server
+    if not (global_vars:= merged_config.get('GLOBALS')):
+        global_vars = pool_config.get('GLOBALS')
+    substitute_jinja_variable(merged_config, global_vars)
     if merged_config.get('GLOBALS'):
         merged_config.pop('GLOBALS')
     return merged_config

examples/example.com.yml

@@ -1,3 +1,13 @@
+GLOBALS: 
+    smtp_password: my_secret_password
+    smtp_password_urlencoded: my_secret_password
+    smtp_user: support
+    smtp_domain: maildomain.com
+    smtp_host: maymailhost.com
+    bbb_secret: my_secret_password
+    turn_secret: my_secret_password
+    basic_auth_password: my_secret_password
+    basic_auth_hash: my_secret_password
 traefik:
 backup-bot-two:
 monitoring-ng: