feat: make alerts configurable

2026-03-31 20:14:58 +02:00
19 changed files with 178 additions and 267 deletions
--- a/.env.sample
+++ b/.env.sample
@ -6,7 +6,8 @@ DOMAIN=monitoring-ng.example.com
 ENABLE_BACKUPS=true

 ## Enable this secret for Promtail / Prometheus
-# SECRET_BASIC_AUTH_VERSION=v1
+#COMPOSE_FILE="$COMPOSE_FILE:compose.basic-auth.yml"
+#SECRET_BASIC_AUTH_VERSION=v1
 #
 # Promtail (Gathering Logs)
 # COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
@ -19,7 +20,6 @@ ENABLE_BACKUPS=true
 #
 ## Prometheus
 # COMPOSE_FILE="$COMPOSE_FILE:compose.prometheus.yml"
-# PROMETHEUS_REMOTE_WRITE_URL=https://prometheus.$DOMAIN/api/v1/write
 # PROMETHEUS_RETENTION_TIME=1y
 #
 ## Prometheus Pushgateway
@ -40,20 +40,20 @@ ENABLE_BACKUPS=true
 # LOKI_AWS_REGION=eu-west-1
 # LOKI_ACCESS_KEY_ID=bush-debrief-approval-robust-scraggly-molecule
 # LOKI_BUCKET_NAMES=loki
-# SECRET_LOKI_AWS_KEY_VERSION=v1
+# SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION=v1
 #
 ## Grafana
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
 # GF_SERVER_ROOT_URL=https://monitoring.example.com
-# SECRET_GF_ADMINPASSWD_VERSION=v1
+# SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
 ## Seperate domain for Grafana
 #GRAFANA_DOMAIN=grafana.example.com
 #
 ## Single-Sign-On with OIDC
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
 # OIDC_ENABLED=1
-# SECRET_GF_OIDC_SECRET_VERSION=v1
+# SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION=v1
 # OIDC_CLIENT_ID=grafana
 # OIDC_AUTH_URL="https://authentik.example.com/application/o/authorize/"
 # OIDC_API_URL="https://authentik.example.com/application/o/userinfo/"
@ -70,19 +70,20 @@ ENABLE_BACKUPS=true
 # GF_SMTP_ENABLED=true
 # GF_SMTP_FROM_ADDRESS=grafana@example.com
 # GF_SMTP_SKIP_VERIFY=false
-# SECRET_GF_SMTP_PASSWD_VERSION=v1
+# SECRET_GRAFANA_SMTP_PASSWORD_VERSION=v1
 #

 ## Grafana Matrix Contact Point (optional)
 #COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
-#SECRET_MATRIX_TOKEN_VERSION=v1
+#SECRET_MATRIX_ACCESS_TOKEN_VERSION=v1
 #GF_MATRIX_USER_ID="<user-id>"
 #GF_MATRIX_ROOM_ID="<room-id>"
 #GF_MATRIX_HOMESERVER_URL="<homeserver-url>"

-# ALerts
-#ALERT_BACKUP_FAILED_ENABLED=true
-#ALERT_BACKUP_MISSING_ENABLED=true
-#ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED=true
-#ALERT_NODE_DISK_SPACE_ENABLED=true
-#ALERT_NODE_MEMORY_USAGE_ENABLED=true
+## ALerts
+
+# Node disk space alert will trigger when free disk space left is below the given number in percent
+#ALERT_NODE_DISK_SPACE_LEFT=10
+
+# Node memory usage alert will trigger when memory usage is above the given number in percent
+#ALERT_NODE_MEMORY_USAGE=85
--- a/README.md
+++ b/README.md
@ -145,7 +145,7 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"

 2. Insert the matrix access token secret:
 ```
-abra app secret insert monitoring.marx.klasse-methode.it matrix_token v1
+abra app secret insert monitoring.marx.klasse-methode.it matrix_access_token v1
 ```

 3. Set required configurations:
@ -156,13 +156,9 @@ GF_MATRIX_HOME_SERVER_URL=
 ```
 4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`

-## alerts
-
-It is possible to enable the following alerts, by setting the corresponding env variable to `true`:
- backupbot failed: `ALERT_BACKUP_FAILED_ENABLED`
- backupbot missing: `ALERT_BACKUP_MISSING_ENABLED`
- backupbot not successfull: `ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED`
- node disk space: `ALERT_NODE_DISK_SPACE_ENABLED`
- node memory usage: `ALERT_NODE_MEMORY_USAGE_ENABLED`
+## Alerts

+It is possible to enable the following alerts, by uncommenting the corresponding env variable:

+- node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
+- node memory usage: `ALERT_NODE_MEMORY_USAGE`
--- a/abra.sh
+++ b/abra.sh
@ -1,18 +1,17 @@
 export ENTRYPOINT_VERSION=v1
-export GF_DATASOURCES_VERSION=v1
-export GF_DASHBOARDS_VERSION=v2
-export GF_SWARM_DASH_VERSION=v2
-export GF_STACKS_DASH_VERSION=v2
-export GF_TRAEFIK_DASH_VERSION=v2
-export GF_BACKUP_DASH_VERSION=v1
-export GF_CUSTOM_INI_VERSION=v4
+export GRAFANA_DATASOURCES_YML_VERSION=v1
+export GRAFANA_DASHBOARDS_YML_VERSION=v2
+export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_BACKUP_DASHBOARD_JSON_VERSION=v1
+export GRAFANA_CUSTOM_INI_VERSION=v4
 export PROMTAIL_YML_VERSION=v3
-export LOKI_YML_VERSION=v3
+export LOKI_YML_VERSION=v2
 export PROMETHEUS_YML_VERSION=v2
-export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
-export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
-export GRAFANA_ALERTS_NODE_VERSION=v1c
-export CONFIG_ALLOY_VERSION=v5
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
+export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
+export GRAFANA_ALERTS_NODE_VERSION=v2

 # creates a default prometheus scrape config for a given node
 add_node(){
@ -23,101 +22,6 @@ add_node(){
  cat "/prometheus/scrape_configs/$name.yml"
 }

-# migrates secrets from old names to new names by reading values from the
-# running containers on the server and re-inserting them under the new names.
-# preview changes: abra app cmd --local <app> migrate_secret_names
-# execute changes: abra app cmd --local <app> migrate_secret_names execute
-migrate_secret_names() {
-  if ! command -v jq &> /dev/null; then
-    echo "jq is required on your local machine to migrate secret names"
-    echo "It could not be found in your PATH, please install jq to proceed."
-    echo "For example: On a debian/ubuntu system, run `apt install jq`"
-    exit 1
-  fi
-
-  # Hardcoded migration mappings: old_secret_name|new_secret_name
-  MIGRATIONS="
-grafana_admin_password|gf_adminpasswd
-grafana_smtp_password|gf_smtp_passwd
-grafana_oidc_client_secret|gf_oidc_secret
-matrix_access_token|matrix_token
-loki_aws_secret_access_key|loki_aws_key
-"
-
-  # Determine which server the app is deployed on
-  SERVER=$(abra app ls -m | jq -r --arg domain "$APP_NAME" '[.[].apps[] | select(.domain == $domain) | .server] | first' 2>/dev/null)
-
-  if [ -z "$SERVER" ]; then
-    echo "Error: could not determine server for app '$APP_NAME'"
-    exit 1
-  fi
-
-  # Build a lookup table of all secrets currently mounted in this stack.
-  # Each line: <secretID> <containerID> <secretName>
-  LOOKUP=$(ssh "$SERVER" "
-    docker stack services ${STACK_NAME} --format '{{.Name}}' | while read svc; do
-      CID=\$(docker ps --no-trunc -q --filter \"name=\${svc}\" | head -1)
-      docker service inspect \"\$svc\" --format '{{json .Spec.TaskTemplate.ContainerSpec.Secrets}}' | \
-        jq -r --arg cid \"\$CID\" '.[]? | .SecretID + \" \" + \$cid + \" \" + .SecretName'
-    done | sort -k3 -r
-  " 2>/dev/null)
-
-  echo "Secret migration plan for: $APP_NAME (server: $SERVER)"
-  echo ""
-  printf "  %-24s %-8s %s\n" "OLD NAME" "FOUND" "ACTION"
-  printf "  %-24s %-8s %s\n" "--------" "-----" "------"
-
-  # Check each old name against the lookup table and display the plan
-  ANY_FOUND=false
-  while IFS='|' read -r OLD_NAME NEW_NAME; do
-    [ -z "$OLD_NAME" ] && continue
-    MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
-    if [ -n "$MATCH" ]; then
-      printf "  %-24s %-8s %s\n" "$OLD_NAME" "yes" "recreate as '$NEW_NAME' version V1"
-      ANY_FOUND=true
-    else
-      printf "  %-24s %-8s %s\n" "$OLD_NAME" "no" "nothing (not found on server)"
-    fi
-  done <<< "$MIGRATIONS"
-
-  echo ""
-
-  if [ "$ANY_FOUND" = false ]; then
-    echo "No old secrets found on server. Nothing to migrate."
-    return 0
-  fi
-
-  if [ "$1" != "execute" ]; then
-    echo "To apply the above changes, run:"
-    echo "  abra app cmd --local $APP_NAME migrate_secret_names execute"
-    return 0
-  fi
-
-  # read each found secret from its container and re-insert with the new name
-  while IFS='|' read -r OLD_NAME NEW_NAME; do
-    [ -z "$OLD_NAME" ] && continue
-
-    MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
-    [ -z "$MATCH" ] && continue
-
-    SECRET_ID=$(echo "$MATCH" | awk '{print $1}')
-    CID=$(echo "$MATCH" | awk '{print $2}')
-    SECRET_VALUE=$(ssh "$SERVER" "cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null || sudo cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null")
-
-    if [ -z "$SECRET_VALUE" ]; then
-      echo "Error: could not read value for '$OLD_NAME', skipping"
-      continue
-    fi
-
-    echo "Migrating: '$OLD_NAME' -> '$NEW_NAME' (v1)"
-    printf '%s' "$SECRET_VALUE" | abra app secret insert -C "$APP_NAME" "$NEW_NAME" v1
-    
-  done <<< "$MIGRATIONS"
-
-  echo ""
-  echo "Done."
-}
-
 # adds a domain to a scrape config or creates a new one
 add_domain(){
  name=$1
--- a/alertmanager-matrix-config.yml.tmpl
+++ b/alertmanager-matrix-config.yml.tmpl
@ -12,7 +12,7 @@ http:
 matrix:
  homeserver-url: "{{ env "GF_MATRIX_HOMESERVER_URL" }}"
  user-id: "{{ env "GF_MATRIX_USER_ID" }}"
-  access-token: "{{ secret "matrix_token" }}"
+  access-token: "{{ secret "matrix_access_token" }}"
  room-mapping:
    matrixroom: "{{ env "GF_MATRIX_ROOM_ID" }}"

--- a/alerts/node.yml.tmpl
+++ b/alerts/node.yml.tmpl
@ -2,13 +2,13 @@ apiVersion: 1

 # List of alert rule UIDs that should be deleted
 deleteRules:
-  {{ if ne (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
+  {{ if not (env "ALERT_NODE_DISK_SPACE_LEFT") }}
  - orgId: 1
-    uid: bds8bhxu97pxca
+    uid: coopcloud_node_disk_space_left
  {{ end }}
-  {{ if ne (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
+  {{ if not (env "ALERT_NODE_MEMORY_USAGE") }}
  - orgId: 1
-    uid: ads8cswmly96oa
+    uid: coopcloud_node_memory_usage
  {{ end }}

 groups:
@ -17,8 +17,8 @@ groups:
  folder: node
  interval: 5m
  rules:
-    {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-    - uid: bds8bhxu97pxca
+    {{ if (env "ALERT_NODE_DISK_SPACE_LEFT") }}
+    - uid: coopcloud_node_disk_space_left
      title: Node Disk Space
      condition: C
      data:
@ -45,7 +45,7 @@ groups:
            conditions:
                - evaluator:
                    params:
-                        - 10
+                        - {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}
                    type: lt
                  operator:
                    type: and
@ -70,13 +70,13 @@ groups:
      annotations:
        description: ""
        runbook_url: ""
-        summary: Less than 10% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
+        summary: Less than {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
      labels:
        "": ""
      isPaused: false
    {{ end }}
-    {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-    - uid: ads8cswmly96oa
+    {{ if (env "ALERT_NODE_MEMORY_USAGE")  }}
+    - uid: coopcloud_node_memory_usage
      title: Node Memory Usage
      condition: C
      data:
@ -103,7 +103,7 @@ groups:
            conditions:
                - evaluator:
                    params:
-                        - 85
+                        - {{ env "ALERT_NODE_MEMORY_USAGE"  }}
                    type: gt
                  operator:
                    type: and
@ -126,6 +126,6 @@ groups:
      execErrState: Error
      for: 5m
      annotations:
-        summary: Memory usage is above 85% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
+        summary: Memory usage is above {{ env "ALERT_NODE_MEMORY_USAGE"  }}% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
      isPaused: false
    {{ end }}
--- a/compose.basic-auth.yml
+++ b/compose.basic-auth.yml
@ -0,0 +1,7 @@
+---
+version: "3.8"
+
+secrets:
+  basic_auth:
+    external: true
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}
--- a/compose.grafana-oidc.yml
+++ b/compose.grafana-oidc.yml
@ -3,7 +3,7 @@ version: '3.8'
 services:
  grafana:
    secrets:
-      - gf_oidc_secret
+      - grafana_oidc_client_secret
    environment:
      - OIDC_API_URL
      - OIDC_AUTH_URL
@ -12,6 +12,6 @@ services:
      - OIDC_TOKEN_URL

 secrets:
-  gf_oidc_secret:
+  grafana_oidc_client_secret:
    external: true
-    name: ${STACK_NAME}_gf_oidc_secret_${SECRET_GF_OIDC_SECRET_VERSION}
+    name: ${STACK_NAME}_grafana_oidc_client_secret_${SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION}
--- a/compose.grafana-smtp.yml
+++ b/compose.grafana-smtp.yml
@ -3,16 +3,16 @@ version: '3.8'
 services:
  grafana:
    secrets:
-      - gf_smtp_passwd
+      - grafana_smtp_password
    environment:
      - GF_SMTP_HOST
      - GF_SMTP_USER
-      - GF_SMTP_PASSWORD__FILE=/run/secrets/gf_smtp_passwd
+      - GF_SMTP_PASSWORD__FILE=/run/secrets/grafana_smtp_password
      - GF_SMTP_ENABLED
      - GF_SMTP_FROM_ADDRESS
      - GF_SMTP_SKIP_VERIFY

 secrets:
-  gf_smtp_passwd:
+  grafana_smtp_password:
    external: true
-    name: ${STACK_NAME}_gf_smtp_passwd_${SECRET_GF_SMTP_PASSWD_VERSION}
+    name: ${STACK_NAME}_grafana_smtp_password_${SECRET_GRAFANA_SMTP_PASSWORD_VERSION}
--- a/compose.grafana.yml
+++ b/compose.grafana.yml
@ -2,25 +2,25 @@ version: '3.8'

 services:
  grafana:
-    image: grafana/grafana:12.4.0
+    image: grafana/grafana:10.4.14
    volumes:
      - grafana-data:/var/lib/grafana:rw
    secrets:
-      - gf_adminpasswd
+      - grafana_admin_password
    configs:
-      - source: gf_custom_ini
+      - source: grafana_custom_ini
        target: /etc/grafana/grafana.ini
-      - source: gf_datasources
+      - source: grafana_datasources_yml
        target: /etc/grafana/provisioning/datasources/datasources.yml
-      - source: gf_dashboards
+      - source: grafana_dashboards_yml
        target: /etc/grafana/provisioning/dashboards/dashboards.yml
-      - source: gf_swarm_dash
+      - source: grafana_swarm_dashboard_json
        target: /var/lib/grafana/dashboards/docker-swarm-nodes.json
-      - source: gf_stacks_dash
+      - source: grafana_stacks_dashboard_json
        target: /var/lib/grafana/dashboards/docker-swarm-stacks.json
-      - source: gf_traefik_dash
+      - source: grafana_traefik_dashboard_json
        target: /var/lib/grafana/dashboards/traefik.json
-      - source: gf_backup_dash
+      - source: grafana_backup_dashboard_json
        target: /var/lib/grafana/dashboards/backup.json
      - source: gf_alerts_node
        target: /etc/grafana/provisioning/alerting/node.yml
@ -29,11 +29,11 @@ services:
      - internal
    environment:
      - GF_SERVER_ROOT_URL
-      - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/gf_adminpasswd
+      - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/grafana_admin_password
      - GF_SECURITY_ALLOW_EMBEDDING
      - GF_INSTALL_PLUGINS
-      - ALERT_NODE_DISK_SPACE_ENABLED
-      - ALERT_NODE_MEMORY_USAGE_ENABLED
+      - ALERT_NODE_DISK_SPACE_LEFT
+      - ALERT_NODE_MEMORY_USAGE
    deploy:
      labels:
        - "traefik.enable=true"
@ -51,27 +51,27 @@ services:
      start_period: 10s

 configs:
-  gf_custom_ini:
+  grafana_custom_ini:
    template_driver: golang
-    name: ${STACK_NAME}_gf_custom_ini_${GF_CUSTOM_INI_VERSION}
+    name: ${STACK_NAME}_grafana_custom_ini_${GRAFANA_CUSTOM_INI_VERSION}
    file: grafana_custom.ini
-  gf_datasources:
-    name: ${STACK_NAME}_gf_datasources_${GF_DATASOURCES_VERSION}
+  grafana_datasources_yml:
+    name: ${STACK_NAME}_g_datasources_yml_${GRAFANA_DATASOURCES_YML_VERSION}
    file: grafana-datasources.yml
-  gf_dashboards:
-    name: ${STACK_NAME}_gf_dashboards_${GF_DASHBOARDS_VERSION}
+  grafana_dashboards_yml:
+    name: ${STACK_NAME}_g_dashboards_yml_${GRAFANA_DASHBOARDS_YML_VERSION}
    file: grafana-dashboards.yml
-  gf_swarm_dash:
-    name: ${STACK_NAME}_gf_swarm_dash_${GF_SWARM_DASH_VERSION}
+  grafana_swarm_dashboard_json:
+    name: ${STACK_NAME}_g_swarm_dashboard_json_${GRAFANA_SWARM_DASHBOARD_JSON_VERSION}
    file: grafana-swarm-dashboard.json
-  gf_stacks_dash:
-    name: ${STACK_NAME}_gf_stacks_dash_${GF_STACKS_DASH_VERSION}
+  grafana_stacks_dashboard_json:
+    name: ${STACK_NAME}_g_stacks_dashboard_json_${GRAFANA_STACKS_DASHBOARD_JSON_VERSION}
    file: grafana-stacks-dashboard.json
-  gf_traefik_dash:
-    name: ${STACK_NAME}_gf_traefik_dash_${GF_TRAEFIK_DASH_VERSION}
+  grafana_traefik_dashboard_json:
+    name: ${STACK_NAME}_g_traefik_dashboard_json_${GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION}
    file: grafana-traefik-dashboard.json
-  gf_backup_dash:
-    name: ${STACK_NAME}_gf_backup_dash_${GF_BACKUP_DASH_VERSION}
+  grafana_backup_dashboard_json:
+    name: ${STACK_NAME}_g_backup_dashboard_json_${GRAFANA_BACKUP_DASHBOARD_JSON_VERSION}
    file: grafana-backup-dashboard.json
  gf_alerts_node:
    template_driver: golang
@ -83,6 +83,6 @@ volumes:


 secrets:
-  gf_adminpasswd:
+  grafana_admin_password:
    external: true
-    name: ${STACK_NAME}_gf_adminpasswd_${SECRET_GF_ADMINPASSWD_VERSION}
+    name: ${STACK_NAME}_grafana_admin_password_${SECRET_GRAFANA_ADMIN_PASSWORD_VERSION}
--- a/compose.loki.yml
+++ b/compose.loki.yml
@ -2,7 +2,7 @@ version: '3.8'

 services:
  loki:
-    image: grafana/loki:3.6.7
+    image: grafana/loki:2.9.11
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - proxy
@ -12,7 +12,7 @@ services:
    volumes:
      - loki-data:/loki
    # secrets:
-    #   - loki_aws_key
+    #   - loki_aws_secret_access_key
    environment:
      - LOKI_ACCESS_KEY_ID
      - LOKI_AWS_ENDPOINT
@ -47,6 +47,6 @@ volumes:
  loki-data:

 # secrets:
-#   loki_aws_key:
+#   loki_aws_secret_access_key:
 #     external: true
-#     name: ${STACK_NAME}_loki_aws_key_${SECRET_LOKI_AWS_KEY_VERSION}
+#     name: ${STACK_NAME}_loki_aws_secret_access_key_${SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION}
--- a/compose.matrix-alertmanager-receiver.yml
+++ b/compose.matrix-alertmanager-receiver.yml
@ -2,9 +2,9 @@ version: '3.8'

 services:
  matrix-alertmanager-receiver:
-    image: metio/matrix-alertmanager-receiver:2026.2.25
+    image: metio/matrix-alertmanager-receiver:2025.2.9
    secrets:
-      - matrix_token
+      - matrix_access_token
    configs:
      - source: matrix-alertmanager-receiver-config
        target: /etc/matrix-alertmanager-receiver/config.yml
@ -23,6 +23,6 @@ configs:
    file: alertmanager-matrix-config.yml.tmpl

 secrets:
-  matrix_token:
+  matrix_access_token:
    external: true
-    name: ${STACK_NAME}_matrix_token_${SECRET_MATRIX_TOKEN_VERSION}
+    name: ${STACK_NAME}_matrix_access_token_${SECRET_MATRIX_ACCESS_TOKEN_VERSION}
--- a/compose.prometheus.yml
+++ b/compose.prometheus.yml
@ -2,7 +2,7 @@ version: '3.8'

 services:
  prometheus:
-    image: prom/prometheus:v3.10.0
+    image: prom/prometheus:v2.55.1
    secrets:
      - basic_auth
    volumes:
@ -16,8 +16,6 @@ services:
      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
      - "--web.console.templates=/usr/share/prometheus/consoles"
      - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME}"
-      - "--enable-feature=remote-write-receiver"
-      - "--web.enable-remote-write-receiver"
    networks:
      - proxy
      - internal
--- a/compose.promtail.yml
+++ b/compose.promtail.yml
@ -2,7 +2,7 @@ version: "3.8"

 services:
  promtail:
-    image: grafana/promtail:3.6.7
+    image: grafana/promtail:2.9.11
    volumes:
      - /var/log:/var/log:ro
      - /var/run/docker.sock:/var/run/docker.sock
@ -23,8 +23,3 @@ configs:
    name: ${STACK_NAME}_promtail_yml_${PROMTAIL_YML_VERSION}
    file: promtail.yml.tmpl
    template_driver: golang
-
-secrets:
-  basic_auth:
-    external: true
-    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}
--- a/compose.pushgateway.yml
+++ b/compose.pushgateway.yml
@ -2,7 +2,7 @@ version: '3.8'

 services:
  pushgateway:
-    image: prom/pushgateway:v1.11.2
+    image: prom/pushgateway:v1.10.0
    command:
      - '--web.listen-address=:9191'
      - '--push.disable-consistency-check'
--- a/compose.yml
+++ b/compose.yml
@ -3,41 +3,89 @@ version: "3.8"

 services:
  app:
-    image: grafana/alloy:v1.16.1
-    hostname: "${DOMAIN}"
-    configs:
-      - source: config_alloy
-        target: /etc/alloy/config.alloy
+    image: prom/node-exporter:v1.8.1
+    user: root
+    environment:
+      - NODE_ID={{.Node.ID}}
    volumes:
-      - alloy-data:/var/lib/alloy/data
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
      - /:/rootfs:ro
+      - /etc/hostname:/etc/nodename:ro
    command:
-      - "run"
-      - "--storage.path=/var/lib/alloy/data"
-      - "/etc/alloy/config.alloy"
+      - "--path.sysfs=/host/sys"
+      - "--path.procfs=/host/proc"
+      - "--path.rootfs=/rootfs"
+      - "--collector.textfile.directory=/etc/node-exporter/"
+      - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)"
+      - "--no-collector.ipvs"
+    configs:
+      - source: entrypoint
+        target: /entrypoint.sh
    networks:
      - internal
-    secrets:
-      - basic_auth
+      - proxy
+    entrypoint: [ "/bin/sh", "-e", "/entrypoint.sh" ]
    deploy:
      restart_policy:
        condition: on-failure
      labels:
        - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "traefik.enable=false"
+        - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
+        - "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
+        - "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-node.tls=true"
+        - "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
        - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
+        - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT}"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.2
+    command:
+      - "-logtostderr"
+      - "--enable_metrics=cpu,cpuLoad,disk,diskIO,process,memory,network"
+      # all possible metrics: advtcp,app,cpu,cpuLoad,cpu_topology,cpuset,disk,diskIO,hugetlb,memory,memory_numa,network,oom_event,percpu,perf_event,process,referenced_memory,resctrl,sched,tcp,udp.
+      - "--housekeeping_interval=120s"
+      - "--docker_only=true"
+    volumes:
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+      - /sys:/sys:ro
+      - /var/run:/var/run:ro
+      - /:/rootfs:ro
+    networks:
+      - internal
+      - proxy
+    deploy:
+      restart_policy:
+        condition: on-failure
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
+        - "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.tls=true"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.middlewares=basicauth@file"
+    healthcheck:
+      test: wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1
+      interval: 15s
+      timeout: 15s
+      retries: 5
+      start_period: 30s
+
 configs:
-  config_alloy:
-    template_driver: golang
-    name: ${STACK_NAME}_config_alloy_${CONFIG_ALLOY_VERSION}
-    file: config.alloy.tmpl
+  entrypoint:
+    name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
+    file: node-exporter-entrypoint.sh
+
+
+
 networks:
  proxy:
    external: true
  internal:
-volumes:
-  alloy-data:
-secrets:
-  basic_auth:
-    external: true
-    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}
--- a/config.alloy.tmpl
+++ b/config.alloy.tmpl
@ -1,36 +0,0 @@
-logging {
-  level = "info"
-  format = "logfmt"
-}
-
-prometheus.exporter.cadvisor "docker" {
-}
-
-prometheus.exporter.unix "default" {
-  include_exporter_metrics = true
-  rootfs_path = "/rootfs"
-}
-
-prometheus.scrape "default" {
-  targets = array.concat(
-    [{
-      job         = "alloy",
-      __address__ = "127.0.0.1:12345",
-    }],
-    prometheus.exporter.unix.default.targets,
-    prometheus.exporter.cadvisor.docker.targets,
-  )
-
-  forward_to = [prometheus.remote_write.prometheus.receiver]
-}
-
-prometheus.remote_write "prometheus" {
-  endpoint {
-    url = "{{ env "PROMETHEUS_REMOTE_WRITE_URL" }}"
-
-    basic_auth {
-      username = "admin"
-      password = "{{ secret "basic_auth" }}"
-    }
-  }
-}
--- a/grafana_custom.ini
+++ b/grafana_custom.ini
@ -21,7 +21,7 @@ tls_skip_verify_insecure = false
 allow_sign_up = true
 auto_login = true
 client_id = {{ env "OIDC_CLIENT_ID" }}
-client_secret = {{ secret "gf_oidc_secret" }}
+client_secret = {{ secret "grafana_oidc_client_secret" }}
 auth_url = {{ env "OIDC_AUTH_URL" }}
 token_url = {{ env "OIDC_TOKEN_URL" }}
 api_url = {{ env "OIDC_API_URL" }}
--- a/loki.yml.tmpl
+++ b/loki.yml.tmpl
@ -34,6 +34,7 @@ ingester:
  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
  wal:
    dir: "/tmp/wal"

@ -52,7 +53,7 @@ schema_config:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
-      schema: v13
+      schema: v11
      index:
        prefix: index_
        period: 24h
@ -62,6 +63,7 @@ storage_config:
    active_index_directory: /loki/boltdb-shipper-active
    cache_location: /loki/boltdb-shipper-cache
    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
  filesystem:
    directory: /loki/chunks
 {{ end }}
@ -70,6 +72,7 @@ schema_config:
  configs:
  - from: 2020-11-25
    store: boltdb-shipper
+    object_store: aws
    schema: v11
    index:
      prefix: index_
@ -86,7 +89,7 @@ storage_config:
   endpoint: {{ env "LOKI_AWS_ENDPOINT" }}
   region: {{ env "LOKI_AWS_REGION" }}
   access_key_id: {{ env "LOKI_ACCESS_KEY_ID" }}
-   secret_access_key: {{ secret "loki_aws_key" }}
+   secret_access_key: {{ secret "loki_aws_secret_access_key" }}
   bucketnames: {{ env "LOKI_BUCKET_NAMES" }}
   insecure: false
   sse_encryption: false
@ -100,24 +103,19 @@ storage_config:

 compactor:
  working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
  compaction_interval: 10m
  retention_enabled: true
  retention_delete_delay: 2h
  retention_delete_worker_count: 150
-{{ if eq (env "LOKI_STORAGE_FILESYSTEM") "1" }}
-  delete_request_store: filesystem
-{{ end }}
-{{ if eq (env "LOKI_STORAGE_S3") "1" }}
-  delete_request_store: aws
-{{ end }}

 limits_config:
+  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  retention_period: {{ env "LOKI_RETENTION_PERIOD" }}
  split_queries_by_interval: 24h
  max_query_parallelism: 100
-  allow_structured_metadata: false

 query_scheduler:
  max_outstanding_requests_per_tenant: 4096
@ -125,6 +123,9 @@ query_scheduler:
 frontend:
  max_outstanding_per_tenant: 4096

+chunk_store_config:
+  max_look_back_period: 0s
+
 table_manager:
  retention_deletes_enabled: false
  retention_period: 0s
--- a/release/next
+++ b/release/next
@ -6,7 +6,4 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"

 COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"

-3. The scrape-config.example.yml file and add_node() command were updated to use a secure endpoint for the traefik metrics instead of http. This requires an updated Traefik recipe that publishes the metrics on https.
-
-4. Secret and config names were shortened to max 14 characters to prevent going over Docker's 64 character limit when STACK_NAME and VERSION are added to it.
-When upgrading, you need to reinsert the secrets with their shorter names. Run `abra app secret list <domain>` to see which secrets aren't created on the server (because their name was shortened) and run `abra app secret insert <domain> <secret_name> v1 <value>` to reinsert them with the shorter name. Or you can use the migrate_secret_names function in abra.sh to reinsert all existing secrets with their shorter name automatically: `abra app cmd --local <domain> migrate_secret_names`
+3. The scape-config.example.yml file and add_node() command were updated to use a secure endpoint for the traefik metrics instead http. This requires an updated Traefik recipe that publishes the metrics on https.