feat: make alerts configurable

This fixes the insecure Traefik metrics endpoint. See coop-cloud/traefik#94 for details.

Reviewed-on: #17
Co-authored-by: Danny Groenewegen <mail@dannygroenewegen.nl>
Co-committed-by: Danny Groenewegen <mail@dannygroenewegen.nl>

.env.sample

@@ -6,7 +6,8 @@ DOMAIN=monitoring-ng.example.com
 ENABLE_BACKUPS=true
 
 ## Enable this secret for Promtail / Prometheus
-# SECRET_BASIC_AUTH_VERSION=v1
+#COMPOSE_FILE="$COMPOSE_FILE:compose.basic-auth.yml"
+#SECRET_BASIC_AUTH_VERSION=v1
 #
 # Promtail (Gathering Logs)
 # COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
@@ -44,10 +45,10 @@ ENABLE_BACKUPS=true
 ## Grafana
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
-## GRAFANA_DOMAIN needs to be set. change it for a different domain
-#GRAFANA_DOMAIN=$DOMAIN
-# GF_SERVER_ROOT_URL=https://${GRAFANA_DOMAIN}
+# GF_SERVER_ROOT_URL=https://monitoring.example.com
 # SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
+## Seperate domain for Grafana
+#GRAFANA_DOMAIN=grafana.example.com
 #
 ## Single-Sign-On with OIDC
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
@@ -79,11 +80,10 @@ ENABLE_BACKUPS=true
 #GF_MATRIX_ROOM_ID="<room-id>"
 #GF_MATRIX_HOMESERVER_URL="<homeserver-url>"
 
-# ALerts
-#ALERT_BACKUP_FAILED_ENABLED=true
-#ALERT_BACKUP_MISSING_ENABLED=true
-#ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED=true
-#ALERT_NODE_DISK_SPACE_ENABLED=true
-#ALERT_NODE_MEMORY_USAGE_ENABLED=true
-#ALERT_RESTIC_CHECK_FAILED_ENABLED=true
-#ALERT_RESTIC_OUTDATED_BACKUP_ENABLED=true
+## ALerts
+
+# Node disk space alert will trigger when free disk space left is below the given number in percent
+#ALERT_NODE_DISK_SPACE_LEFT=10
+
+# Node memory usage alert will trigger when memory usage is above the given number in percent
+#ALERT_NODE_MEMORY_USAGE=85

README.md

@@ -156,13 +156,9 @@ GF_MATRIX_HOME_SERVER_URL=
 ```
 4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`
 
-## alerts
-
-It is possible to enable the following alerts, by setting the corresponding env variable to `true`:
-- backupbot failed: `ALERT_BACKUP_FAILED_ENABLED`
-- backupbot missing: `ALERT_BACKUP_MISSING_ENABLED`
-- backupbot not successfull: `ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED`
-- node disk space: `ALERT_NODE_DISK_SPACE_ENABLED`
-- node memory usage: `ALERT_NODE_MEMORY_USAGE_ENABLED`
+## Alerts
 
+It is possible to enable the following alerts, by uncommenting the corresponding env variable:
 
+- node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
+- node memory usage: `ALERT_NODE_MEMORY_USAGE`

abra.sh

@@ -9,15 +9,14 @@ export GRAFANA_CUSTOM_INI_VERSION=v4
 export PROMTAIL_YML_VERSION=v3
 export LOKI_YML_VERSION=v2
 export PROMETHEUS_YML_VERSION=v2
-export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
-export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
-export GRAFANA_ALERTS_NODE_VERSION=v1c
-export GRAFANA_ALERTS_RESTIC_VERSION=v2
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
+export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
+export GRAFANA_ALERTS_NODE_VERSION=v2
 
 # creates a default prometheus scrape config for a given node
 add_node(){
   name=$1
-  add_domain "$name" "$name:8082"
+  add_domain "$name" "metrics.traefik.$name"
   add_domain "$name" "node.monitoring.$name"
   add_domain "$name" "cadvisor.monitoring.$name"
   cat "/prometheus/scrape_configs/$name.yml"

alerts/node.yml.tmpl

@@ -2,13 +2,13 @@ apiVersion: 1
 
 # List of alert rule UIDs that should be deleted
 deleteRules:
-  {{ if ne (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
+  {{ if not (env "ALERT_NODE_DISK_SPACE_LEFT") }}
   - orgId: 1
-    uid: bds8bhxu97pxca
+    uid: coopcloud_node_disk_space_left
   {{ end }}
-  {{ if ne (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
+  {{ if not (env "ALERT_NODE_MEMORY_USAGE") }}
   - orgId: 1
-    uid: ads8cswmly96oa
+    uid: coopcloud_node_memory_usage
   {{ end }}
 
 groups:
@@ -17,8 +17,8 @@ groups:
   folder: node
   interval: 5m
   rules:
-    {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-    - uid: bds8bhxu97pxca
+    {{ if (env "ALERT_NODE_DISK_SPACE_LEFT") }}
+    - uid: coopcloud_node_disk_space_left
       title: Node Disk Space
       condition: C
       data:
@@ -45,7 +45,7 @@ groups:
             conditions:
                 - evaluator:
                     params:
-                        - 10
+                        - {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}
                     type: lt
                   operator:
                     type: and
@@ -70,13 +70,13 @@ groups:
       annotations:
         description: ""
         runbook_url: ""
-        summary: Less than 10% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
+        summary: Less than {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
       labels:
         "": ""
       isPaused: false
     {{ end }}
-    {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-    - uid: ads8cswmly96oa
+    {{ if (env "ALERT_NODE_MEMORY_USAGE")  }}
+    - uid: coopcloud_node_memory_usage
       title: Node Memory Usage
       condition: C
       data:
@@ -103,7 +103,7 @@ groups:
             conditions:
                 - evaluator:
                     params:
-                        - 85
+                        - {{ env "ALERT_NODE_MEMORY_USAGE"  }}
                     type: gt
                   operator:
                     type: and
@@ -126,6 +126,6 @@ groups:
       execErrState: Error
       for: 5m
       annotations:
-        summary: Memory usage is above 85% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
+        summary: Memory usage is above {{ env "ALERT_NODE_MEMORY_USAGE"  }}% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
       isPaused: false
     {{ end }}

alerts/restic.yml.tmpl

@@ -1,191 +0,0 @@
-apiVersion: 1
-
-deleteRules:
-  {{ if ne (env "ALERT_RESTIC_CHECK_FAILED_ENABLED") "true" }}
-  - orgId: 1
-    uid: ffglj6egxy8e8c
-  {{ end }}
-  {{ if ne (env "ALERT_RESTIC_OUTDATED_BACKUP_ENABLED") "true" }}
-  - orgId: 1
-    uid: ffgljntkp9ce8b
-  {{ end }}
-
-groups:
-    - orgId: 1
-      name: restic
-      folder: restic
-      interval: 5m
-      rules:
-      {{ if eq (env "ALERT_RESTIC_CHECK_FAILED_ENABLED") "true" }}
-        - uid: ffglj6egxy8e8c
-          title: Restic Check Failed
-          condition: C
-          data:
-            - refId: A
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: prometheus
-              model:
-                disableTextWrap: false
-                editorMode: builder
-                expr: restic_check_success
-                fullMetaSearch: false
-                includeNullMetadata: true
-                instant: true
-                intervalMs: 1000
-                legendFormat: __auto
-                maxDataPoints: 43200
-                range: false
-                refId: A
-                useBackend: false
-            - refId: B
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: __expr__
-              model:
-                conditions:
-                    - evaluator:
-                        params: []
-                        type: gt
-                      operator:
-                        type: and
-                      query:
-                        params:
-                            - B
-                      reducer:
-                        params: []
-                        type: last
-                      type: query
-                datasource:
-                    type: __expr__
-                    uid: __expr__
-                expression: A
-                intervalMs: 1000
-                maxDataPoints: 43200
-                reducer: last
-                refId: B
-                type: reduce
-            - refId: C
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: __expr__
-              model:
-                conditions:
-                    - evaluator:
-                        params:
-                            - 1
-                            - 0
-                        type: lt
-                      operator:
-                        type: and
-                      query:
-                        params:
-                            - C
-                      reducer:
-                        params: []
-                        type: last
-                      type: query
-                datasource:
-                    type: __expr__
-                    uid: __expr__
-                expression: B
-                intervalMs: 1000
-                maxDataPoints: 43200
-                refId: C
-                type: threshold
-          noDataState: Alerting
-          execErrState: Error
-          for: 5m
-          annotations: {}
-          labels: {}
-          isPaused: false
-        {{ end }}
-        {{ if eq (env "ALERT_RESTIC_OUTDATED_BACKUP_ENABLED") "true" }}
-        - uid: ffgljntkp9ce8b
-          title: Restic Outdated Backup
-          condition: C
-          data:
-            - refId: A
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: prometheus
-              model:
-                disableTextWrap: false
-                editorMode: builder
-                expr: time() - max by(instance) (restic_backup_timestamp)
-                fullMetaSearch: false
-                includeNullMetadata: true
-                instant: true
-                intervalMs: 1000
-                legendFormat: __auto
-                maxDataPoints: 43200
-                range: false
-                refId: A
-                useBackend: false
-            - refId: B
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: __expr__
-              model:
-                conditions:
-                    - evaluator:
-                        params: []
-                        type: gt
-                      operator:
-                        type: and
-                      query:
-                        params:
-                            - B
-                      reducer:
-                        params: []
-                        type: last
-                      type: query
-                datasource:
-                    type: __expr__
-                    uid: __expr__
-                expression: A
-                intervalMs: 1000
-                maxDataPoints: 43200
-                reducer: last
-                refId: B
-                type: reduce
-            - refId: C
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: __expr__
-              model:
-                conditions:
-                    - evaluator:
-                        params:
-                            - 93600
-                        type: gt
-                      operator:
-                        type: and
-                      query:
-                        params:
-                            - C
-                      reducer:
-                        params: []
-                        type: last
-                      type: query
-                datasource:
-                    type: __expr__
-                    uid: __expr__
-                expression: B
-                intervalMs: 1000
-                maxDataPoints: 43200
-                refId: C
-                type: threshold
-          noDataState: NoData
-          execErrState: Error
-          for: 5m
-          annotations: {}
-          labels: {}
-          isPaused: false
-        {{ end }}

compose.basic-auth.yml

@@ -0,0 +1,7 @@
+---
+version: "3.8"
+
+secrets:
+  basic_auth:
+    external: true
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.grafana.yml

@@ -24,8 +24,6 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
-      - source: gf_alerts_restic
-        target: /etc/grafana/provisioning/alerting/restic.yml
     networks:
       - proxy
       - internal
@@ -34,17 +32,14 @@ services:
       - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/grafana_admin_password
       - GF_SECURITY_ALLOW_EMBEDDING
       - GF_INSTALL_PLUGINS
-      - ALERT_NODE_DISK_SPACE_ENABLED
-      - ALERT_NODE_MEMORY_USAGE_ENABLED
-      - ALERT_RESTIC_CHECK_FAILED_ENABLED
-      - ALERT_RESTIC_OUTDATED_BACKUP_ENABLED
-      - DOMAIN
+      - ALERT_NODE_DISK_SPACE_LEFT
+      - ALERT_NODE_MEMORY_USAGE
     deploy:
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
-        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
@@ -82,10 +77,6 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GRAFANA_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
-  gf_alerts_restic:
-    template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_restiv_${GRAFANA_ALERTS_RESTIC_VERSION}
-    file: alerts/restic.yml.tmpl
 
 volumes:
   grafana-data:

compose.prometheus.yml

@@ -39,8 +39,3 @@ configs:
 
 volumes:
   prometheus-data:
-
-secrets:
-  basic_auth:
-    external: true
-    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.promtail.yml

@@ -23,8 +23,3 @@ configs:
     name: ${STACK_NAME}_promtail_yml_${PROMTAIL_YML_VERSION}
     file: promtail.yml.tmpl
     template_driver: golang
-
-secrets:
-  basic_auth:
-    external: true
-    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

grafana-dashboards.yml

@@ -11,3 +11,13 @@ providers:
     options:
       path: /var/lib/grafana/dashboards
       foldersFromFilesStructure: true
+  - name: 'default-alert-provider'
+    orgId: 1
+    folder: 'default-alerts'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/alerts
+      foldersFromFilesStructure: true

grafana-datasources.yml

@@ -3,7 +3,6 @@ apiVersion: 1
 datasources:
   - name: Prometheus
     type: prometheus
-    uid: prometheus
     access: proxy
     orgId: 1
     url: http://prometheus:9090
@@ -11,7 +10,6 @@ datasources:
     editable: false
   - name: Loki
     type: loki
-    uid: loki
     access: proxy
     orgId: 1
     url: http://loki:3100

release/next

@@ -5,3 +5,5 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
 2. SMTP was moved into a seperate compose file. If you have smtp configured you need to add the following line to you .env file:
 
 COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"
+
+3. The scape-config.example.yml file and add_node() command were updated to use a secure endpoint for the traefik metrics instead http. This requires an updated Traefik recipe that publishes the metrics on https.

scrape-config.example.yml

@@ -1,4 +1,4 @@
 - targets:
-  - 'example.org:8082'
+  - 'metrics.traefik.example.org'
   - 'node.monitoring.example.org'
   - 'cadvisor.monitoring.example.org'