From 8900ace6a23a942c16f350a925b9939646650206 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Tue, 18 Feb 2025 16:02:49 +0100 Subject: [PATCH] feat: Adds matrix contact point for grafana alerts --- .env.sample | 7 +++ README.md | 22 +++++++ abra.sh | 2 + alertmanager-matrix-config.yml.tmpl | 74 ++++++++++++++++++++++++ compose.matrix-alertmanager-receiver.yml | 28 +++++++++ 5 files changed, 133 insertions(+) create mode 100644 alertmanager-matrix-config.yml.tmpl create mode 100644 compose.matrix-alertmanager-receiver.yml diff --git a/.env.sample b/.env.sample index 84c8551..e65805a 100644 --- a/.env.sample +++ b/.env.sample @@ -67,6 +67,13 @@ ENABLE_BACKUPS=true # SECRET_GRAFANA_SMTP_PASSWORD_VERSION=v1 # +## Grafana Matrix Contact Point (optional) +#COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml" +#SECRET_MATRIX_ACCESS_TOKEN_VERSION=v1 +#GF_MATRIX_USER_ID="" +#GF_MATRIX_ROOM_ID="" +#GF_MATRIX_HOMESERVER_URL="" + # ALerts #ALERT_BACKUP_FAILED_ENABLED=true #ALERT_BACKUP_MISSING_ENABLED=true diff --git a/README.md b/README.md index 36166ca..439b980 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,26 @@ After that you need to add the `pushgateway.${DOMAIN}` to the scare config. THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal +## Adding Matrix as Alert Contact point + +1. Enable the [matrix-alertmanager-receiver](https://github.com/metio/matrix-alertmanager-receiver/): +``` +COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml" +``` + +2. Insert the matrix access token secret: +``` +abra app secret insert monitoring.marx.klasse-methode.it matrix_access_token v1 +``` + +3. Set required configurations: +``` +GF_MATRIX_USER_ID= +GF_MATRIX_ROOM_ID= +GF_MATRIX_HOME_SERVER_URL= +``` +4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/` + ## alerts It is possible to enable the following alerts, by setting the corresponding env variable to `true`: @@ -138,3 +158,5 @@ It is possible to enable the following alerts, by setting the corresponding env - backupbot not successfull: `ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED` - node disk space: `ALERT_NODE_DISK_SPACE_ENABLED` - node memory usage: `ALERT_NODE_MEMORY_USAGE_ENABLED` + + diff --git a/abra.sh b/abra.sh index 85dc51d..fa5dcc4 100644 --- a/abra.sh +++ b/abra.sh @@ -10,6 +10,8 @@ export GRAFANA_CUSTOM_INI_VERSION=v4 export PROMTAIL_YML_VERSION=v3 export LOKI_YML_VERSION=v2 export PROMETHEUS_YML_VERSION=v2 +export MATRIX_ALERTMANAGER_CONFIG_VERSION=e +export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a # creates a default prometheus scrape config for a given node add_node(){ diff --git a/alertmanager-matrix-config.yml.tmpl b/alertmanager-matrix-config.yml.tmpl new file mode 100644 index 0000000..8e6878b --- /dev/null +++ b/alertmanager-matrix-config.yml.tmpl @@ -0,0 +1,74 @@ +# configuration of the HTTP server +http: + ## address: 127.0.0.1 # bind address for this service. Can be left unspecified to bind on all interfaces + port: 12345 # port used by this service + alerts-path-prefix: /alerts # URL path for the webhook receiver called by an Alertmanager. Defaults to /alerts + metrics-path: /metrics # URL path to collect metrics. Defaults to /metrics + metrics-enabled: true # Whether to enable metrics or not. Defaults to false + # basic-username: alertmanager # Username for basic authentication. Defaults to alertmanager + # basic-password: secret # If set, the alerts endpoint expects basic-auth credentials with the configured username and password + +# configuration for the Matrix connection +matrix: + homeserver-url: "{{ env "GF_MATRIX_HOMESERVER_URL" }}" + user-id: "{{ env "GF_MATRIX_USER_ID" }}" + access-token: "{{ secret "matrix_access_token" }}" + room-mapping: + matrixroom: "{{ env "GF_MATRIX_ROOM_ID" }}" + +templating: + # mapping of ExternalURL values + external-url-mapping: + # key is the original value taken from the Alertmanager payload + # value is the mapped value which will be available as '.ExternalURL' in templates + "http://alertmanager:9093": https://alertmanager.example.com + # mapping of GeneratorURL values + generator-url-mapping: + # key is the original value taken from the Alertmanager payload + # value is the mapped value which will be available as '.GeneratorURL' in templates + "http://prometheus:8080": https://prometheus.example.com + + # computation of arbitrary values based on matching alert annotations, labels, or status + # values will be evaluated top to bottom, last entry wins + computed-values: + - values: # always set 'color' to 'yellow' + color: yellow + - values: # set 'color' to 'orange' when alert label 'severity' is 'warning' + color: orange + when-matching-labels: + severity: warning + - values: # set 'color' to 'red' when alert label 'severity' is 'critical' + color: red + when-matching-labels: + severity: critical + - values: # set 'color' to 'green' when alert status is 'resolved' + color: green + when-matching-status: resolved + + # template for alerts in status 'firing' + firing-template: '{{` +

+ {{ .Alert.Status | ToUpper }} + {{ if .Alert.Labels.name }} + {{ .Alert.Labels.name }} + {{ else if .Alert.Labels.alertname }} + {{ .Alert.Labels.alertname }} + {{ end }} + >> + {{ if .Alert.Labels.severity }} + {{ .Alert.Labels.severity | ToUpper }}: + {{ end }} + {{ if .Alert.Annotations.description }} + {{ .Alert.Annotations.description }} + {{ else if .Alert.Annotations.summary }} + {{ .Alert.Annotations.summary }} + {{ end }} + >> + {{ if .Alert.Annotations.runbook }} + Runbook | + {{ end }} + {{ if .Alert.Annotations.dashboard }} + Dashboard | + {{ end }} + Silence +

`}}' diff --git a/compose.matrix-alertmanager-receiver.yml b/compose.matrix-alertmanager-receiver.yml new file mode 100644 index 0000000..2df1161 --- /dev/null +++ b/compose.matrix-alertmanager-receiver.yml @@ -0,0 +1,28 @@ +version: '3.8' + +services: + matrix-alertmanager-receiver: + image: metio/matrix-alertmanager-receiver:2025.2.9 + secrets: + - matrix_access_token + configs: + - source: matrix-alertmanager-receiver-config + target: /etc/matrix-alertmanager-receiver/config.yml + networks: + - internal + environment: + - GF_MATRIX_USER_ID + - GF_MATRIX_ROOM_ID + - GF_MATRIX_HOMESERVER_URL + command: "--config-path=/etc/matrix-alertmanager-receiver/config.yml" + +configs: + matrix-alertmanager-receiver-config: + template_driver: golang + name: ${STACK_NAME}_mar_config_${MATRIX_ALERTMANAGER_CONFIG_VERSION} + file: alertmanager-matrix-config.yml.tmpl + +secrets: + matrix_access_token: + external: true + name: ${STACK_NAME}_matrix_access_token_${SECRET_MATRIX_ACCESS_TOKEN_VERSION}