From f1b88a154678b27336b4b63eecaab3024c6d5551 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Mon, 12 Oct 2026 20:27:13 +0200 Subject: [PATCH 1/2] feat(alert): adds traefik apdex alert --- .env.sample | 3 ++ README.md | 4 ++ abra.sh | 5 +++ alerts/traefik.yml.tmpl | 81 +++++++++++++++++++++++++++++++++++++++++ compose.grafana.yml | 25 +++++++++++++ 5 files changed, 118 insertions(+) create mode 100644 alerts/traefik.yml.tmpl diff --git a/.env.sample b/.env.sample index 6da93e3..5a1aa9b 100644 --- a/.env.sample +++ b/.env.sample @@ -133,3 +133,6 @@ SECRET_BASIC_AUTH_VERSION=v1 # Node memory usage alert will trigger when memory usage is above the given number in percent #ALERT_NODE_MEMORY_USAGE=85 + +# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored +#ALERT_TRAEFIK_APDEX=0.5 diff --git a/README.md b/README.md index 93d9b97..5deb15e 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ It is possible to enable the following alerts, by uncommenting the corresponding - node disk space: `ALERT_NODE_DISK_SPACE_LEFT` - node memory usage: `ALERT_NODE_MEMORY_USAGE` +<<<<<<< Updated upstream ## smart monitoring @@ -182,3 +183,6 @@ To be able monitor hard drive health data, you need to configure `collect-smartctl-json.sh` script provided here (via cronjob or as a `smartd` hook). This is a limitation on Docker Swarm, which prevents the `smartctl_exporter` from running on privileged mode. +======= +- traefik apdex: `ALERT_TRAEFIK_APDEX` +>>>>>>> Stashed changes diff --git a/abra.sh b/abra.sh index 10aaa5c..62659df 100644 --- a/abra.sh +++ b/abra.sh @@ -10,6 +10,11 @@ export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 export GF_ALERTS_NODE_VERSION=v3 +<<<<<<< Updated upstream +======= +export GF_ALERTS_LOKI_VERSION=v5 +export GF_ALERTS_TRAEFIK_VERSION=v1 +>>>>>>> Stashed changes export CONFIG_ALLOY_VERSION=v3 # migrates secrets from old names to new names by reading values from the diff --git a/alerts/traefik.yml.tmpl b/alerts/traefik.yml.tmpl new file mode 100644 index 0000000..546da09 --- /dev/null +++ b/alerts/traefik.yml.tmpl @@ -0,0 +1,81 @@ +apiVersion: 1 + +# List of alert rule UIDs that should be deleted +deleteRules: + {{ if not (env "ALERT_TRAEFIK_APDEX") }} + - orgId: 1 + uid: coopcloud_traefik_apdex + {{ end }} + +groups: +- orgId: 1 + name: traefik + folder: recipes + interval: 5m + rules: + {{ if (env "ALERT_TRAEFIK_APDEX") }} + - uid: coopcloud_traefik_apdex + title: Apdex + condition: B + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + # apdex = (satisfied + tolerating / 2) / total, which reduces to + # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is + # cumulative and therefore already contains the satisfied requests. + # services with less than 10 requests per 10m are ignored. + expr: |- + ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) + + sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 / + sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service)) + and on(service) + ( + sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10 + ) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_TRAEFIK_APDEX" }} + type: lt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + noDataState: OK + execErrState: Error + for: 10m + annotations: + description: "" + runbook_url: "" + summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}}) + isPaused: false + {{ end }} diff --git a/compose.grafana.yml b/compose.grafana.yml index d88c7c8..cb4aba0 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -24,6 +24,13 @@ services: target: /var/lib/grafana/dashboards/backup.json - source: gf_alerts_node target: /etc/grafana/provisioning/alerting/node.yml +<<<<<<< Updated upstream +======= + - source: gf_alerts_loki + target: /etc/grafana/provisioning/alerting/loki.yml + - source: gf_alerts_traefik + target: /etc/grafana/provisioning/alerting/traefik.yml +>>>>>>> Stashed changes networks: - proxy - internal @@ -34,6 +41,13 @@ services: - GF_INSTALL_PLUGINS - ALERT_NODE_DISK_SPACE_LEFT - ALERT_NODE_MEMORY_USAGE +<<<<<<< Updated upstream +======= + - ALERT_LOKI_INGESTION_STALLED + - ALERT_LOKI_WRITE_ERRORS + - ALERT_LOKI_WAL_DISK_USAGE + - ALERT_TRAEFIK_APDEX +>>>>>>> Stashed changes deploy: labels: - "traefik.enable=true" @@ -77,6 +91,17 @@ configs: template_driver: golang name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION} file: alerts/node.yml.tmpl +<<<<<<< Updated upstream +======= + gf_alerts_loki: + template_driver: golang + name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION} + file: alerts/loki.yml.tmpl + gf_alerts_traefik: + template_driver: golang + name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION} + file: alerts/traefik.yml.tmpl +>>>>>>> Stashed changes volumes: grafana-data: -- 2.54.0 From 1eb89006241dc3bc3940c6ab2b67792ef6224993 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Tue, 13 Oct 2026 09:18:08 +0200 Subject: [PATCH 2/2] resolve merge conflicts --- README.md | 5 +---- abra.sh | 4 ---- compose.grafana.yml | 18 ------------------ 3 files changed, 1 insertion(+), 26 deletions(-) diff --git a/README.md b/README.md index 5deb15e..902a131 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ It is possible to enable the following alerts, by uncommenting the corresponding - node disk space: `ALERT_NODE_DISK_SPACE_LEFT` - node memory usage: `ALERT_NODE_MEMORY_USAGE` -<<<<<<< Updated upstream +- traefik apdex: `ALERT_TRAEFIK_APDEX` ## smart monitoring @@ -183,6 +183,3 @@ To be able monitor hard drive health data, you need to configure `collect-smartctl-json.sh` script provided here (via cronjob or as a `smartd` hook). This is a limitation on Docker Swarm, which prevents the `smartctl_exporter` from running on privileged mode. -======= -- traefik apdex: `ALERT_TRAEFIK_APDEX` ->>>>>>> Stashed changes diff --git a/abra.sh b/abra.sh index 62659df..87eab31 100644 --- a/abra.sh +++ b/abra.sh @@ -10,11 +10,7 @@ export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 export GF_ALERTS_NODE_VERSION=v3 -<<<<<<< Updated upstream -======= -export GF_ALERTS_LOKI_VERSION=v5 export GF_ALERTS_TRAEFIK_VERSION=v1 ->>>>>>> Stashed changes export CONFIG_ALLOY_VERSION=v3 # migrates secrets from old names to new names by reading values from the diff --git a/compose.grafana.yml b/compose.grafana.yml index cb4aba0..6a95197 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -24,13 +24,8 @@ services: target: /var/lib/grafana/dashboards/backup.json - source: gf_alerts_node target: /etc/grafana/provisioning/alerting/node.yml -<<<<<<< Updated upstream -======= - - source: gf_alerts_loki - target: /etc/grafana/provisioning/alerting/loki.yml - source: gf_alerts_traefik target: /etc/grafana/provisioning/alerting/traefik.yml ->>>>>>> Stashed changes networks: - proxy - internal @@ -41,13 +36,7 @@ services: - GF_INSTALL_PLUGINS - ALERT_NODE_DISK_SPACE_LEFT - ALERT_NODE_MEMORY_USAGE -<<<<<<< Updated upstream -======= - - ALERT_LOKI_INGESTION_STALLED - - ALERT_LOKI_WRITE_ERRORS - - ALERT_LOKI_WAL_DISK_USAGE - ALERT_TRAEFIK_APDEX ->>>>>>> Stashed changes deploy: labels: - "traefik.enable=true" @@ -91,17 +80,10 @@ configs: template_driver: golang name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION} file: alerts/node.yml.tmpl -<<<<<<< Updated upstream -======= - gf_alerts_loki: - template_driver: golang - name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION} - file: alerts/loki.yml.tmpl gf_alerts_traefik: template_driver: golang name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION} file: alerts/traefik.yml.tmpl ->>>>>>> Stashed changes volumes: grafana-data: -- 2.54.0