diff --git a/.env.sample b/.env.sample index 6da93e3..5a1aa9b 100644 --- a/.env.sample +++ b/.env.sample @@ -133,3 +133,6 @@ SECRET_BASIC_AUTH_VERSION=v1 # Node memory usage alert will trigger when memory usage is above the given number in percent #ALERT_NODE_MEMORY_USAGE=85 + +# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored +#ALERT_TRAEFIK_APDEX=0.5 diff --git a/README.md b/README.md index 93d9b97..902a131 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ It is possible to enable the following alerts, by uncommenting the corresponding - node disk space: `ALERT_NODE_DISK_SPACE_LEFT` - node memory usage: `ALERT_NODE_MEMORY_USAGE` +- traefik apdex: `ALERT_TRAEFIK_APDEX` ## smart monitoring diff --git a/abra.sh b/abra.sh index 10aaa5c..87eab31 100644 --- a/abra.sh +++ b/abra.sh @@ -10,6 +10,7 @@ export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 export GF_ALERTS_NODE_VERSION=v3 +export GF_ALERTS_TRAEFIK_VERSION=v1 export CONFIG_ALLOY_VERSION=v3 # migrates secrets from old names to new names by reading values from the diff --git a/alerts/traefik.yml.tmpl b/alerts/traefik.yml.tmpl new file mode 100644 index 0000000..546da09 --- /dev/null +++ b/alerts/traefik.yml.tmpl @@ -0,0 +1,81 @@ +apiVersion: 1 + +# List of alert rule UIDs that should be deleted +deleteRules: + {{ if not (env "ALERT_TRAEFIK_APDEX") }} + - orgId: 1 + uid: coopcloud_traefik_apdex + {{ end }} + +groups: +- orgId: 1 + name: traefik + folder: recipes + interval: 5m + rules: + {{ if (env "ALERT_TRAEFIK_APDEX") }} + - uid: coopcloud_traefik_apdex + title: Apdex + condition: B + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + # apdex = (satisfied + tolerating / 2) / total, which reduces to + # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is + # cumulative and therefore already contains the satisfied requests. + # services with less than 10 requests per 10m are ignored. + expr: |- + ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) + + sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 / + sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service)) + and on(service) + ( + sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10 + ) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_TRAEFIK_APDEX" }} + type: lt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + noDataState: OK + execErrState: Error + for: 10m + annotations: + description: "" + runbook_url: "" + summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}}) + isPaused: false + {{ end }} diff --git a/compose.grafana.yml b/compose.grafana.yml index d88c7c8..6a95197 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -24,6 +24,8 @@ services: target: /var/lib/grafana/dashboards/backup.json - source: gf_alerts_node target: /etc/grafana/provisioning/alerting/node.yml + - source: gf_alerts_traefik + target: /etc/grafana/provisioning/alerting/traefik.yml networks: - proxy - internal @@ -34,6 +36,7 @@ services: - GF_INSTALL_PLUGINS - ALERT_NODE_DISK_SPACE_LEFT - ALERT_NODE_MEMORY_USAGE + - ALERT_TRAEFIK_APDEX deploy: labels: - "traefik.enable=true" @@ -77,6 +80,10 @@ configs: template_driver: golang name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION} file: alerts/node.yml.tmpl + gf_alerts_traefik: + template_driver: golang + name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION} + file: alerts/traefik.yml.tmpl volumes: grafana-data: