From 536fa7f37d756873c2e4093e4f9b07499e7610dc Mon Sep 17 00:00:00 2001 From: Danny Groenewegen Date: Mon, 17 Aug 2026 13:30:49 +0200 Subject: [PATCH] fix Traefik dashboard --- abra.sh | 2 +- config.alloy.tmpl | 50 ++++++++++++++++++++++ grafana-traefik-dashboard.json | 76 ++++++++++------------------------ 3 files changed, 74 insertions(+), 54 deletions(-) diff --git a/abra.sh b/abra.sh index a152097..dbcdb5b 100644 --- a/abra.sh +++ b/abra.sh @@ -2,7 +2,7 @@ export GF_DATASOURCES_VERSION=v1 export GF_DASHBOARDS_VERSION=v2 export GF_SWARM_DASH_VERSION=v2 export GF_STACKS_DASH_VERSION=v2 -export GF_TRAEFIK_DASH_VERSION=v2 +export GF_TRAEFIK_DASH_VERSION=v3 export GF_BACKUP_DASH_VERSION=v1 export GF_CUSTOM_INI_VERSION=v4 export LOKI_YML_VERSION=v3 diff --git a/config.alloy.tmpl b/config.alloy.tmpl index ee00346..1b730f2 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -140,6 +140,56 @@ discovery.relabel "metrics" { separator = "." target_label = "instance" } + + rule { + source_labels = ["__meta_dockerswarm_task_slot"] + target_label = "task_slot" + } + + // coop-cloud's STACK_NAME is the domain with "." replaced by "_" + // Derive a readable dotted domain label from it. RE2 has no global + // replace, so this is done by chaining multiple replacements. Each + // rule swaps the first remaining _ for a "." until none are left. + rule { + source_labels = ["__meta_dockerswarm_service_label_com_docker_stack_namespace"] + target_label = "domain" + } + rule { + source_labels = ["domain"] + regex = `([^_]*)_(.*)` + target_label = "domain" + replacement = "$1.$2" + } + rule { + source_labels = ["domain"] + regex = `([^_]*)_(.*)` + target_label = "domain" + replacement = "$1.$2" + } + rule { + source_labels = ["domain"] + regex = `([^_]*)_(.*)` + target_label = "domain" + replacement = "$1.$2" + } + rule { + source_labels = ["domain"] + regex = `([^_]*)_(.*)` + target_label = "domain" + replacement = "$1.$2" + } + rule { + source_labels = ["domain"] + regex = `([^_]*)_(.*)` + target_label = "domain" + replacement = "$1.$2" + } + rule { + source_labels = ["domain"] + regex = `([^_]*)_(.*)` + target_label = "domain" + replacement = "$1.$2" + } } discovery.relabel "metrics_noauth" { diff --git a/grafana-traefik-dashboard.json b/grafana-traefik-dashboard.json index 72a35f4..b207987 100644 --- a/grafana-traefik-dashboard.json +++ b/grafana-traefik-dashboard.json @@ -116,12 +116,12 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": false, - "expr": "time() - process_start_time_seconds{job=\"$job\"}", + "expr": "time() - label_replace(process_start_time_seconds{instance=~\".*$domain.*\"} or process_start_time_seconds{domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 2, - "legendFormat": "{{ instance }}", + "legendFormat": "{{ domain }}", "refId": "A" } ], @@ -200,7 +200,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(increase(traefik_service_requests_total{code=\"499\", instance=\"$instance\"}[$interval]))", + "expr": "sum(label_replace(increase(traefik_service_requests_total{code=\"499\", instance=~\".*$domain.*\"}[$interval]) or increase(traefik_service_requests_total{code=\"499\", domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "interval": "", "legendFormat": "", "refId": "A" @@ -280,7 +280,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(increase(traefik_service_requests_total{code=\"404\",protocol=~\"$protocol\", instance=\"$instance\"}[$interval]))", + "expr": "sum(label_replace(increase(traefik_service_requests_total{code=\"404\",protocol=~\"$protocol\", instance=~\".*$domain.*\"}[$interval]) or increase(traefik_service_requests_total{code=\"404\",protocol=~\"$protocol\", domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -365,7 +365,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(increase(traefik_service_requests_total{code=\"200\",protocol=~\"$protocol\", instance=\"$instance\"}[$interval]))", + "expr": "sum(label_replace(increase(traefik_service_requests_total{code=\"200\",protocol=~\"$protocol\", instance=~\".*$domain.*\"}[$interval]) or increase(traefik_service_requests_total{code=\"200\",protocol=~\"$protocol\", domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -417,7 +417,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "topk(5, sum(traefik_service_requests_total{protocol=~\"$protocol\", instance=\"$instance\"}) by (code))", + "expr": "topk(5, sum(label_replace(traefik_service_requests_total{protocol=~\"$protocol\", instance=~\".*$domain.*\"} or traefik_service_requests_total{protocol=~\"$protocol\", domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by (code))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -489,7 +489,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(increase(traefik_service_requests_total{code=\"499\",method=\"GET\", instance=\"$instance\"}[$interval])) by (service)", + "expr": "sum(label_replace(increase(traefik_service_requests_total{code=\"499\",method=\"GET\", instance=~\".*$domain.*\"}[$interval]) or increase(traefik_service_requests_total{code=\"499\",method=\"GET\", domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by (service)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -590,7 +590,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "(sum(traefik_entrypoint_request_duration_seconds_sum{protocol=~\"$protocol\", instance=\"$instance\"}) / sum(traefik_entrypoint_requests_total{protocol=~\"$protocol\", instance=\"$instance\"}) * 1000) - (sum(traefik_service_request_duration_seconds_sum{protocol=~\"$protocol\", instance=\"$instance\"}) / sum(traefik_service_requests_total{protocol=~\"$protocol\", instance=\"$instance\"}) * 1000)", + "expr": "(sum(label_replace(traefik_entrypoint_request_duration_seconds_sum{protocol=~\"$protocol\", instance=~\".*$domain.*\"} or traefik_entrypoint_request_duration_seconds_sum{protocol=~\"$protocol\", domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) / sum(label_replace(traefik_entrypoint_requests_total{protocol=~\"$protocol\", instance=~\".*$domain.*\"} or traefik_entrypoint_requests_total{protocol=~\"$protocol\", domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) * 1000) - (sum(label_replace(traefik_service_request_duration_seconds_sum{protocol=~\"$protocol\", instance=~\".*$domain.*\"} or traefik_service_request_duration_seconds_sum{protocol=~\"$protocol\", domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) / sum(label_replace(traefik_service_requests_total{protocol=~\"$protocol\", instance=~\".*$domain.*\"} or traefik_service_requests_total{protocol=~\"$protocol\", domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) * 1000)", "format": "time_series", "instant": false, "interval": "", @@ -709,7 +709,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(traefik_entrypoint_request_duration_seconds_sum{instance=\"$instance\"}) / sum(traefik_entrypoint_requests_total{instance=\"$instance\"}) * 1000", + "expr": "sum(label_replace(traefik_entrypoint_request_duration_seconds_sum{instance=~\".*$domain.*\"} or traefik_entrypoint_request_duration_seconds_sum{domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) / sum(label_replace(traefik_entrypoint_requests_total{instance=~\".*$domain.*\"} or traefik_entrypoint_requests_total{domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) * 1000", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -781,7 +781,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(delta(traefik_service_requests_total{instance=\"${instance:raw}\"}[$interval]))", + "expr": "sum(label_replace(delta(traefik_service_requests_total{instance=~\".*$domain.*\"}[$interval]) or delta(traefik_service_requests_total{domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -879,7 +879,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(rate(traefik_service_requests_total{instance=\"$instance\"}[$interval]))", + "expr": "sum(label_replace(rate(traefik_service_requests_total{instance=~\".*$domain.*\"}[$interval]) or rate(traefik_service_requests_total{domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -984,7 +984,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum(rate(traefik_service_request_duration_seconds_sum{ instance=\"$instance\" }[5m])) by(service)", + "expr": "sum(label_replace(rate(traefik_service_request_duration_seconds_sum{instance=~\".*$domain.*\"}[5m]) or rate(traefik_service_request_duration_seconds_sum{domain=\"$domain\"}[5m]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by(service)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1053,11 +1053,11 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "process_open_fds{job=~\"$job\", instance=\"$instance\"}", + "expr": "label_replace(process_open_fds{instance=~\".*$domain.*\"} or process_open_fds{domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)", "format": "time_series", "interval": "", "intervalFactor": 2, - "legendFormat": "{{ instance }}", + "legendFormat": "{{ domain }}", "refId": "A", "step": 240 } @@ -1154,7 +1154,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(rate(traefik_service_requests_total{protocol=~\"http|https\", instance=\"$instance\"}[$interval])) by (service)", + "expr": "sum(label_replace(rate(traefik_service_requests_total{protocol=~\"http|https\", instance=~\".*$domain.*\"}[$interval]) or rate(traefik_service_requests_total{protocol=~\"http|https\", domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by (service)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1255,7 +1255,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(traefik_entrypoint_open_connections{instance=\"$instance\"}) by (method)", + "expr": "sum(label_replace(traefik_entrypoint_open_connections{instance=~\".*$domain.*\"} or traefik_entrypoint_open_connections{domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by (method)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1355,7 +1355,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(traefik_service_open_connections{instance=\"$instance\"}) by (method)", + "expr": "sum(label_replace(traefik_service_open_connections{instance=~\".*$domain.*\"} or traefik_service_open_connections{domain=\"$domain\"}, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by (method)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1459,7 +1459,7 @@ "uid": "PBFA97CFB590B2093" }, "exemplar": true, - "expr": "sum(increase(traefik_service_requests_total{protocol=~\"$protocol\", instance=\"$instance\"}[$interval])) by (code)", + "expr": "sum(label_replace(increase(traefik_service_requests_total{protocol=~\"$protocol\", instance=~\".*$domain.*\"}[$interval]) or increase(traefik_service_requests_total{protocol=~\"$protocol\", domain=\"$domain\"}[$interval]), \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`)) by (code)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1514,36 +1514,6 @@ ], "templating": { "list": [ - { - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "", - "hide": 0, - "includeAll": false, - "label": "Job:", - "multi": false, - "name": "job", - "options": [], - "query": { - "query": "label_values(job)", - "refId": "Prometheus-job-Variable-Query" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 2, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, { "current": { "selected": true, @@ -1663,19 +1633,19 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "definition": "label_values(instance)", + "definition": "query_result(label_replace(traefik_config_reloads_total, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "hide": 0, "includeAll": false, - "label": "Instance:", + "label": "Domain:", "multi": false, - "name": "instance", + "name": "domain", "options": [], "query": { - "query": "label_values(instance)", + "query": "query_result(label_replace(traefik_config_reloads_total, \"domain\", \"$1\", \"instance\", `^metrics\\.(.+)$`))", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, - "regex": ".*8082", + "regex": "/domain=\"([^\"]+)\"/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "",