From 60044473cbd88d9c81383e5f073f3bf4af392cc9 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 10 Mar 2026 17:55:43 +0000 Subject: [PATCH 01/13] Add renovate.json --- renovate.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..5db72dd --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +} From e5028c8648c9ba2c06f97240d69df514f9010faf Mon Sep 17 00:00:00 2001 From: f Date: Mon, 5 Oct 2026 12:46:00 -0300 Subject: [PATCH 02/13] fix: monitoring-ng --- MAINTENANCE.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 256f105..19a0c82 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -1,4 +1,4 @@ -# Traefik Recipe Maintenance +# Monitoring NG Recipe Maintenance All contributions should be made via a pull request. This is to ensure a certain quality and consistency, that others can rely on. @@ -28,5 +28,8 @@ is always better if all maintainers looked at the pull request and approved it. Everyone can apply to be a recipe maintainer: 1. Watch the repository to always get updates 2. Simply add your self to the list in the [README.md](./README.md) and open a new pull request with the change. -3. Once the pull request gets merged you will be added to the [traefik maintainers team](https://git.coopcloud.tech/org/coop-cloud/teams/traefik-maintainers). -4. Join the room [#cc-|-traefik-maintenance:matrix.org](#cc-|-traefik-maintenance:matrix.org) and chat to other maintainers. +3. Once the pull request gets merged you will be added to the + [monitoring-ng maintainers + team](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers). +4. Join the room + [#cc-|-monitoring-ng-maintenance:matrix.org](#cc-|-monitoring-ng-maintenance:matrix.org) and chat to other maintainers. From 1babe0c7e8a12503286698ab4997d80157d989e3 Mon Sep 17 00:00:00 2001 From: f Date: Sun, 11 Oct 2026 23:38:27 +0000 Subject: [PATCH 03/13] feat: smart monitoring (#25) it requires a helper script on the host because swarm doesn't support privileged mode Reviewed-on: https://git.coopcloud.tech/coop-cloud/monitoring-ng/pulls/25 Reviewed-by: Danny Groenewegen <247+dannygroenewegen@noreply.git.coopcloud.tech> Co-authored-by: f --- .env.sample | 3 ++ README.md | 8 +++++ collect-smartctl-json.service | 6 ++++ collect-smartctl-json.sh | 61 +++++++++++++++++++++++++++++++++++ collect-smartctl-json.timer | 9 ++++++ compose.smartctl.yml | 17 ++++++++++ 6 files changed, 104 insertions(+) create mode 100644 collect-smartctl-json.service create mode 100755 collect-smartctl-json.sh create mode 100644 collect-smartctl-json.timer create mode 100644 compose.smartctl.yml diff --git a/.env.sample b/.env.sample index b97f05b..6da93e3 100644 --- a/.env.sample +++ b/.env.sample @@ -58,6 +58,9 @@ SECRET_BASIC_AUTH_VERSION=v1 # COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml" +# Monitor physical disks health +# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml" + # Monitoring Server # ## Prometheus diff --git a/README.md b/README.md index 78f8cab..93d9b97 100644 --- a/README.md +++ b/README.md @@ -174,3 +174,11 @@ It is possible to enable the following alerts, by uncommenting the corresponding - node disk space: `ALERT_NODE_DISK_SPACE_LEFT` - node memory usage: `ALERT_NODE_MEMORY_USAGE` + +## smart monitoring + +To be able monitor hard drive health data, you need to configure +`smartd` to run on the host system, and also the +`collect-smartctl-json.sh` script provided here (via cronjob or as +a `smartd` hook). This is a limitation on Docker Swarm, which prevents +the `smartctl_exporter` from running on privileged mode. diff --git a/collect-smartctl-json.service b/collect-smartctl-json.service new file mode 100644 index 0000000..b4b8e7c --- /dev/null +++ b/collect-smartctl-json.service @@ -0,0 +1,6 @@ +[Unit] +Description=Collect SMART data + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/collect-smartctl-json.sh diff --git a/collect-smartctl-json.sh b/collect-smartctl-json.sh new file mode 100755 index 0000000..9044bd7 --- /dev/null +++ b/collect-smartctl-json.sh @@ -0,0 +1,61 @@ +#! /bin/bash +# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh + +# Data directory to dump smartctl output +# This directory will be created if it doesn't exist +data_dir="/var/lib/smartmontools/json" + +# The original script used --xall but that doesn't work +# This matches the command in readSMARTctl() +smartctl_args="--json --info --health --attributes --tolerance=verypermissive \ +--nocheck=standby --format=brief --log=error" + +# Ignore this devices +smartctl_ignore_dev_regex="^(/dev/bus)" + +# Determine the json query tool to use +if command -v jq >/dev/null; then + json_tool="jq" + json_args="--raw-output" +elif command -v yq >/dev/null; then + json_tool="yq" + json_args="--unwrapScalar" +else + echo -e "One of 'yq' or 'jq' is required. Please try again after \ +installing one of them" + exit 1 +fi + +if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then + # Not root and sudo doesn't exist + echo "sudo does not exist. Please run this as root" + exit 1 +fi + +SUDO="sudo" +if [[ "${UID}" -eq 0 ]]; then + # Don't use sudo if root + SUDO="" +fi + +[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}" + +if [[ $# -ne 0 ]]; then + devices="${1}" +else + devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \ +".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")" + mapfile -t devices <<< "${devices[@]}" +fi + +for device in "${devices[@]}" + do + echo -n "Collecting data for '${device}'..." + # shellcheck disable=SC2086 + data="$($SUDO smartctl ${smartctl_args} ${device})" + # Accommodate a smartmontools pre-7.3 bug + data=${data#" Pending defect count:"} + device_name="$(basename "${device}")" + echo -e "\tSaving to ${device_name}.json" + echo "${data}" > "${data_dir}/${device_name}.json" +done diff --git a/collect-smartctl-json.timer b/collect-smartctl-json.timer new file mode 100644 index 0000000..a56e8b3 --- /dev/null +++ b/collect-smartctl-json.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Collect SMART data + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/compose.smartctl.yml b/compose.smartctl.yml new file mode 100644 index 0000000..1291dc8 --- /dev/null +++ b/compose.smartctl.yml @@ -0,0 +1,17 @@ +--- +version: "3.8" +services: + smartctl: + image: "prometheuscommunity/smartctl-exporter:v0.14.0" + volumes: + - "/var/lib/smartmontools/json:/debug" + command: + - "--smartctl.fake-data" + - "--smartctl.interval=1h" + networks: + - "proxy" + deploy: + labels: + - "prometheus.io/scrape=true" + - "prometheus.io/port=9633" + - "prometheus.io/path=/metrics" From 5028630b04fccf9bf1231f46a793bb794616e8af Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 12 Oct 2026 01:16:59 +0000 Subject: [PATCH 04/13] chore(deps): update grafana/alloy docker tag to v1.20.1 --- compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index b4b6a8b..5d42783 100644 --- a/compose.yml +++ b/compose.yml @@ -3,7 +3,7 @@ version: "3.8" services: app: - image: grafana/alloy:v1.18.1 + image: grafana/alloy:v1.20.1 hostname: "${DOMAIN}" configs: - source: config_alloy From f980185f8f228558cfc5b231f6416f1b274ad465 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 12 Oct 2026 03:16:46 +0000 Subject: [PATCH 05/13] chore(deps): update prom/prometheus docker tag to v3.15.0 --- compose.prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.prometheus.yml b/compose.prometheus.yml index c28ff1d..0682c45 100644 --- a/compose.prometheus.yml +++ b/compose.prometheus.yml @@ -2,7 +2,7 @@ version: '3.8' services: prometheus: - image: prom/prometheus:v3.12.0 + image: prom/prometheus:v3.15.0 secrets: - basic_auth volumes: From 0a749fe4fe21656f44ef5665f51a93fd37af42c3 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 12 Oct 2026 03:16:50 +0000 Subject: [PATCH 06/13] chore(deps): update prometheuscommunity/smartctl-exporter docker tag to v0.15.0 --- compose.smartctl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.smartctl.yml b/compose.smartctl.yml index 1291dc8..32123bd 100644 --- a/compose.smartctl.yml +++ b/compose.smartctl.yml @@ -2,7 +2,7 @@ version: "3.8" services: smartctl: - image: "prometheuscommunity/smartctl-exporter:v0.14.0" + image: "prometheuscommunity/smartctl-exporter:v0.15.0" volumes: - "/var/lib/smartmontools/json:/debug" command: From 01111b992d07874713f0db9a8f0ee77da57fbf89 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Mon, 12 Oct 2026 13:02:52 +0200 Subject: [PATCH 07/13] fix(alert): invert memory usage alert --- abra.sh | 2 +- alerts/node.yml.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/abra.sh b/abra.sh index 10aaa5c..2bf76f2 100644 --- a/abra.sh +++ b/abra.sh @@ -9,7 +9,7 @@ export LOKI_YML_VERSION=v3 export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 -export GF_ALERTS_NODE_VERSION=v3 +export GF_ALERTS_NODE_VERSION=v4 export CONFIG_ALLOY_VERSION=v3 # migrates secrets from old names to new names by reading values from the diff --git a/alerts/node.yml.tmpl b/alerts/node.yml.tmpl index 244db8c..2f8cf74 100644 --- a/alerts/node.yml.tmpl +++ b/alerts/node.yml.tmpl @@ -87,7 +87,7 @@ groups: datasourceUid: PBFA97CFB590B2093 model: editorMode: code - expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 + expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) instant: true intervalMs: 1000 legendFormat: __auto From 24927d6331a8185523014fe0fbb9756fb23c7fac Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 12 Oct 2026 02:16:16 +0000 Subject: [PATCH 08/13] chore(deps): update grafana/grafana docker tag to v13.2.3 --- compose.grafana.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.grafana.yml b/compose.grafana.yml index 5120196..d88c7c8 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -2,7 +2,7 @@ version: '3.8' services: grafana: - image: grafana/grafana:13.0.6 + image: grafana/grafana:13.2.3 volumes: - grafana-data:/var/lib/grafana:rw secrets: From 48e8318e416d3cbbf5211ddce6354151927c95e3 Mon Sep 17 00:00:00 2001 From: f Date: Mon, 12 Oct 2026 13:11:16 -0300 Subject: [PATCH 09/13] chore: publish 2.3.0+v1.20.1 release --- compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index 5d42783..472d49b 100644 --- a/compose.yml +++ b/compose.yml @@ -53,7 +53,7 @@ services: condition: on-failure labels: - "backupbot.backup=${ENABLE_BACKUPS:-true}" - - "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1" + - "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1" configs: config_alloy: template_driver: golang From ace2d6d76df3a701bc2ff1b41d8610165b64ccbb Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 21 Oct 2026 14:19:00 +0000 Subject: [PATCH 10/13] chore(deps): update prom/prometheus docker tag to v3.15.1 --- compose.prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.prometheus.yml b/compose.prometheus.yml index 0682c45..741cff6 100644 --- a/compose.prometheus.yml +++ b/compose.prometheus.yml @@ -2,7 +2,7 @@ version: '3.8' services: prometheus: - image: prom/prometheus:v3.15.0 + image: prom/prometheus:v3.15.1 secrets: - basic_auth volumes: From ebcb12163fb5203c17efdd2ebb1a3652149a4b74 Mon Sep 17 00:00:00 2001 From: Ammar Hussein Date: Fri, 23 Oct 2026 23:17:11 -0700 Subject: [PATCH 11/13] alloy and prometheus restart-policy update --- .gitignore | 1 + compose.prometheus.yml | 2 +- compose.yml | 2 +- release/next | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 release/next diff --git a/.gitignore b/.gitignore index 37b52cc..ec1b785 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /.envrc +*~ \ No newline at end of file diff --git a/compose.prometheus.yml b/compose.prometheus.yml index 0682c45..28e95b6 100644 --- a/compose.prometheus.yml +++ b/compose.prometheus.yml @@ -28,7 +28,7 @@ services: start_period: 30s deploy: restart_policy: - condition: on-failure + condition: unless-stopped labels: - "traefik.enable=true" - "traefik.swarm.network=proxy" diff --git a/compose.yml b/compose.yml index 472d49b..0cd3382 100644 --- a/compose.yml +++ b/compose.yml @@ -50,7 +50,7 @@ services: start_period: 30s deploy: restart_policy: - condition: on-failure + condition: unless-stopped labels: - "backupbot.backup=${ENABLE_BACKUPS:-true}" - "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1" diff --git a/release/next b/release/next new file mode 100644 index 0000000..cdd836d --- /dev/null +++ b/release/next @@ -0,0 +1 @@ +- alloy and prometheus containers restart policy from on-failure to unless-stopped \ No newline at end of file From 8c7afe5d8f19a3cd60a0de0c4bdb572a09c94dce Mon Sep 17 00:00:00 2001 From: Ammar Hussein Date: Sat, 24 Oct 2026 16:57:09 -0700 Subject: [PATCH 12/13] increase grafana healthcheck start_period and reduce error logs --- abra.sh | 2 +- alerts/node.yml.tmpl | 3 +-- compose.grafana.yml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/abra.sh b/abra.sh index 2bf76f2..858e7a7 100644 --- a/abra.sh +++ b/abra.sh @@ -9,7 +9,7 @@ export LOKI_YML_VERSION=v3 export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 -export GF_ALERTS_NODE_VERSION=v4 +export GF_ALERTS_NODE_VERSION=v5 export CONFIG_ALLOY_VERSION=v3 # migrates secrets from old names to new names by reading values from the diff --git a/alerts/node.yml.tmpl b/alerts/node.yml.tmpl index 2f8cf74..6eff7c9 100644 --- a/alerts/node.yml.tmpl +++ b/alerts/node.yml.tmpl @@ -71,8 +71,7 @@ groups: description: "" runbook_url: "" summary: Less than {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left) - labels: - "": "" + labels: {} isPaused: false {{ end }} {{ if (env "ALERT_NODE_MEMORY_USAGE") }} diff --git a/compose.grafana.yml b/compose.grafana.yml index d88c7c8..62833e3 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -48,7 +48,7 @@ services: interval: 5s timeout: 10s retries: 3 - start_period: 10s + start_period: 300s configs: gf_custom_ini: From 25f811bd4fd93783b332e1a42d25665d4e907e85 Mon Sep 17 00:00:00 2001 From: p4u1 Date: Fri, 30 Oct 2026 13:48:38 +0000 Subject: [PATCH 13/13] feat(alert): adds traefik apdex alert (#49) Reviewed-on: https://git.coopcloud.tech/coop-cloud/monitoring-ng/pulls/49 Reviewed-by: ammaratef45 Reviewed-by: fauno Co-authored-by: p4u1 --- .env.sample | 3 ++ README.md | 1 + alerts/traefik.yml.tmpl | 81 +++++++++++++++++++++++++++++++++++++++++ compose.grafana.yml | 7 ++++ 4 files changed, 92 insertions(+) create mode 100644 alerts/traefik.yml.tmpl diff --git a/.env.sample b/.env.sample index 6da93e3..5a1aa9b 100644 --- a/.env.sample +++ b/.env.sample @@ -133,3 +133,6 @@ SECRET_BASIC_AUTH_VERSION=v1 # Node memory usage alert will trigger when memory usage is above the given number in percent #ALERT_NODE_MEMORY_USAGE=85 + +# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored +#ALERT_TRAEFIK_APDEX=0.5 diff --git a/README.md b/README.md index 93d9b97..902a131 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ It is possible to enable the following alerts, by uncommenting the corresponding - node disk space: `ALERT_NODE_DISK_SPACE_LEFT` - node memory usage: `ALERT_NODE_MEMORY_USAGE` +- traefik apdex: `ALERT_TRAEFIK_APDEX` ## smart monitoring diff --git a/alerts/traefik.yml.tmpl b/alerts/traefik.yml.tmpl new file mode 100644 index 0000000..546da09 --- /dev/null +++ b/alerts/traefik.yml.tmpl @@ -0,0 +1,81 @@ +apiVersion: 1 + +# List of alert rule UIDs that should be deleted +deleteRules: + {{ if not (env "ALERT_TRAEFIK_APDEX") }} + - orgId: 1 + uid: coopcloud_traefik_apdex + {{ end }} + +groups: +- orgId: 1 + name: traefik + folder: recipes + interval: 5m + rules: + {{ if (env "ALERT_TRAEFIK_APDEX") }} + - uid: coopcloud_traefik_apdex + title: Apdex + condition: B + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + # apdex = (satisfied + tolerating / 2) / total, which reduces to + # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is + # cumulative and therefore already contains the satisfied requests. + # services with less than 10 requests per 10m are ignored. + expr: |- + ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) + + sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 / + sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service)) + and on(service) + ( + sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10 + ) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_TRAEFIK_APDEX" }} + type: lt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + noDataState: OK + execErrState: Error + for: 10m + annotations: + description: "" + runbook_url: "" + summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}}) + isPaused: false + {{ end }} diff --git a/compose.grafana.yml b/compose.grafana.yml index 62833e3..438f3eb 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -24,6 +24,8 @@ services: target: /var/lib/grafana/dashboards/backup.json - source: gf_alerts_node target: /etc/grafana/provisioning/alerting/node.yml + - source: gf_alerts_traefik + target: /etc/grafana/provisioning/alerting/traefik.yml networks: - proxy - internal @@ -34,6 +36,7 @@ services: - GF_INSTALL_PLUGINS - ALERT_NODE_DISK_SPACE_LEFT - ALERT_NODE_MEMORY_USAGE + - ALERT_TRAEFIK_APDEX deploy: labels: - "traefik.enable=true" @@ -77,6 +80,10 @@ configs: template_driver: golang name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION} file: alerts/node.yml.tmpl + gf_alerts_traefik: + template_driver: golang + name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION} + file: alerts/traefik.yml.tmpl volumes: grafana-data: