resolve merge conflicts

Reviewed-on: #46
Reviewed-by: fauno <fauno@sutty.coop.ar>

.env.sample

@@ -58,6 +58,9 @@ SECRET_BASIC_AUTH_VERSION=v1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
 
 
+# Monitor physical disks health
+# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml"
+
 # Monitoring Server
 #
 ## Prometheus
@@ -130,3 +133,6 @@ SECRET_BASIC_AUTH_VERSION=v1
 
 # Node memory usage alert will trigger when memory usage is above the given number in percent
 #ALERT_NODE_MEMORY_USAGE=85
+
+# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored
+#ALERT_TRAEFIK_APDEX=0.5

.gitea/CODEOWNERS

@@ -0,0 +1 @@
+* @coop-cloud/monitoring-ng-maintainers

MAINTENANCE.md

@@ -0,0 +1,35 @@
+# Monitoring NG Recipe Maintenance
+
+All contributions should be made via a pull request. This is to ensure a
+certain quality and consistency, that others can rely on.
+
+## Maintainer Responsibilities
+
+A recipe maintainer has the following responsibilities:
+
+- Respond to pull requests / issues within two weeks
+- Make image security updates within a week
+- Make image major updates every three months
+
+In order to fullfill these responsibilities a recipe maintainer:
+
+- Has to watch the repository (to get notifications)
+- Needs to make sure renovate is configured properly
+
+## Pull Requests
+
+A pull request can be merged if it is approved by at least one maintainer. For
+pull requests opened by a maintainer they need to be approved by another
+maintainer. Even though it is okay to merge a pull request with one approval, it
+is always better if all maintainers looked at the pull request and approved it.
+
+## Become a maintainer
+
+Everyone can apply to be a recipe maintainer:
+1. Watch the repository to always get updates
+2. Simply add your self to the list in the [README.md](./README.md) and open a new pull request with the change.
+3. Once the pull request gets merged you will be added to the
+   [monitoring-ng maintainers
+   team](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers).
+4. Join the room
+   [#cc-|-monitoring-ng-maintenance:matrix.org](#cc-|-monitoring-ng-maintenance:matrix.org) and chat to other maintainers.

README.md

@@ -7,7 +7,7 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou
 Metrics and logs are collected by [Grafana Alloy](https://grafana.com/docs/alloy/latest/) and pushed to a central Prometheus/Loki (via `remote_write`/`loki push`). Every `monitoring-ng` instance runs its own Alloy, whether or not it also runs the central stack.
 
 <!-- metadata -->
-
+- **Maintainer**: [@coop-cloud/monitoring-ng-maintainers](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers)
 - **Category**: Apps
 - **Status**: 2, beta
 - **Image**: [`grafana/grafana`](https://hub.docker.com/r/grafana/grafana), 4, upstream
@@ -174,3 +174,12 @@ It is possible to enable the following alerts, by uncommenting the corresponding
 
 - node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
 - node memory usage: `ALERT_NODE_MEMORY_USAGE`
+- traefik apdex: `ALERT_TRAEFIK_APDEX`
+
+## smart monitoring
+
+To be able monitor hard drive health data, you need to configure
+`smartd` to run on the host system, and also the
+`collect-smartctl-json.sh` script provided here (via cronjob or as
+a `smartd` hook).  This is a limitation on Docker Swarm, which prevents
+the `smartctl_exporter` from running on privileged mode.

abra.sh

@@ -10,6 +10,7 @@ export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
+export GF_ALERTS_TRAEFIK_VERSION=v1
 export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the

alerts/traefik.yml.tmpl

@@ -0,0 +1,81 @@
+apiVersion: 1
+
+# List of alert rule UIDs that should be deleted
+deleteRules:
+  {{ if not (env "ALERT_TRAEFIK_APDEX") }}
+  - orgId: 1
+    uid: coopcloud_traefik_apdex
+  {{ end }}
+
+groups:
+- orgId: 1
+  name: traefik
+  folder: recipes
+  interval: 5m
+  rules:
+    {{ if (env "ALERT_TRAEFIK_APDEX") }}
+    - uid: coopcloud_traefik_apdex
+      title: Apdex
+      condition: B
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            # apdex = (satisfied + tolerating / 2) / total, which reduces to
+            # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is
+            # cumulative and therefore already contains the satisfied requests.
+            # services with less than 10 requests per 10m are ignored.
+            expr: |-
+              ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) +
+               sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 /
+               sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service))
+              and on(service)
+              (
+                sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10
+              )
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: B
+          relativeTimeRange:
+            from: 600
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_TRAEFIK_APDEX" }}
+                    type: lt
+                  operator:
+                    type: and
+                  query:
+                    params: []
+                  reducer:
+                    params: []
+                    type: avg
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: B
+            type: threshold
+      noDataState: OK
+      execErrState: Error
+      for: 10m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}})
+      isPaused: false
+    {{ end }}

collect-smartctl-json.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Collect SMART data
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/collect-smartctl-json.sh

collect-smartctl-json.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh
+
+# Data directory to dump smartctl output
+# This directory will be created if it doesn't exist
+data_dir="/var/lib/smartmontools/json"
+
+# The original script used --xall but that doesn't work
+# This matches the command in readSMARTctl()
+smartctl_args="--json --info --health --attributes --tolerance=verypermissive \
+--nocheck=standby --format=brief --log=error"
+
+# Ignore this devices
+smartctl_ignore_dev_regex="^(/dev/bus)"
+
+# Determine the json query tool to use
+if command -v jq >/dev/null; then
+	json_tool="jq"
+	json_args="--raw-output"
+elif command -v yq >/dev/null; then
+	json_tool="yq"
+	json_args="--unwrapScalar"
+else
+	echo -e "One of 'yq' or 'jq' is required. Please try again after \
+installing one of them"
+	exit 1
+fi
+
+if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then
+	# Not root and sudo doesn't exist
+	echo "sudo does not exist. Please run this as root"
+	exit 1
+fi
+
+SUDO="sudo"
+if [[ "${UID}" -eq 0 ]]; then
+	# Don't use sudo if root
+	SUDO=""
+fi
+
+[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}"
+
+if [[ $# -ne 0 ]]; then
+	devices="${1}"
+else
+	devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \
+".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")"
+  mapfile -t devices <<< "${devices[@]}"
+fi
+
+for device in "${devices[@]}"
+  do
+	echo -n "Collecting data for '${device}'..."
+	# shellcheck disable=SC2086
+	data="$($SUDO smartctl ${smartctl_args} ${device})"
+	# Accommodate a smartmontools pre-7.3 bug
+	data=${data#"  Pending defect count:"}
+	device_name="$(basename "${device}")"
+	echo -e "\tSaving to ${device_name}.json"
+	echo "${data}" > "${data_dir}/${device_name}.json"
+done

collect-smartctl-json.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Collect SMART data
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

compose.grafana.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:13.0.6
+    image: grafana/grafana:13.2.3
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:
@@ -24,6 +24,8 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
+      - source: gf_alerts_traefik
+        target: /etc/grafana/provisioning/alerting/traefik.yml
     networks:
       - proxy
       - internal
@@ -34,6 +36,7 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_LEFT
       - ALERT_NODE_MEMORY_USAGE
+      - ALERT_TRAEFIK_APDEX
     deploy:
       labels:
         - "traefik.enable=true"
@@ -77,6 +80,10 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
+  gf_alerts_traefik:
+    template_driver: golang
+    name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION}
+    file: alerts/traefik.yml.tmpl
 
 volumes:
   grafana-data:

compose.prometheus.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v3.12.0
+    image: prom/prometheus:v3.15.0
     secrets:
       - basic_auth
     volumes:

compose.smartctl.yml

@@ -0,0 +1,17 @@
+---
+version: "3.8"
+services:
+  smartctl:
+    image: "prometheuscommunity/smartctl-exporter:v0.15.0"
+    volumes:
+    - "/var/lib/smartmontools/json:/debug"
+    command:
+    - "--smartctl.fake-data"
+    - "--smartctl.interval=1h"
+    networks:
+    - "proxy"
+    deploy:
+      labels:
+        - "prometheus.io/scrape=true"
+        - "prometheus.io/port=9633"
+        - "prometheus.io/path=/metrics"

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: grafana/alloy:v1.18.1
+    image: grafana/alloy:v1.20.1
     hostname: "${DOMAIN}"
     configs:
       - source: config_alloy
@@ -46,14 +46,14 @@ services:
       test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/12345 && printf 'GET /-/ready HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && head -1 <&3 | grep -q 200"]
       interval: 5s
       timeout: 10s
-      retries: 3
-      start_period: 10s
+      retries: 10
+      start_period: 30s
     deploy:
       restart_policy:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "coop-cloud.${STACK_NAME}.version=2.1.0+v1.18.1"
+        - "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1"
 configs:
   config_alloy:
     template_driver: golang

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}