change folder to monitoring

.env.sample

@@ -58,9 +58,6 @@ SECRET_BASIC_AUTH_VERSION=v1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
 
 
-# Monitor physical disks health
-# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml"
-
 # Monitoring Server
 #
 ## Prometheus
@@ -75,6 +72,8 @@ SECRET_BASIC_AUTH_VERSION=v1
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.loki.yml"
 #
+# LOKI_LOG_LEVEL=warn
+#
 # Set to 0 to disable retention
 # LOKI_RETENTION_PERIOD=744h
 # LOKI_STORAGE_FILESYSTEM=1
@@ -134,5 +133,11 @@ SECRET_BASIC_AUTH_VERSION=v1
 # Node memory usage alert will trigger when memory usage is above the given number in percent
 #ALERT_NODE_MEMORY_USAGE=85
 
-# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored
-#ALERT_TRAEFIK_APDEX=0.5
+# Loki ingestion alert will trigger when a host ships fewer than the given number of log entries per second
+#ALERT_LOKI_INGESTION_STALLED=0.01
+
+# Loki write error alert will trigger when log pushes fail at more than the given rate per second
+#ALERT_LOKI_WRITE_ERRORS=0
+
+# Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90
+#ALERT_LOKI_WAL_DISK_USAGE=85

README.md

@@ -174,12 +174,3 @@ It is possible to enable the following alerts, by uncommenting the corresponding
 
 - node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
 - node memory usage: `ALERT_NODE_MEMORY_USAGE`
-- traefik apdex: `ALERT_TRAEFIK_APDEX`
-
-## smart monitoring
-
-To be able monitor hard drive health data, you need to configure
-`smartd` to run on the host system, and also the
-`collect-smartctl-json.sh` script provided here (via cronjob or as
-a `smartd` hook).  This is a limitation on Docker Swarm, which prevents
-the `smartctl_exporter` from running on privileged mode.

abra.sh

@@ -5,12 +5,12 @@ export GF_STACKS_DASH_VERSION=v3
 export GF_TRAEFIK_DASH_VERSION=v3
 export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
-export LOKI_YML_VERSION=v3
+export LOKI_YML_VERSION=v4
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
-export GF_ALERTS_TRAEFIK_VERSION=v1
+export GF_ALERTS_LOKI_VERSION=v5
 export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the

alerts/loki.yml.tmpl

@@ -0,0 +1,191 @@
+apiVersion: 1
+
+# List of alert rule UIDs that should be deleted
+deleteRules:
+  {{ if not (env "ALERT_LOKI_INGESTION_STALLED") }}
+  - orgId: 1
+    uid: coopcloud_loki_ingestion_stalled
+  {{ end }}
+  {{ if not (env "ALERT_LOKI_WRITE_ERRORS") }}
+  - orgId: 1
+    uid: coopcloud_loki_write_errors
+  {{ end }}
+  {{ if not (env "ALERT_LOKI_WAL_DISK_USAGE") }}
+  - orgId: 1
+    uid: coopcloud_loki_wal_disk_usage
+  {{ end }}
+
+groups:
+- orgId: 1
+  name: loki
+  folder: monitoring
+  interval: 5m
+  rules:
+    {{ if (env "ALERT_LOKI_INGESTION_STALLED") }}
+    - uid: coopcloud_loki_ingestion_stalled
+      title: Loki Ingestion Stalled
+      condition: C
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 1800
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            expr: sum by (instance) (rate(loki_write_sent_entries_total[15m]))
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: C
+          relativeTimeRange:
+            from: 1800
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_LOKI_INGESTION_STALLED" }}
+                    type: lt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                        - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: C
+            type: threshold
+      noDataState: Alerting
+      execErrState: Error
+      for: 15m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Loki received no logs from {{`{{ $labels.instance }}`}} for 15m ({{`{{ printf "%.4f" (index $values "A").Value }}`}} entries/sec, threshold {{ env "ALERT_LOKI_INGESTION_STALLED" }})
+      isPaused: false
+    {{ end }}
+    {{ if (env "ALERT_LOKI_WRITE_ERRORS") }}
+    - uid: coopcloud_loki_write_errors
+      title: Loki Write Errors
+      condition: C
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 1200
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            expr: sum by (instance, status_code) (rate(loki_write_request_duration_seconds_count{status_code!~"2.."}[10m]))
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: C
+          relativeTimeRange:
+            from: 1200
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_LOKI_WRITE_ERRORS" }}
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                        - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: C
+            type: threshold
+      noDataState: NoData
+      execErrState: Error
+      for: 10m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Loki is rejecting log pushes from {{`{{ $labels.instance }}`}} with HTTP {{`{{ $labels.status_code }}`}} ({{`{{ printf "%.2f" (index $values "A").Value }}`}} failed pushes/sec)
+      isPaused: false
+    {{ end }}
+    {{ if (env "ALERT_LOKI_WAL_DISK_USAGE") }}
+    - uid: coopcloud_loki_wal_disk_usage
+      title: Loki WAL Disk Usage
+      condition: C
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            expr: max by (instance) (loki_ingester_wal_disk_usage_percent * 100)
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: C
+          relativeTimeRange:
+            from: 600
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                        - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: C
+            type: threshold
+      noDataState: NoData
+      execErrState: Error
+      for: 10m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Loki WAL disk {{`{{ printf "%.1f" (index $values "A").Value }}`}}% full on {{`{{ $labels.instance }}`}} (threshold {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}%, writes stop at 90%)
+      isPaused: false
+    {{ end }}

alerts/traefik.yml.tmpl

@@ -1,81 +0,0 @@
-apiVersion: 1
-
-# List of alert rule UIDs that should be deleted
-deleteRules:
-  {{ if not (env "ALERT_TRAEFIK_APDEX") }}
-  - orgId: 1
-    uid: coopcloud_traefik_apdex
-  {{ end }}
-
-groups:
-- orgId: 1
-  name: traefik
-  folder: recipes
-  interval: 5m
-  rules:
-    {{ if (env "ALERT_TRAEFIK_APDEX") }}
-    - uid: coopcloud_traefik_apdex
-      title: Apdex
-      condition: B
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            # apdex = (satisfied + tolerating / 2) / total, which reduces to
-            # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is
-            # cumulative and therefore already contains the satisfied requests.
-            # services with less than 10 requests per 10m are ignored.
-            expr: |-
-              ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) +
-               sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 /
-               sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service))
-              and on(service)
-              (
-                sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10
-              )
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: B
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_TRAEFIK_APDEX" }}
-                    type: lt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: B
-            type: threshold
-      noDataState: OK
-      execErrState: Error
-      for: 10m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}})
-      isPaused: false
-    {{ end }}

collect-smartctl-json.service

@@ -1,6 +0,0 @@
-[Unit]
-Description=Collect SMART data
-
-[Service]
-Type=oneshot
-ExecStart=/usr/local/bin/collect-smartctl-json.sh

collect-smartctl-json.sh

@@ -1,61 +0,0 @@
-#! /bin/bash
-# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh
-
-# Data directory to dump smartctl output
-# This directory will be created if it doesn't exist
-data_dir="/var/lib/smartmontools/json"
-
-# The original script used --xall but that doesn't work
-# This matches the command in readSMARTctl()
-smartctl_args="--json --info --health --attributes --tolerance=verypermissive \
---nocheck=standby --format=brief --log=error"
-
-# Ignore this devices
-smartctl_ignore_dev_regex="^(/dev/bus)"
-
-# Determine the json query tool to use
-if command -v jq >/dev/null; then
-	json_tool="jq"
-	json_args="--raw-output"
-elif command -v yq >/dev/null; then
-	json_tool="yq"
-	json_args="--unwrapScalar"
-else
-	echo -e "One of 'yq' or 'jq' is required. Please try again after \
-installing one of them"
-	exit 1
-fi
-
-if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then
-	# Not root and sudo doesn't exist
-	echo "sudo does not exist. Please run this as root"
-	exit 1
-fi
-
-SUDO="sudo"
-if [[ "${UID}" -eq 0 ]]; then
-	# Don't use sudo if root
-	SUDO=""
-fi
-
-[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}"
-
-if [[ $# -ne 0 ]]; then
-	devices="${1}"
-else
-	devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \
-".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")"
-  mapfile -t devices <<< "${devices[@]}"
-fi
-
-for device in "${devices[@]}"
-  do
-	echo -n "Collecting data for '${device}'..."
-	# shellcheck disable=SC2086
-	data="$($SUDO smartctl ${smartctl_args} ${device})"
-	# Accommodate a smartmontools pre-7.3 bug
-	data=${data#"  Pending defect count:"}
-	device_name="$(basename "${device}")"
-	echo -e "\tSaving to ${device_name}.json"
-	echo "${data}" > "${data_dir}/${device_name}.json"
-done

collect-smartctl-json.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Collect SMART data
-
-[Timer]
-OnCalendar=hourly
-Persistent=true
-
-[Install]
-WantedBy=timers.target

compose.grafana.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:13.2.3
+    image: grafana/grafana:13.0.6
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:
@@ -24,8 +24,8 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
-      - source: gf_alerts_traefik
-        target: /etc/grafana/provisioning/alerting/traefik.yml
+      - source: gf_alerts_loki
+        target: /etc/grafana/provisioning/alerting/loki.yml
     networks:
       - proxy
       - internal
@@ -36,7 +36,9 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_LEFT
       - ALERT_NODE_MEMORY_USAGE
-      - ALERT_TRAEFIK_APDEX
+      - ALERT_LOKI_INGESTION_STALLED
+      - ALERT_LOKI_WRITE_ERRORS
+      - ALERT_LOKI_WAL_DISK_USAGE
     deploy:
       labels:
         - "traefik.enable=true"
@@ -80,10 +82,10 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
-  gf_alerts_traefik:
+  gf_alerts_loki:
     template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION}
-    file: alerts/traefik.yml.tmpl
+    name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION}
+    file: alerts/loki.yml.tmpl
 
 volumes:
   grafana-data:

compose.loki.yml

@@ -19,9 +19,16 @@ services:
       - LOKI_AWS_REGION
       - LOKI_BUCKET_NAMES
       - LOKI_RETENTION_PERIOD
+      - LOKI_LOG_LEVEL
       - STACK_NAME
       - LOKI_STORAGE_FILESYSTEM
       - LOKI_STORAGE_S3
+    healthcheck:
+      test: ["CMD", "/usr/bin/loki", "-health"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
+      start_period: 30s
     deploy:
       restart_policy:
         condition: on-failure
@@ -34,6 +41,8 @@ services:
         - "traefik.http.routers.${STACK_NAME}-loki.tls=true"
         - "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
+        - "prometheus.io/scrape=true"
+        - "prometheus.io/port=3100"
 
 
 configs:

compose.prometheus.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v3.15.0
+    image: prom/prometheus:v3.12.0
     secrets:
       - basic_auth
     volumes:

compose.smartctl.yml

@@ -1,17 +0,0 @@
----
-version: "3.8"
-services:
-  smartctl:
-    image: "prometheuscommunity/smartctl-exporter:v0.15.0"
-    volumes:
-    - "/var/lib/smartmontools/json:/debug"
-    command:
-    - "--smartctl.fake-data"
-    - "--smartctl.interval=1h"
-    networks:
-    - "proxy"
-    deploy:
-      labels:
-        - "prometheus.io/scrape=true"
-        - "prometheus.io/port=9633"
-        - "prometheus.io/path=/metrics"

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: grafana/alloy:v1.20.1
+    image: grafana/alloy:v1.18.1
     hostname: "${DOMAIN}"
     configs:
       - source: config_alloy
@@ -53,7 +53,7 @@ services:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1"
+        - "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1"
 configs:
   config_alloy:
     template_driver: golang

loki.yml.tmpl

@@ -15,7 +15,7 @@ ruler:
 
 server:
   http_listen_port: 3100
-  log_level: warn
+  log_level: {{ or (env "LOKI_LOG_LEVEL") "warn" }}
 
 distributor:
   ring:
@@ -127,4 +127,4 @@ frontend:
 
 table_manager:
   retention_deletes_enabled: false
-  retention_period: 0s
+  retention_period: 0s

renovate.json

@@ -1,6 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended"
-  ]
-}