fix: remove unused vars

.env.sample

@@ -53,10 +53,9 @@ SECRET_BASIC_AUTH_VERSION=v1
 # Not needed if you just want to read local log files — use SYSLOG_FILES instead.
 # SYSLOG=1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml"
-#
-# Scrape Docker-Events via official docker-cli image using docker socket
-# COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
 
+# Monitor physical disks health
+# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml"
 
 # Monitoring Server
 #
@@ -72,8 +71,6 @@ SECRET_BASIC_AUTH_VERSION=v1
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.loki.yml"
 #
-# LOKI_LOG_LEVEL=warn
-#
 # Set to 0 to disable retention
 # LOKI_RETENTION_PERIOD=744h
 # LOKI_STORAGE_FILESYSTEM=1
@@ -132,12 +129,3 @@ SECRET_BASIC_AUTH_VERSION=v1
 
 # Node memory usage alert will trigger when memory usage is above the given number in percent
 #ALERT_NODE_MEMORY_USAGE=85
-
-# Loki ingestion alert will trigger when a host ships fewer than the given number of log entries per second
-#ALERT_LOKI_INGESTION_STALLED=0.01
-
-# Loki write error alert will trigger when log pushes fail at more than the given rate per second
-#ALERT_LOKI_WRITE_ERRORS=0
-
-# Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90
-#ALERT_LOKI_WAL_DISK_USAGE=85

.gitea/CODEOWNERS

@@ -1 +0,0 @@
-* @coop-cloud/monitoring-ng-maintainers

MAINTENANCE.md

@@ -1,35 +0,0 @@
-# Monitoring NG Recipe Maintenance
-
-All contributions should be made via a pull request. This is to ensure a
-certain quality and consistency, that others can rely on.
-
-## Maintainer Responsibilities
-
-A recipe maintainer has the following responsibilities:
-
-- Respond to pull requests / issues within two weeks
-- Make image security updates within a week
-- Make image major updates every three months
-
-In order to fullfill these responsibilities a recipe maintainer:
-
-- Has to watch the repository (to get notifications)
-- Needs to make sure renovate is configured properly
-
-## Pull Requests
-
-A pull request can be merged if it is approved by at least one maintainer. For
-pull requests opened by a maintainer they need to be approved by another
-maintainer. Even though it is okay to merge a pull request with one approval, it
-is always better if all maintainers looked at the pull request and approved it.
-
-## Become a maintainer
-
-Everyone can apply to be a recipe maintainer:
-1. Watch the repository to always get updates
-2. Simply add your self to the list in the [README.md](./README.md) and open a new pull request with the change.
-3. Once the pull request gets merged you will be added to the
-   [monitoring-ng maintainers
-   team](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers).
-4. Join the room
-   [#cc-|-monitoring-ng-maintenance:matrix.org](#cc-|-monitoring-ng-maintenance:matrix.org) and chat to other maintainers.

README.md

@@ -7,7 +7,7 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou
 Metrics and logs are collected by [Grafana Alloy](https://grafana.com/docs/alloy/latest/) and pushed to a central Prometheus/Loki (via `remote_write`/`loki push`). Every `monitoring-ng` instance runs its own Alloy, whether or not it also runs the central stack.
 
 <!-- metadata -->
-- **Maintainer**: [@coop-cloud/monitoring-ng-maintainers](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers)
+
 - **Category**: Apps
 - **Status**: 2, beta
 - **Image**: [`grafana/grafana`](https://hub.docker.com/r/grafana/grafana), 4, upstream
@@ -54,14 +54,6 @@ This is what a gathering host pushes into. It also runs its own Alloy, so it mon
 
 ## Additional features
 
-### Scrape docker events
-
-Just uncomment the following line in your .env to collect docker events using
-the official docker-cli image and the docker socket:
-```
-COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
-```
-
 ### Discovering metrics from other apps
 
 Alloy auto-discovers and scrapes other Docker Swarm services running on the same host, on the `proxy` network, that opt in via labels. No manual scrape config needed. On the app's `compose.yml`:
@@ -174,3 +166,11 @@ It is possible to enable the following alerts, by uncommenting the corresponding
 
 - node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
 - node memory usage: `ALERT_NODE_MEMORY_USAGE`
+
+## smart monitoring
+
+To be able monitor hard drive health data, you need to configure
+`smartd` to run on the host system, and also the
+`collect-smartctl-json.sh` script provided here (via cronjob or as
+a `smartd` hook).  This is a limitation on Docker Swarm, which prevents
+the `smartctl_exporter` from running on privileged mode.

abra.sh

@@ -5,13 +5,12 @@ export GF_STACKS_DASH_VERSION=v3
 export GF_TRAEFIK_DASH_VERSION=v3
 export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
-export LOKI_YML_VERSION=v4
+export LOKI_YML_VERSION=v3
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
-export GF_ALERTS_LOKI_VERSION=v5
+export CONFIG_ALLOY_VERSION=v2
-export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the
 # running containers on the server and re-inserting them under the new names.

alerts/loki.yml.tmpl

@@ -1,191 +0,0 @@
-apiVersion: 1
-
-# List of alert rule UIDs that should be deleted
-deleteRules:
-  {{ if not (env "ALERT_LOKI_INGESTION_STALLED") }}
-  - orgId: 1
-    uid: coopcloud_loki_ingestion_stalled
-  {{ end }}
-  {{ if not (env "ALERT_LOKI_WRITE_ERRORS") }}
-  - orgId: 1
-    uid: coopcloud_loki_write_errors
-  {{ end }}
-  {{ if not (env "ALERT_LOKI_WAL_DISK_USAGE") }}
-  - orgId: 1
-    uid: coopcloud_loki_wal_disk_usage
-  {{ end }}
-
-groups:
-- orgId: 1
-  name: loki
-  folder: monitoring
-  interval: 5m
-  rules:
-    {{ if (env "ALERT_LOKI_INGESTION_STALLED") }}
-    - uid: coopcloud_loki_ingestion_stalled
-      title: Loki Ingestion Stalled
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 1800
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: sum by (instance) (rate(loki_write_sent_entries_total[15m]))
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 1800
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_LOKI_INGESTION_STALLED" }}
-                    type: lt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: Alerting
-      execErrState: Error
-      for: 15m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Loki received no logs from {{`{{ $labels.instance }}`}} for 15m ({{`{{ printf "%.4f" (index $values "A").Value }}`}} entries/sec, threshold {{ env "ALERT_LOKI_INGESTION_STALLED" }})
-      isPaused: false
-    {{ end }}
-    {{ if (env "ALERT_LOKI_WRITE_ERRORS") }}
-    - uid: coopcloud_loki_write_errors
-      title: Loki Write Errors
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 1200
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: sum by (instance, status_code) (rate(loki_write_request_duration_seconds_count{status_code!~"2.."}[10m]))
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 1200
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_LOKI_WRITE_ERRORS" }}
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 10m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Loki is rejecting log pushes from {{`{{ $labels.instance }}`}} with HTTP {{`{{ $labels.status_code }}`}} ({{`{{ printf "%.2f" (index $values "A").Value }}`}} failed pushes/sec)
-      isPaused: false
-    {{ end }}
-    {{ if (env "ALERT_LOKI_WAL_DISK_USAGE") }}
-    - uid: coopcloud_loki_wal_disk_usage
-      title: Loki WAL Disk Usage
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: max by (instance) (loki_ingester_wal_disk_usage_percent * 100)
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 10m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Loki WAL disk {{`{{ printf "%.1f" (index $values "A").Value }}`}}% full on {{`{{ $labels.instance }}`}} (threshold {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}%, writes stop at 90%)
-      isPaused: false
-    {{ end }}

collect-smartctl-json.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Collect SMART data
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/collect-smartctl-json.sh

collect-smartctl-json.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh
+
+# Data directory to dump smartctl output
+# This directory will be created if it doesn't exist
+data_dir="/var/lib/smartmontools/json"
+
+# The original script used --xall but that doesn't work
+# This matches the command in readSMARTctl()
+smartctl_args="--json --info --health --attributes --tolerance=verypermissive \
+--nocheck=standby --format=brief --log=error"
+
+# Ignore this devices
+smartctl_ignore_dev_regex="^(/dev/bus)"
+
+# Determine the json query tool to use
+if command -v jq >/dev/null; then
+	json_tool="jq"
+	json_args="--raw-output"
+elif command -v yq >/dev/null; then
+	json_tool="yq"
+	json_args="--unwrapScalar"
+else
+	echo -e "One of 'yq' or 'jq' is required. Please try again after \
+installing one of them"
+	exit 1
+fi
+
+if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then
+	# Not root and sudo doesn't exist
+	echo "sudo does not exist. Please run this as root"
+	exit 1
+fi
+
+SUDO="sudo"
+if [[ "${UID}" -eq 0 ]]; then
+	# Don't use sudo if root
+	SUDO=""
+fi
+
+[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}"
+
+if [[ $# -ne 0 ]]; then
+	devices="${1}"
+else
+	devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \
+".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")"
+  mapfile -t devices <<< "${devices[@]}"
+fi
+
+for device in "${devices[@]}"
+  do
+	echo -n "Collecting data for '${device}'..."
+	# shellcheck disable=SC2086
+	data="$($SUDO smartctl ${smartctl_args} ${device})"
+	# Accommodate a smartmontools pre-7.3 bug
+	data=${data#"  Pending defect count:"}
+	device_name="$(basename "${device}")"
+	echo -e "\tSaving to ${device_name}.json"
+	echo "${data}" > "${data_dir}/${device_name}.json"
+done

collect-smartctl-json.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Collect SMART data
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

compose.docker-events.yml

@@ -1,20 +0,0 @@
-version: "3.8"
-services:
-  docker-events:
-    image: docker:29.8.1-cli
-    command: >
-      docker events
-      --format '{{json .}}'
-      --filter type=container
-      --filter event=start
-      --filter event=die
-      --filter event=kill
-      --filter event=oom
-      --filter event=restart
-      --filter event=stop
-      --filter event=health_status
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    deploy:
-      restart_policy:
-        condition: on-failure

compose.grafana.yml

@@ -24,8 +24,6 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
-      - source: gf_alerts_loki
-        target: /etc/grafana/provisioning/alerting/loki.yml
     networks:
       - proxy
       - internal
@@ -36,9 +34,6 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_LEFT
       - ALERT_NODE_MEMORY_USAGE
-      - ALERT_LOKI_INGESTION_STALLED
-      - ALERT_LOKI_WRITE_ERRORS
-      - ALERT_LOKI_WAL_DISK_USAGE
     deploy:
       labels:
         - "traefik.enable=true"
@@ -82,10 +77,6 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
-  gf_alerts_loki:
-    template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION}
-    file: alerts/loki.yml.tmpl
 
 volumes:
   grafana-data:

compose.loki.yml

@@ -19,16 +19,9 @@ services:
       - LOKI_AWS_REGION
       - LOKI_BUCKET_NAMES
       - LOKI_RETENTION_PERIOD
-      - LOKI_LOG_LEVEL
       - STACK_NAME
       - LOKI_STORAGE_FILESYSTEM
       - LOKI_STORAGE_S3
-    healthcheck:
-      test: ["CMD", "/usr/bin/loki", "-health"]
-      interval: 10s
-      timeout: 10s
-      retries: 10
-      start_period: 30s
     deploy:
       restart_policy:
         condition: on-failure
@@ -41,8 +34,6 @@ services:
         - "traefik.http.routers.${STACK_NAME}-loki.tls=true"
         - "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
-        - "prometheus.io/scrape=true"
-        - "prometheus.io/port=3100"
 
 
 configs:

compose.smartctl.yml

@@ -0,0 +1,17 @@
+---
+version: "3.8"
+services:
+  smartctl:
+    image: "prometheuscommunity/smartctl-exporter:v0.14.0"
+    volumes:
+    - "/var/lib/smartmontools/json:/debug"
+    command:
+    - "--smartctl.fake-data"
+    - "--smartctl.interval=1h"
+    networks:
+    - "proxy"
+    deploy:
+      labels:
+        - "prometheus.io/scrape=true"
+        - "prometheus.io/port=9633"
+        - "prometheus.io/path=/metrics"

compose.yml

@@ -46,14 +46,14 @@ services:
       test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/12345 && printf 'GET /-/ready HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && head -1 <&3 | grep -q 200"]
       interval: 5s
       timeout: 10s
-      retries: 10
+      retries: 3
-      start_period: 30s
+      start_period: 10s
     deploy:
       restart_policy:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1"
+        - "coop-cloud.${STACK_NAME}.version=2.0.0+v1.18.1"
 configs:
   config_alloy:
     template_driver: golang

config.alloy.tmpl

@@ -52,32 +52,7 @@ prometheus.scrape "default" {
     prometheus.exporter.cadvisor.docker.targets,
   )
 
-  forward_to = [prometheus.relabel.container_meta.receiver]
-}
-
-prometheus.relabel "container_meta" {
   forward_to = [prometheus.remote_write.prometheus.receiver]
-
-  // remove sha tail: nginx:1.31.1@sha256:608a... -> nginx:1.31.1
-  rule {
-    source_labels = ["image"]
-    regex         = "([^@]+)@sha256:.*"
-    target_label  = "image"
-    replacement   = "$1"
-  }
-  // split image in name and tag
-  rule {
-    source_labels = ["image"]
-    regex         = "(.+):[^:/]+"
-    target_label  = "image_name"
-    replacement   = "$1"
-  }
-  rule {
-    source_labels = ["image"]
-    regex         = ".+:([^:/]+)"
-    target_label  = "image_tag"
-    replacement   = "$1"
-  }
 }
 
 prometheus.remote_write "prometheus" {
@@ -315,35 +290,7 @@ loki.source.docker "docker" {
 loki.source.journal "journal" {
   path = "/rootfs/var/log/journal"
   labels = { job = "{{ env "DOMAIN" }}" }
-  relabel_rules = loki.relabel.journal.rules
-  forward_to = [loki.process.journal.receiver]
-}
-
-loki.relabel "journal" {
-  forward_to = []
-
-  rule {
-    source_labels = ["__journal__systemd_unit"]
-    target_label  = "unit"
-  }
-  rule {
-    source_labels = ["__journal_syslog_identifier"]
-    target_label  = "ident"
-  }
-  rule {
-    source_labels = ["__journal_priority_keyword"]
-    target_label  = "priority"
-  }
-}
-
-// drop network db stats (15-20% of kernel logs)
-loki.process "journal" {
   forward_to = [loki.write.loki.receiver]
-
-  stage.drop {
-    expression          = ".*NetworkDB stats.*"
-    drop_counter_reason = "networkdb_stats"
-  }
 }
 {{ end }}
 

loki.yml.tmpl

@@ -15,7 +15,7 @@ ruler:
 
 server:
   http_listen_port: 3100
-  log_level: {{ or (env "LOKI_LOG_LEVEL") "warn" }}
+  log_level: warn
 
 distributor:
   ring:
@@ -127,4 +127,4 @@ frontend:
 
 table_manager:
   retention_deletes_enabled: false
-  retention_period: 0s
+  retention_period: 0s

release/2.1.0+v1.18.1

@@ -1 +0,0 @@
-alloy: split image label in "image_name" and "image_tag", relabel journald logs and drop NetworkDB stats

release/2.2.0+v1.18.1

@@ -1 +0,0 @@
-allow scraping of docker events via new compose.docker-events.yml