feat(matrix): show more information in matrix alerts

Reviewed-on: #11

.env.sample

@@ -58,6 +58,9 @@ SECRET_BASIC_AUTH_VERSION=v1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
 
 
+# Monitor physical disks health
+# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml"
+
 # Monitoring Server
 #
 ## Prometheus

README.md

@@ -174,3 +174,11 @@ It is possible to enable the following alerts, by uncommenting the corresponding
 
 - node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
 - node memory usage: `ALERT_NODE_MEMORY_USAGE`
+
+## smart monitoring
+
+To be able monitor hard drive health data, you need to configure
+`smartd` to run on the host system, and also the
+`collect-smartctl-json.sh` script provided here (via cronjob or as
+a `smartd` hook).  This is a limitation on Docker Swarm, which prevents
+the `smartctl_exporter` from running on privileged mode.

abra.sh

@@ -7,7 +7,7 @@ export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
 export LOKI_YML_VERSION=v3
 export PROMETHEUS_YML_VERSION=v2
-export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=v2
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
 export CONFIG_ALLOY_VERSION=v3

alertmanager-matrix-config.yml.tmpl

@@ -54,6 +54,12 @@ templating:
       {{ else if .Alert.Labels.alertname }}
         {{ .Alert.Labels.alertname }}
       {{ end }}
+      {{ if .Alert.Labels.rulename }}
+        &rarr; <strong>{{ .Alert.Labels.rulename }}</strong>
+      {{ end }}
+      {{ if .Alert.Labels.grafana_folder }}
+        <em>({{ .Alert.Labels.grafana_folder }})</em>
+      {{ end }}
       >>
       {{ if .Alert.Labels.severity }}
         {{ .Alert.Labels.severity | ToUpper }}:
@@ -62,6 +68,11 @@ templating:
         {{ .Alert.Annotations.description }}
       {{ else if .Alert.Annotations.summary }}
         {{ .Alert.Annotations.summary }}
+      {{ else if .Alert.Labels.rulename }}
+        Query {{ .Alert.Labels.ref_id }} returned no data (datasource {{ .Alert.Labels.datasource_uid }})
+      {{ end }}
+      {{ if .Alert.Labels.instance }}
+        on {{ .Alert.Labels.instance }}
       {{ end }}
       >>
       {{ if .Alert.Annotations.runbook }}
@@ -70,5 +81,8 @@ templating:
       {{ if .Alert.Annotations.dashboard }}
         <a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> |
       {{ end }}
+      {{ if .GeneratorURL }}
+        <a href="{{ .GeneratorURL }}">Rule</a> |
+      {{ end }}
       <a href="{{ .SilenceURL }}">Silence</a>
     </p>`}}'

collect-smartctl-json.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Collect SMART data
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/collect-smartctl-json.sh

collect-smartctl-json.sh

@@ -0,0 +1,61 @@
+#! /bin/bash
+# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh
+
+# Data directory to dump smartctl output
+# This directory will be created if it doesn't exist
+data_dir="/var/lib/smartmontools/json"
+
+# The original script used --xall but that doesn't work
+# This matches the command in readSMARTctl()
+smartctl_args="--json --info --health --attributes --tolerance=verypermissive \
+--nocheck=standby --format=brief --log=error"
+
+# Ignore this devices
+smartctl_ignore_dev_regex="^(/dev/bus)"
+
+# Determine the json query tool to use
+if command -v jq >/dev/null; then
+	json_tool="jq"
+	json_args="--raw-output"
+elif command -v yq >/dev/null; then
+	json_tool="yq"
+	json_args="--unwrapScalar"
+else
+	echo -e "One of 'yq' or 'jq' is required. Please try again after \
+installing one of them"
+	exit 1
+fi
+
+if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then
+	# Not root and sudo doesn't exist
+	echo "sudo does not exist. Please run this as root"
+	exit 1
+fi
+
+SUDO="sudo"
+if [[ "${UID}" -eq 0 ]]; then
+	# Don't use sudo if root
+	SUDO=""
+fi
+
+[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}"
+
+if [[ $# -ne 0 ]]; then
+	devices="${1}"
+else
+	devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \
+".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")"
+  mapfile -t devices <<< "${devices[@]}"
+fi
+
+for device in "${devices[@]}"
+  do
+	echo -n "Collecting data for '${device}'..."
+	# shellcheck disable=SC2086
+	data="$($SUDO smartctl ${smartctl_args} ${device})"
+	# Accommodate a smartmontools pre-7.3 bug
+	data=${data#"  Pending defect count:"}
+	device_name="$(basename "${device}")"
+	echo -e "\tSaving to ${device_name}.json"
+	echo "${data}" > "${data_dir}/${device_name}.json"
+done

collect-smartctl-json.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Collect SMART data
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

compose.smartctl.yml

@@ -0,0 +1,17 @@
+---
+version: "3.8"
+services:
+  smartctl:
+    image: "prometheuscommunity/smartctl-exporter:v0.14.0"
+    volumes:
+    - "/var/lib/smartmontools/json:/debug"
+    command:
+    - "--smartctl.fake-data"
+    - "--smartctl.interval=1h"
+    networks:
+    - "proxy"
+    deploy:
+      labels:
+        - "prometheus.io/scrape=true"
+        - "prometheus.io/port=9633"
+        - "prometheus.io/path=/metrics"

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}