diff --git a/.env.sample b/.env.sample index 9b73272..5e5ef78 100644 --- a/.env.sample +++ b/.env.sample @@ -58,6 +58,9 @@ SECRET_BASIC_AUTH_VERSION=v1 # COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml" +# Monitor physical disks health +# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml" + # Monitoring Server # ## Prometheus @@ -141,3 +144,6 @@ SECRET_BASIC_AUTH_VERSION=v1 # Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90 #ALERT_LOKI_WAL_DISK_USAGE=85 + +# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored +#ALERT_TRAEFIK_APDEX=0.5 diff --git a/.gitignore b/.gitignore index 37b52cc..ec1b785 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /.envrc +*~ \ No newline at end of file diff --git a/README.md b/README.md index 78f8cab..902a131 100644 --- a/README.md +++ b/README.md @@ -174,3 +174,12 @@ It is possible to enable the following alerts, by uncommenting the corresponding - node disk space: `ALERT_NODE_DISK_SPACE_LEFT` - node memory usage: `ALERT_NODE_MEMORY_USAGE` +- traefik apdex: `ALERT_TRAEFIK_APDEX` + +## smart monitoring + +To be able monitor hard drive health data, you need to configure +`smartd` to run on the host system, and also the +`collect-smartctl-json.sh` script provided here (via cronjob or as +a `smartd` hook). This is a limitation on Docker Swarm, which prevents +the `smartctl_exporter` from running on privileged mode. diff --git a/abra.sh b/abra.sh index c161ae3..75f6470 100644 --- a/abra.sh +++ b/abra.sh @@ -9,7 +9,7 @@ export LOKI_YML_VERSION=v4 export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 -export GF_ALERTS_NODE_VERSION=v3 +export GF_ALERTS_NODE_VERSION=v5 export GF_ALERTS_LOKI_VERSION=v5 export CONFIG_ALLOY_VERSION=v3 diff --git a/alerts/node.yml.tmpl b/alerts/node.yml.tmpl index 244db8c..6eff7c9 100644 --- a/alerts/node.yml.tmpl +++ b/alerts/node.yml.tmpl @@ -71,8 +71,7 @@ groups: description: "" runbook_url: "" summary: Less than {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left) - labels: - "": "" + labels: {} isPaused: false {{ end }} {{ if (env "ALERT_NODE_MEMORY_USAGE") }} @@ -87,7 +86,7 @@ groups: datasourceUid: PBFA97CFB590B2093 model: editorMode: code - expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 + expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100) instant: true intervalMs: 1000 legendFormat: __auto diff --git a/alerts/traefik.yml.tmpl b/alerts/traefik.yml.tmpl new file mode 100644 index 0000000..546da09 --- /dev/null +++ b/alerts/traefik.yml.tmpl @@ -0,0 +1,81 @@ +apiVersion: 1 + +# List of alert rule UIDs that should be deleted +deleteRules: + {{ if not (env "ALERT_TRAEFIK_APDEX") }} + - orgId: 1 + uid: coopcloud_traefik_apdex + {{ end }} + +groups: +- orgId: 1 + name: traefik + folder: recipes + interval: 5m + rules: + {{ if (env "ALERT_TRAEFIK_APDEX") }} + - uid: coopcloud_traefik_apdex + title: Apdex + condition: B + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + # apdex = (satisfied + tolerating / 2) / total, which reduces to + # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is + # cumulative and therefore already contains the satisfied requests. + # services with less than 10 requests per 10m are ignored. + expr: |- + ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) + + sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 / + sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service)) + and on(service) + ( + sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10 + ) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_TRAEFIK_APDEX" }} + type: lt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: B + type: threshold + noDataState: OK + execErrState: Error + for: 10m + annotations: + description: "" + runbook_url: "" + summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}}) + isPaused: false + {{ end }} diff --git a/collect-smartctl-json.service b/collect-smartctl-json.service new file mode 100644 index 0000000..b4b8e7c --- /dev/null +++ b/collect-smartctl-json.service @@ -0,0 +1,6 @@ +[Unit] +Description=Collect SMART data + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/collect-smartctl-json.sh diff --git a/collect-smartctl-json.sh b/collect-smartctl-json.sh new file mode 100755 index 0000000..9044bd7 --- /dev/null +++ b/collect-smartctl-json.sh @@ -0,0 +1,61 @@ +#! /bin/bash +# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh + +# Data directory to dump smartctl output +# This directory will be created if it doesn't exist +data_dir="/var/lib/smartmontools/json" + +# The original script used --xall but that doesn't work +# This matches the command in readSMARTctl() +smartctl_args="--json --info --health --attributes --tolerance=verypermissive \ +--nocheck=standby --format=brief --log=error" + +# Ignore this devices +smartctl_ignore_dev_regex="^(/dev/bus)" + +# Determine the json query tool to use +if command -v jq >/dev/null; then + json_tool="jq" + json_args="--raw-output" +elif command -v yq >/dev/null; then + json_tool="yq" + json_args="--unwrapScalar" +else + echo -e "One of 'yq' or 'jq' is required. Please try again after \ +installing one of them" + exit 1 +fi + +if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then + # Not root and sudo doesn't exist + echo "sudo does not exist. Please run this as root" + exit 1 +fi + +SUDO="sudo" +if [[ "${UID}" -eq 0 ]]; then + # Don't use sudo if root + SUDO="" +fi + +[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}" + +if [[ $# -ne 0 ]]; then + devices="${1}" +else + devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \ +".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")" + mapfile -t devices <<< "${devices[@]}" +fi + +for device in "${devices[@]}" + do + echo -n "Collecting data for '${device}'..." + # shellcheck disable=SC2086 + data="$($SUDO smartctl ${smartctl_args} ${device})" + # Accommodate a smartmontools pre-7.3 bug + data=${data#" Pending defect count:"} + device_name="$(basename "${device}")" + echo -e "\tSaving to ${device_name}.json" + echo "${data}" > "${data_dir}/${device_name}.json" +done diff --git a/collect-smartctl-json.timer b/collect-smartctl-json.timer new file mode 100644 index 0000000..a56e8b3 --- /dev/null +++ b/collect-smartctl-json.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Collect SMART data + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/compose.grafana.yml b/compose.grafana.yml index 236d3fd..73e691e 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -2,7 +2,7 @@ version: '3.8' services: grafana: - image: grafana/grafana:13.0.6 + image: grafana/grafana:13.2.3 volumes: - grafana-data:/var/lib/grafana:rw secrets: @@ -26,6 +26,8 @@ services: target: /etc/grafana/provisioning/alerting/node.yml - source: gf_alerts_loki target: /etc/grafana/provisioning/alerting/loki.yml + - source: gf_alerts_traefik + target: /etc/grafana/provisioning/alerting/traefik.yml networks: - proxy - internal @@ -39,6 +41,7 @@ services: - ALERT_LOKI_INGESTION_STALLED - ALERT_LOKI_WRITE_ERRORS - ALERT_LOKI_WAL_DISK_USAGE + - ALERT_TRAEFIK_APDEX deploy: labels: - "traefik.enable=true" @@ -53,7 +56,7 @@ services: interval: 5s timeout: 10s retries: 3 - start_period: 10s + start_period: 300s configs: gf_custom_ini: @@ -86,6 +89,10 @@ configs: template_driver: golang name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION} file: alerts/loki.yml.tmpl + gf_alerts_traefik: + template_driver: golang + name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION} + file: alerts/traefik.yml.tmpl volumes: grafana-data: diff --git a/compose.prometheus.yml b/compose.prometheus.yml index c28ff1d..a7d7e29 100644 --- a/compose.prometheus.yml +++ b/compose.prometheus.yml @@ -2,7 +2,7 @@ version: '3.8' services: prometheus: - image: prom/prometheus:v3.12.0 + image: prom/prometheus:v3.15.1 secrets: - basic_auth volumes: @@ -28,7 +28,7 @@ services: start_period: 30s deploy: restart_policy: - condition: on-failure + condition: unless-stopped labels: - "traefik.enable=true" - "traefik.swarm.network=proxy" diff --git a/compose.smartctl.yml b/compose.smartctl.yml new file mode 100644 index 0000000..32123bd --- /dev/null +++ b/compose.smartctl.yml @@ -0,0 +1,17 @@ +--- +version: "3.8" +services: + smartctl: + image: "prometheuscommunity/smartctl-exporter:v0.15.0" + volumes: + - "/var/lib/smartmontools/json:/debug" + command: + - "--smartctl.fake-data" + - "--smartctl.interval=1h" + networks: + - "proxy" + deploy: + labels: + - "prometheus.io/scrape=true" + - "prometheus.io/port=9633" + - "prometheus.io/path=/metrics" diff --git a/compose.yml b/compose.yml index b4b6a8b..0cd3382 100644 --- a/compose.yml +++ b/compose.yml @@ -3,7 +3,7 @@ version: "3.8" services: app: - image: grafana/alloy:v1.18.1 + image: grafana/alloy:v1.20.1 hostname: "${DOMAIN}" configs: - source: config_alloy @@ -50,10 +50,10 @@ services: start_period: 30s deploy: restart_policy: - condition: on-failure + condition: unless-stopped labels: - "backupbot.backup=${ENABLE_BACKUPS:-true}" - - "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1" + - "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1" configs: config_alloy: template_driver: golang diff --git a/release/next b/release/next new file mode 100644 index 0000000..cdd836d --- /dev/null +++ b/release/next @@ -0,0 +1 @@ +- alloy and prometheus containers restart policy from on-failure to unless-stopped \ No newline at end of file diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..5db72dd --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +}