diff --git a/.env.sample b/.env.sample index b97f05b..9b73272 100644 --- a/.env.sample +++ b/.env.sample @@ -72,6 +72,8 @@ SECRET_BASIC_AUTH_VERSION=v1 # # COMPOSE_FILE="$COMPOSE_FILE:compose.loki.yml" # +# LOKI_LOG_LEVEL=warn +# # Set to 0 to disable retention # LOKI_RETENTION_PERIOD=744h # LOKI_STORAGE_FILESYSTEM=1 @@ -130,3 +132,12 @@ SECRET_BASIC_AUTH_VERSION=v1 # Node memory usage alert will trigger when memory usage is above the given number in percent #ALERT_NODE_MEMORY_USAGE=85 + +# Loki ingestion alert will trigger when a host ships fewer than the given number of log entries per second +#ALERT_LOKI_INGESTION_STALLED=0.01 + +# Loki write error alert will trigger when log pushes fail at more than the given rate per second +#ALERT_LOKI_WRITE_ERRORS=0 + +# Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90 +#ALERT_LOKI_WAL_DISK_USAGE=85 diff --git a/abra.sh b/abra.sh index 10aaa5c..c161ae3 100644 --- a/abra.sh +++ b/abra.sh @@ -5,11 +5,12 @@ export GF_STACKS_DASH_VERSION=v3 export GF_TRAEFIK_DASH_VERSION=v3 export GF_BACKUP_DASH_VERSION=v1 export GF_CUSTOM_INI_VERSION=v4 -export LOKI_YML_VERSION=v3 +export LOKI_YML_VERSION=v4 export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1 export GF_ALERTS_NODE_VERSION=v3 +export GF_ALERTS_LOKI_VERSION=v5 export CONFIG_ALLOY_VERSION=v3 # migrates secrets from old names to new names by reading values from the diff --git a/alerts/loki.yml.tmpl b/alerts/loki.yml.tmpl new file mode 100644 index 0000000..e3d4c1b --- /dev/null +++ b/alerts/loki.yml.tmpl @@ -0,0 +1,191 @@ +apiVersion: 1 + +# List of alert rule UIDs that should be deleted +deleteRules: + {{ if not (env "ALERT_LOKI_INGESTION_STALLED") }} + - orgId: 1 + uid: coopcloud_loki_ingestion_stalled + {{ end }} + {{ if not (env "ALERT_LOKI_WRITE_ERRORS") }} + - orgId: 1 + uid: coopcloud_loki_write_errors + {{ end }} + {{ if not (env "ALERT_LOKI_WAL_DISK_USAGE") }} + - orgId: 1 + uid: coopcloud_loki_wal_disk_usage + {{ end }} + +groups: +- orgId: 1 + name: loki + folder: monitoring + interval: 5m + rules: + {{ if (env "ALERT_LOKI_INGESTION_STALLED") }} + - uid: coopcloud_loki_ingestion_stalled + title: Loki Ingestion Stalled + condition: C + data: + - refId: A + relativeTimeRange: + from: 1800 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + expr: sum by (instance) (rate(loki_write_sent_entries_total[15m])) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + relativeTimeRange: + from: 1800 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_LOKI_INGESTION_STALLED" }} + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: Alerting + execErrState: Error + for: 15m + annotations: + description: "" + runbook_url: "" + summary: Loki received no logs from {{`{{ $labels.instance }}`}} for 15m ({{`{{ printf "%.4f" (index $values "A").Value }}`}} entries/sec, threshold {{ env "ALERT_LOKI_INGESTION_STALLED" }}) + isPaused: false + {{ end }} + {{ if (env "ALERT_LOKI_WRITE_ERRORS") }} + - uid: coopcloud_loki_write_errors + title: Loki Write Errors + condition: C + data: + - refId: A + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + expr: sum by (instance, status_code) (rate(loki_write_request_duration_seconds_count{status_code!~"2.."}[10m])) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + relativeTimeRange: + from: 1200 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_LOKI_WRITE_ERRORS" }} + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 10m + annotations: + description: "" + runbook_url: "" + summary: Loki is rejecting log pushes from {{`{{ $labels.instance }}`}} with HTTP {{`{{ $labels.status_code }}`}} ({{`{{ printf "%.2f" (index $values "A").Value }}`}} failed pushes/sec) + isPaused: false + {{ end }} + {{ if (env "ALERT_LOKI_WAL_DISK_USAGE") }} + - uid: coopcloud_loki_wal_disk_usage + title: Loki WAL Disk Usage + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: PBFA97CFB590B2093 + model: + editorMode: code + expr: max by (instance) (loki_ingester_wal_disk_usage_percent * 100) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - {{ env "ALERT_LOKI_WAL_DISK_USAGE" }} + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 10m + annotations: + description: "" + runbook_url: "" + summary: Loki WAL disk {{`{{ printf "%.1f" (index $values "A").Value }}`}}% full on {{`{{ $labels.instance }}`}} (threshold {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}%, writes stop at 90%) + isPaused: false + {{ end }} diff --git a/compose.grafana.yml b/compose.grafana.yml index 5120196..236d3fd 100644 --- a/compose.grafana.yml +++ b/compose.grafana.yml @@ -24,6 +24,8 @@ services: target: /var/lib/grafana/dashboards/backup.json - source: gf_alerts_node target: /etc/grafana/provisioning/alerting/node.yml + - source: gf_alerts_loki + target: /etc/grafana/provisioning/alerting/loki.yml networks: - proxy - internal @@ -34,6 +36,9 @@ services: - GF_INSTALL_PLUGINS - ALERT_NODE_DISK_SPACE_LEFT - ALERT_NODE_MEMORY_USAGE + - ALERT_LOKI_INGESTION_STALLED + - ALERT_LOKI_WRITE_ERRORS + - ALERT_LOKI_WAL_DISK_USAGE deploy: labels: - "traefik.enable=true" @@ -77,6 +82,10 @@ configs: template_driver: golang name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION} file: alerts/node.yml.tmpl + gf_alerts_loki: + template_driver: golang + name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION} + file: alerts/loki.yml.tmpl volumes: grafana-data: diff --git a/compose.loki.yml b/compose.loki.yml index 8c2aa66..6ae034e 100644 --- a/compose.loki.yml +++ b/compose.loki.yml @@ -19,9 +19,16 @@ services: - LOKI_AWS_REGION - LOKI_BUCKET_NAMES - LOKI_RETENTION_PERIOD + - LOKI_LOG_LEVEL - STACK_NAME - LOKI_STORAGE_FILESYSTEM - LOKI_STORAGE_S3 + healthcheck: + test: ["CMD", "/usr/bin/loki", "-health"] + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s deploy: restart_policy: condition: on-failure @@ -34,6 +41,8 @@ services: - "traefik.http.routers.${STACK_NAME}-loki.tls=true" - "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}" - "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file" + - "prometheus.io/scrape=true" + - "prometheus.io/port=3100" configs: diff --git a/loki.yml.tmpl b/loki.yml.tmpl index e399700..2d74f1d 100644 --- a/loki.yml.tmpl +++ b/loki.yml.tmpl @@ -15,7 +15,7 @@ ruler: server: http_listen_port: 3100 - log_level: warn + log_level: {{ or (env "LOKI_LOG_LEVEL") "warn" }} distributor: ring: @@ -127,4 +127,4 @@ frontend: table_manager: retention_deletes_enabled: false - retention_period: 0s \ No newline at end of file + retention_period: 0s