fix: monitoring-ng

.env.sample

@@ -72,8 +72,6 @@ SECRET_BASIC_AUTH_VERSION=v1
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.loki.yml"
 #
-# LOKI_LOG_LEVEL=warn
-#
 # Set to 0 to disable retention
 # LOKI_RETENTION_PERIOD=744h
 # LOKI_STORAGE_FILESYSTEM=1
@@ -132,12 +130,3 @@ SECRET_BASIC_AUTH_VERSION=v1
 
 # Node memory usage alert will trigger when memory usage is above the given number in percent
 #ALERT_NODE_MEMORY_USAGE=85
-
-# Loki ingestion alert will trigger when a host ships fewer than the given number of log entries per second
-#ALERT_LOKI_INGESTION_STALLED=0.01
-
-# Loki write error alert will trigger when log pushes fail at more than the given rate per second
-#ALERT_LOKI_WRITE_ERRORS=0
-
-# Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90
-#ALERT_LOKI_WAL_DISK_USAGE=85

abra.sh

@@ -5,12 +5,11 @@ export GF_STACKS_DASH_VERSION=v3
 export GF_TRAEFIK_DASH_VERSION=v3
 export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
-export LOKI_YML_VERSION=v4
+export LOKI_YML_VERSION=v3
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
-export GF_ALERTS_LOKI_VERSION=v5
 export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the

alerts/loki.yml.tmpl

@@ -1,191 +0,0 @@
-apiVersion: 1
-
-# List of alert rule UIDs that should be deleted
-deleteRules:
-  {{ if not (env "ALERT_LOKI_INGESTION_STALLED") }}
-  - orgId: 1
-    uid: coopcloud_loki_ingestion_stalled
-  {{ end }}
-  {{ if not (env "ALERT_LOKI_WRITE_ERRORS") }}
-  - orgId: 1
-    uid: coopcloud_loki_write_errors
-  {{ end }}
-  {{ if not (env "ALERT_LOKI_WAL_DISK_USAGE") }}
-  - orgId: 1
-    uid: coopcloud_loki_wal_disk_usage
-  {{ end }}
-
-groups:
-- orgId: 1
-  name: loki
-  folder: monitoring
-  interval: 5m
-  rules:
-    {{ if (env "ALERT_LOKI_INGESTION_STALLED") }}
-    - uid: coopcloud_loki_ingestion_stalled
-      title: Loki Ingestion Stalled
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 1800
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: sum by (instance) (rate(loki_write_sent_entries_total[15m]))
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 1800
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_LOKI_INGESTION_STALLED" }}
-                    type: lt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: Alerting
-      execErrState: Error
-      for: 15m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Loki received no logs from {{`{{ $labels.instance }}`}} for 15m ({{`{{ printf "%.4f" (index $values "A").Value }}`}} entries/sec, threshold {{ env "ALERT_LOKI_INGESTION_STALLED" }})
-      isPaused: false
-    {{ end }}
-    {{ if (env "ALERT_LOKI_WRITE_ERRORS") }}
-    - uid: coopcloud_loki_write_errors
-      title: Loki Write Errors
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 1200
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: sum by (instance, status_code) (rate(loki_write_request_duration_seconds_count{status_code!~"2.."}[10m]))
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 1200
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_LOKI_WRITE_ERRORS" }}
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 10m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Loki is rejecting log pushes from {{`{{ $labels.instance }}`}} with HTTP {{`{{ $labels.status_code }}`}} ({{`{{ printf "%.2f" (index $values "A").Value }}`}} failed pushes/sec)
-      isPaused: false
-    {{ end }}
-    {{ if (env "ALERT_LOKI_WAL_DISK_USAGE") }}
-    - uid: coopcloud_loki_wal_disk_usage
-      title: Loki WAL Disk Usage
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: max by (instance) (loki_ingester_wal_disk_usage_percent * 100)
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 10m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Loki WAL disk {{`{{ printf "%.1f" (index $values "A").Value }}`}}% full on {{`{{ $labels.instance }}`}} (threshold {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}%, writes stop at 90%)
-      isPaused: false
-    {{ end }}

compose.grafana.yml

@@ -24,8 +24,6 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
-      - source: gf_alerts_loki
-        target: /etc/grafana/provisioning/alerting/loki.yml
     networks:
       - proxy
       - internal
@@ -36,9 +34,6 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_LEFT
       - ALERT_NODE_MEMORY_USAGE
-      - ALERT_LOKI_INGESTION_STALLED
-      - ALERT_LOKI_WRITE_ERRORS
-      - ALERT_LOKI_WAL_DISK_USAGE
     deploy:
       labels:
         - "traefik.enable=true"
@@ -82,10 +77,6 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
-  gf_alerts_loki:
-    template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION}
-    file: alerts/loki.yml.tmpl
 
 volumes:
   grafana-data:

compose.loki.yml

@@ -19,16 +19,9 @@ services:
       - LOKI_AWS_REGION
       - LOKI_BUCKET_NAMES
       - LOKI_RETENTION_PERIOD
-      - LOKI_LOG_LEVEL
       - STACK_NAME
       - LOKI_STORAGE_FILESYSTEM
       - LOKI_STORAGE_S3
-    healthcheck:
-      test: ["CMD", "/usr/bin/loki", "-health"]
-      interval: 10s
-      timeout: 10s
-      retries: 10
-      start_period: 30s
     deploy:
       restart_policy:
         condition: on-failure
@@ -41,8 +34,6 @@ services:
         - "traefik.http.routers.${STACK_NAME}-loki.tls=true"
         - "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
-        - "prometheus.io/scrape=true"
-        - "prometheus.io/port=3100"
 
 
 configs:

loki.yml.tmpl

@@ -15,7 +15,7 @@ ruler:
 
 server:
   http_listen_port: 3100
-  log_level: {{ or (env "LOKI_LOG_LEVEL") "warn" }}
+  log_level: warn
 
 distributor:
   ring:
@@ -127,4 +127,4 @@ frontend:
 
 table_manager:
   retention_deletes_enabled: false
-  retention_period: 0s
+  retention_period: 0s