feat: improve loki debugging

another one! closes #15

again, the compose-snippet was generated using Claude, but verified and tested manually.
Thought of using a socket-proxy just as traefik does, but since alloy has socket access anyways and the official docker image should be as safe as it gets, I think this should be alright.

## Loki log preview
![grafik.png](/attachments/013605e9-9abc-47ef-8cb9-c118fbaec3b5)

Reviewed-on: #32
Reviewed-by: moritz <98+moritz@noreply.git.coopcloud.tech>
Co-authored-by: Simon <s.thiessen@local-it.org>

.env.sample

@@ -53,6 +53,10 @@ SECRET_BASIC_AUTH_VERSION=v1
 # Not needed if you just want to read local log files — use SYSLOG_FILES instead.
 # SYSLOG=1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml"
+#
+# Scrape Docker-Events via official docker-cli image using docker socket
+# COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
+
 
 # Monitoring Server
 #
@@ -68,6 +72,8 @@ SECRET_BASIC_AUTH_VERSION=v1
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.loki.yml"
 #
+# LOKI_LOG_LEVEL=warn
+#
 # Set to 0 to disable retention
 # LOKI_RETENTION_PERIOD=744h
 # LOKI_STORAGE_FILESYSTEM=1
@@ -127,9 +133,11 @@ SECRET_BASIC_AUTH_VERSION=v1
 # Node memory usage alert will trigger when memory usage is above the given number in percent
 #ALERT_NODE_MEMORY_USAGE=85
 
-# Tell Traefik to keep access logs (could be very verbose). See Access
-# logs on Traefik recipe.
-#ALLOY_ACCESS_LOGS=true
-#PROMETHEUS_ACCESS_LOGS=true
-#LOKI_ACCESS_LOGS=true
-#GRAFANA_ACCESS_LOGS=true
+# Loki ingestion alert will trigger when a host ships fewer than the given number of log entries per second
+#ALERT_LOKI_INGESTION_STALLED=0.01
+
+# Loki write error alert will trigger when log pushes fail at more than the given rate per second
+#ALERT_LOKI_WRITE_ERRORS=0
+
+# Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90
+#ALERT_LOKI_WAL_DISK_USAGE=85

README.md

@@ -54,6 +54,14 @@ This is what a gathering host pushes into. It also runs its own Alloy, so it mon
 
 ## Additional features
 
+### Scrape docker events
+
+Just uncomment the following line in your .env to collect docker events using
+the official docker-cli image and the docker socket:
+```
+COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
+```
+
 ### Discovering metrics from other apps
 
 Alloy auto-discovers and scrapes other Docker Swarm services running on the same host, on the `proxy` network, that opt in via labels. No manual scrape config needed. On the app's `compose.yml`:

abra.sh

@@ -5,12 +5,13 @@ export GF_STACKS_DASH_VERSION=v3
 export GF_TRAEFIK_DASH_VERSION=v3
 export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
-export LOKI_YML_VERSION=v3
+export LOKI_YML_VERSION=v4
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
-export CONFIG_ALLOY_VERSION=v2
+export GF_ALERTS_LOKI_VERSION=v3
+export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the
 # running containers on the server and re-inserting them under the new names.

alerts/loki.yml.tmpl

@@ -0,0 +1,191 @@
+apiVersion: 1
+
+# List of alert rule UIDs that should be deleted
+deleteRules:
+  {{ if not (env "ALERT_LOKI_INGESTION_STALLED") }}
+  - orgId: 1
+    uid: coopcloud_loki_ingestion_stalled
+  {{ end }}
+  {{ if not (env "ALERT_LOKI_WRITE_ERRORS") }}
+  - orgId: 1
+    uid: coopcloud_loki_write_errors
+  {{ end }}
+  {{ if not (env "ALERT_LOKI_WAL_DISK_USAGE") }}
+  - orgId: 1
+    uid: coopcloud_loki_wal_disk_usage
+  {{ end }}
+
+groups:
+- orgId: 1
+  name: loki
+  folder: loki
+  interval: 5m
+  rules:
+    {{ if (env "ALERT_LOKI_INGESTION_STALLED") }}
+    - uid: coopcloud_loki_ingestion_stalled
+      title: Loki Ingestion Stalled
+      condition: C
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 1800
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            expr: sum by (instance) (rate(loki_write_sent_entries_total[15m]))
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: C
+          relativeTimeRange:
+            from: 1800
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_LOKI_INGESTION_STALLED" }}
+                    type: lt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                        - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: C
+            type: threshold
+      noDataState: Alerting
+      execErrState: Error
+      for: 15m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Loki received no logs from {{`{{ $labels.instance }}`}} for 15m ({{`{{ printf "%.4f" (index $values "A").Value }}`}} entries/sec, threshold {{ env "ALERT_LOKI_INGESTION_STALLED" }})
+      isPaused: false
+    {{ end }}
+    {{ if (env "ALERT_LOKI_WRITE_ERRORS") }}
+    - uid: coopcloud_loki_write_errors
+      title: Loki Write Errors
+      condition: C
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 1200
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            expr: sum by (instance, status_code) (rate(loki_write_request_duration_seconds_count{status_code!~"2.."}[10m]))
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: C
+          relativeTimeRange:
+            from: 1200
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_LOKI_WRITE_ERRORS" }}
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                        - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: C
+            type: threshold
+      noDataState: NoData
+      execErrState: Error
+      for: 10m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Loki is rejecting log pushes from {{`{{ $labels.instance }}`}} with HTTP {{`{{ $labels.status_code }}`}} ({{`{{ printf "%.2f" (index $values "A").Value }}`}} failed pushes/sec)
+      isPaused: false
+    {{ end }}
+    {{ if (env "ALERT_LOKI_WAL_DISK_USAGE") }}
+    - uid: coopcloud_loki_wal_disk_usage
+      title: Loki WAL Disk Usage
+      condition: C
+      data:
+        - refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+          datasourceUid: PBFA97CFB590B2093
+          model:
+            editorMode: code
+            expr: max by (instance) (loki_ingester_wal_disk_usage_percent * 100)
+            instant: true
+            intervalMs: 1000
+            legendFormat: __auto
+            maxDataPoints: 43200
+            range: false
+            refId: A
+        - refId: C
+          relativeTimeRange:
+            from: 600
+            to: 0
+          datasourceUid: __expr__
+          model:
+            conditions:
+                - evaluator:
+                    params:
+                        - {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}
+                    type: gt
+                  operator:
+                    type: and
+                  query:
+                    params:
+                        - C
+                  reducer:
+                    params: []
+                    type: last
+                  type: query
+            datasource:
+                type: __expr__
+                uid: __expr__
+            expression: A
+            intervalMs: 1000
+            maxDataPoints: 43200
+            refId: C
+            type: threshold
+      noDataState: NoData
+      execErrState: Error
+      for: 10m
+      annotations:
+        description: ""
+        runbook_url: ""
+        summary: Loki WAL disk {{`{{ printf "%.1f" (index $values "A").Value }}`}}% full on {{`{{ $labels.instance }}`}} (threshold {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}%, writes stop at 90%)
+      isPaused: false
+    {{ end }}

compose.alloy-webui.yml

@@ -14,4 +14,3 @@ services:
         - "traefik.http.routers.${STACK_NAME}-alloy.tls=true"
         - "traefik.http.routers.${STACK_NAME}-alloy.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-alloy.middlewares=basicauth@file"
-        - "traefik.http.routers.${STACK_NAME}-alloy.observability.accesslogs=${ALLOY_ACCESS_LOGS:-false}"

compose.docker-events.yml

@@ -0,0 +1,20 @@
+version: "3.8"
+services:
+  docker-events:
+    image: docker:29.8.1-cli
+    command: >
+      docker events
+      --format '{{json .}}'
+      --filter type=container
+      --filter event=start
+      --filter event=die
+      --filter event=kill
+      --filter event=oom
+      --filter event=restart
+      --filter event=stop
+      --filter event=health_status
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    deploy:
+      restart_policy:
+        condition: on-failure

compose.grafana.yml

@@ -24,6 +24,8 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
+      - source: gf_alerts_loki
+        target: /etc/grafana/provisioning/alerting/loki.yml
     networks:
       - proxy
       - internal
@@ -34,6 +36,9 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_LEFT
       - ALERT_NODE_MEMORY_USAGE
+      - ALERT_LOKI_INGESTION_STALLED
+      - ALERT_LOKI_WRITE_ERRORS
+      - ALERT_LOKI_WAL_DISK_USAGE
     deploy:
       labels:
         - "traefik.enable=true"
@@ -43,7 +48,6 @@ services:
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "traefik.http.routers.${STACK_NAME}-grafana.observability.accesslogs=${GRAFANA_ACCESS_LOGS:-false}"
     healthcheck:
       test: "wget -q http://localhost:3000/healthz -O/dev/null"
       interval: 5s
@@ -78,6 +82,10 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
+  gf_alerts_loki:
+    template_driver: golang
+    name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION}
+    file: alerts/loki.yml.tmpl
 
 volumes:
   grafana-data:

compose.loki.yml

@@ -19,9 +19,16 @@ services:
       - LOKI_AWS_REGION
       - LOKI_BUCKET_NAMES
       - LOKI_RETENTION_PERIOD
+      - LOKI_LOG_LEVEL
       - STACK_NAME
       - LOKI_STORAGE_FILESYSTEM
       - LOKI_STORAGE_S3
+    healthcheck:
+      test: ["CMD", "/usr/bin/loki", "-health"]
+      interval: 10s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
     deploy:
       restart_policy:
         condition: on-failure
@@ -34,7 +41,8 @@ services:
         - "traefik.http.routers.${STACK_NAME}-loki.tls=true"
         - "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
-        - "traefik.http.routers.${STACK_NAME}-loki.observability.accesslogs=${LOKI_ACCESS_LOGS:-false}"
+        - "prometheus.io/scrape=true"
+        - "prometheus.io/port=3100"
 
 
 configs:

compose.prometheus.yml

@@ -38,7 +38,6 @@ services:
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
-        - "traefik.http.routers.${STACK_NAME}-prometheus.observability.accesslogs=${PROMETHEUS_ACCESS_LOGS:-false}"
 
 configs:
   prometheus_yml:

compose.yml

@@ -53,7 +53,7 @@ services:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "coop-cloud.${STACK_NAME}.version=2.0.0+v1.18.1"
+        - "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1"
 configs:
   config_alloy:
     template_driver: golang

config.alloy.tmpl

@@ -52,7 +52,32 @@ prometheus.scrape "default" {
     prometheus.exporter.cadvisor.docker.targets,
   )
 
+  forward_to = [prometheus.relabel.container_meta.receiver]
+}
+
+prometheus.relabel "container_meta" {
   forward_to = [prometheus.remote_write.prometheus.receiver]
+
+  // remove sha tail: nginx:1.31.1@sha256:608a... -> nginx:1.31.1
+  rule {
+    source_labels = ["image"]
+    regex         = "([^@]+)@sha256:.*"
+    target_label  = "image"
+    replacement   = "$1"
+  }
+  // split image in name and tag
+  rule {
+    source_labels = ["image"]
+    regex         = "(.+):[^:/]+"
+    target_label  = "image_name"
+    replacement   = "$1"
+  }
+  rule {
+    source_labels = ["image"]
+    regex         = ".+:([^:/]+)"
+    target_label  = "image_tag"
+    replacement   = "$1"
+  }
 }
 
 prometheus.remote_write "prometheus" {
@@ -290,7 +315,35 @@ loki.source.docker "docker" {
 loki.source.journal "journal" {
   path = "/rootfs/var/log/journal"
   labels = { job = "{{ env "DOMAIN" }}" }
+  relabel_rules = loki.relabel.journal.rules
+  forward_to = [loki.process.journal.receiver]
+}
+
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal_syslog_identifier"]
+    target_label  = "ident"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "priority"
+  }
+}
+
+// drop network db stats (15-20% of kernel logs)
+loki.process "journal" {
   forward_to = [loki.write.loki.receiver]
+
+  stage.drop {
+    expression          = ".*NetworkDB stats.*"
+    drop_counter_reason = "networkdb_stats"
+  }
 }
 {{ end }}
 

loki.yml.tmpl

@@ -15,7 +15,7 @@ ruler:
 
 server:
   http_listen_port: 3100
-  log_level: warn
+  log_level: {{ or (env "LOKI_LOG_LEVEL") "warn" }}
 
 distributor:
   ring:
@@ -127,4 +127,4 @@ frontend:
 
 table_manager:
   retention_deletes_enabled: false
-  retention_period: 0s
+  retention_period: 0s

release/2.1.0+v1.18.1

@@ -0,0 +1 @@
+alloy: split image label in "image_name" and "image_tag", relabel journald logs and drop NetworkDB stats

release/2.2.0+v1.18.1

@@ -0,0 +1 @@
+allow scraping of docker events via new compose.docker-events.yml