feat(matrix): show more information in matrix alerts

Reviewed-on: #11

.env.sample

@@ -53,6 +53,10 @@ SECRET_BASIC_AUTH_VERSION=v1
 # Not needed if you just want to read local log files — use SYSLOG_FILES instead.
 # SYSLOG=1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml"
+#
+# Scrape Docker-Events via official docker-cli image using docker socket
+# COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
+
 
 # Monitor physical disks health
 # COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml"

.gitea/CODEOWNERS

@@ -0,0 +1 @@
+* @coop-cloud/monitoring-ng-maintainers

MAINTENANCE.md

@@ -0,0 +1,35 @@
+# Monitoring NG Recipe Maintenance
+
+All contributions should be made via a pull request. This is to ensure a
+certain quality and consistency, that others can rely on.
+
+## Maintainer Responsibilities
+
+A recipe maintainer has the following responsibilities:
+
+- Respond to pull requests / issues within two weeks
+- Make image security updates within a week
+- Make image major updates every three months
+
+In order to fullfill these responsibilities a recipe maintainer:
+
+- Has to watch the repository (to get notifications)
+- Needs to make sure renovate is configured properly
+
+## Pull Requests
+
+A pull request can be merged if it is approved by at least one maintainer. For
+pull requests opened by a maintainer they need to be approved by another
+maintainer. Even though it is okay to merge a pull request with one approval, it
+is always better if all maintainers looked at the pull request and approved it.
+
+## Become a maintainer
+
+Everyone can apply to be a recipe maintainer:
+1. Watch the repository to always get updates
+2. Simply add your self to the list in the [README.md](./README.md) and open a new pull request with the change.
+3. Once the pull request gets merged you will be added to the
+   [monitoring-ng maintainers
+   team](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers).
+4. Join the room
+   [#cc-|-monitoring-ng-maintenance:matrix.org](#cc-|-monitoring-ng-maintenance:matrix.org) and chat to other maintainers.

README.md

@@ -7,7 +7,7 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou
 Metrics and logs are collected by [Grafana Alloy](https://grafana.com/docs/alloy/latest/) and pushed to a central Prometheus/Loki (via `remote_write`/`loki push`). Every `monitoring-ng` instance runs its own Alloy, whether or not it also runs the central stack.
 
 <!-- metadata -->
-
+- **Maintainer**: [@coop-cloud/monitoring-ng-maintainers](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers)
 - **Category**: Apps
 - **Status**: 2, beta
 - **Image**: [`grafana/grafana`](https://hub.docker.com/r/grafana/grafana), 4, upstream
@@ -54,6 +54,14 @@ This is what a gathering host pushes into. It also runs its own Alloy, so it mon
 
 ## Additional features
 
+### Scrape docker events
+
+Just uncomment the following line in your .env to collect docker events using
+the official docker-cli image and the docker socket:
+```
+COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
+```
+
 ### Discovering metrics from other apps
 
 Alloy auto-discovers and scrapes other Docker Swarm services running on the same host, on the `proxy` network, that opt in via labels. No manual scrape config needed. On the app's `compose.yml`:

abra.sh

@@ -7,10 +7,10 @@ export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
 export LOKI_YML_VERSION=v3
 export PROMETHEUS_YML_VERSION=v2
-export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=v2
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v3
-export CONFIG_ALLOY_VERSION=v2
+export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the
 # running containers on the server and re-inserting them under the new names.

alertmanager-matrix-config.yml.tmpl

@@ -54,21 +54,35 @@ templating:
       {{ else if .Alert.Labels.alertname }}
         {{ .Alert.Labels.alertname }}
       {{ end }}
+      {{ if .Alert.Labels.rulename }}
+        &rarr; <strong>{{ .Alert.Labels.rulename }}</strong>
+      {{ end }}
+      {{ if .Alert.Labels.grafana_folder }}
+        <em>({{ .Alert.Labels.grafana_folder }})</em>
+      {{ end }}
       >>
       {{ if .Alert.Labels.severity }}
-        {{ .Alert.Labels.severity | ToUpper }}: 
+        {{ .Alert.Labels.severity | ToUpper }}:
       {{ end }}
       {{ if .Alert.Annotations.description }}
         {{ .Alert.Annotations.description }}
       {{ else if .Alert.Annotations.summary }}
         {{ .Alert.Annotations.summary }}
+      {{ else if .Alert.Labels.rulename }}
+        Query {{ .Alert.Labels.ref_id }} returned no data (datasource {{ .Alert.Labels.datasource_uid }})
+      {{ end }}
+      {{ if .Alert.Labels.instance }}
+        on {{ .Alert.Labels.instance }}
       {{ end }}
       >>
       {{ if .Alert.Annotations.runbook }}
-        <a href="{{ .Alert.Annotations.runbook }}">Runbook</a> | 
+        <a href="{{ .Alert.Annotations.runbook }}">Runbook</a> |
       {{ end }}
       {{ if .Alert.Annotations.dashboard }}
-        <a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> | 
+        <a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> |
+      {{ end }}
+      {{ if .GeneratorURL }}
+        <a href="{{ .GeneratorURL }}">Rule</a> |
       {{ end }}
       <a href="{{ .SilenceURL }}">Silence</a>
     </p>`}}'

compose.docker-events.yml

@@ -0,0 +1,20 @@
+version: "3.8"
+services:
+  docker-events:
+    image: docker:29.8.1-cli
+    command: >
+      docker events
+      --format '{{json .}}'
+      --filter type=container
+      --filter event=start
+      --filter event=die
+      --filter event=kill
+      --filter event=oom
+      --filter event=restart
+      --filter event=stop
+      --filter event=health_status
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    deploy:
+      restart_policy:
+        condition: on-failure

compose.yml

@@ -46,14 +46,14 @@ services:
       test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/12345 && printf 'GET /-/ready HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && head -1 <&3 | grep -q 200"]
       interval: 5s
       timeout: 10s
-      retries: 3
-      start_period: 10s
+      retries: 10
+      start_period: 30s
     deploy:
       restart_policy:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "coop-cloud.${STACK_NAME}.version=2.0.0+v1.18.1"
+        - "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1"
 configs:
   config_alloy:
     template_driver: golang

config.alloy.tmpl

@@ -52,7 +52,32 @@ prometheus.scrape "default" {
     prometheus.exporter.cadvisor.docker.targets,
   )
 
+  forward_to = [prometheus.relabel.container_meta.receiver]
+}
+
+prometheus.relabel "container_meta" {
   forward_to = [prometheus.remote_write.prometheus.receiver]
+
+  // remove sha tail: nginx:1.31.1@sha256:608a... -> nginx:1.31.1
+  rule {
+    source_labels = ["image"]
+    regex         = "([^@]+)@sha256:.*"
+    target_label  = "image"
+    replacement   = "$1"
+  }
+  // split image in name and tag
+  rule {
+    source_labels = ["image"]
+    regex         = "(.+):[^:/]+"
+    target_label  = "image_name"
+    replacement   = "$1"
+  }
+  rule {
+    source_labels = ["image"]
+    regex         = ".+:([^:/]+)"
+    target_label  = "image_tag"
+    replacement   = "$1"
+  }
 }
 
 prometheus.remote_write "prometheus" {
@@ -290,7 +315,35 @@ loki.source.docker "docker" {
 loki.source.journal "journal" {
   path = "/rootfs/var/log/journal"
   labels = { job = "{{ env "DOMAIN" }}" }
+  relabel_rules = loki.relabel.journal.rules
+  forward_to = [loki.process.journal.receiver]
+}
+
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+  rule {
+    source_labels = ["__journal_syslog_identifier"]
+    target_label  = "ident"
+  }
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "priority"
+  }
+}
+
+// drop network db stats (15-20% of kernel logs)
+loki.process "journal" {
   forward_to = [loki.write.loki.receiver]
+
+  stage.drop {
+    expression          = ".*NetworkDB stats.*"
+    drop_counter_reason = "networkdb_stats"
+  }
 }
 {{ end }}
 

release/2.1.0+v1.18.1

@@ -0,0 +1 @@
+alloy: split image label in "image_name" and "image_tag", relabel journald logs and drop NetworkDB stats

release/2.2.0+v1.18.1

@@ -0,0 +1 @@
+allow scraping of docker events via new compose.docker-events.yml

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}