chore: publish 2.4.0+v1.21.3 release

Reviewed-on: #60
Reviewed-by: fauno <fauno@sutty.coop.ar>

.env.sample

@@ -81,6 +81,12 @@ SECRET_BASIC_AUTH_VERSION=v1
 # LOKI_RETENTION_PERIOD=744h
 # LOKI_STORAGE_FILESYSTEM=1
 #
+# Opt in to migrating the deprecated boltdb-shipper index store to TSDB, from this date
+# onward (YYYY-MM-DD). MUST be a FUTURE date, e.g. the day after you deploy this: a past
+# or same-day date makes both stores claim that day's data, breaking queries for it.
+# Logs from before this date are still ingested and queried from boltdb-shipper.
+# LOKI_TSDB_FROM_DATE=2026-11-01
+#
 ## S3 Storage
 # LOKI_STORAGE_S3=1
 # LOKI_AWS_ENDPOINT=https://minio.autonomic.zone

README.md

@@ -130,6 +130,18 @@ $ echo '{
 $ systemctl restart docker.service
 ```
 
+### Upgrading: Loki TSDB migration
+
+Loki's index store defaults to `boltdb-shipper`, which is deprecated upstream in favour of
+TSDB. To migrate an existing deployment without losing access to already-ingested logs, set
+`LOKI_TSDB_FROM_DATE` in your `.env` (see `.env.sample`) to a date on or after the day after
+your deploy, then `abra app deploy`. Logs from before that date stay on boltdb-shipper and
+remain queryable as before; only logs from that date onward are indexed with TSDB.
+
+**The date must be in the future when you deploy it, never today or earlier**: both stores share the same
+per-day table, so a past or same-day date makes them collide over that day's data and
+breaks querying it.
+
 ### Setup Push Gateway
 
 1. Enable in the env file by uncommenting the following lines:
@@ -154,19 +166,28 @@ THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal
 ```
 COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
 ```
+2. Create a matrix bot account and get an access token:
 
-2. Insert the matrix access token secret:
+```
+abra app run matrix.example.com app -- sh -c 'register_new_matrix_user -u <botusername> --no-admin -k "$(cat /var/run/secrets/registration)" http://localhost:8008'
+curl -XPOST --json '{"type": "m.login.password", "identifier": {"user": "botusername", "type": "m.id.user"}, "password": "passwordforuser"}' "https://matrix.example.com/_matrix/client/v3/login" 
+```
+
+3. Insert the matrix access token secret:
 ```
 abra app secret insert monitoring.marx.klasse-methode.it matrix_token v1
 ```
 
-3. Set required configurations:
+4. Set required configurations:
 ```
-GF_MATRIX_USER_ID=
-GF_MATRIX_ROOM_ID=
-GF_MATRIX_HOME_SERVER_URL=
+GF_MATRIX_USER_ID=<botusername>
+GF_MATRIX_ROOM_ID=<room-id>
+GF_MATRIX_HOME_SERVER_URL=matrix.example.com
 ```
-4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`
+5. Configure Alertmanager webhook:
+  - Create contact point under Grafana → Alerting → Contact Points → New Contact Point
+  - Set Integration to "Webhook"
+  - Set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`
 
 ### Alerts
 

abra.sh

@@ -7,11 +7,12 @@ export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
 export LOKI_YML_VERSION=v4
 export PROMETHEUS_YML_VERSION=v2
-export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=v2
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
 export GF_ALERTS_NODE_VERSION=v5
 export GF_ALERTS_LOKI_VERSION=v5
 export CONFIG_ALLOY_VERSION=v3
+export GF_ALERTS_TRAEFIK_VERSION=v1
 
 # migrates secrets from old names to new names by reading values from the
 # running containers on the server and re-inserting them under the new names.

alertmanager-matrix-config.yml.tmpl

@@ -54,21 +54,35 @@ templating:
       {{ else if .Alert.Labels.alertname }}
         {{ .Alert.Labels.alertname }}
       {{ end }}
+      {{ if .Alert.Labels.rulename }}
+        &rarr; <strong>{{ .Alert.Labels.rulename }}</strong>
+      {{ end }}
+      {{ if .Alert.Labels.grafana_folder }}
+        <em>({{ .Alert.Labels.grafana_folder }})</em>
+      {{ end }}
       >>
       {{ if .Alert.Labels.severity }}
-        {{ .Alert.Labels.severity | ToUpper }}: 
+        {{ .Alert.Labels.severity | ToUpper }}:
       {{ end }}
       {{ if .Alert.Annotations.description }}
         {{ .Alert.Annotations.description }}
       {{ else if .Alert.Annotations.summary }}
         {{ .Alert.Annotations.summary }}
+      {{ else if .Alert.Labels.rulename }}
+        Query {{ .Alert.Labels.ref_id }} returned no data (datasource {{ .Alert.Labels.datasource_uid }})
+      {{ end }}
+      {{ if .Alert.Labels.instance }}
+        on {{ .Alert.Labels.instance }}
       {{ end }}
       >>
       {{ if .Alert.Annotations.runbook }}
-        <a href="{{ .Alert.Annotations.runbook }}">Runbook</a> | 
+        <a href="{{ .Alert.Annotations.runbook }}">Runbook</a> |
       {{ end }}
       {{ if .Alert.Annotations.dashboard }}
-        <a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> | 
+        <a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> |
+      {{ end }}
+      {{ if .GeneratorURL }}
+        <a href="{{ .GeneratorURL }}">Rule</a> |
       {{ end }}
       <a href="{{ .SilenceURL }}">Silence</a>
     </p>`}}'

compose.grafana.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:13.2.3
+    image: grafana/grafana:13.3.0
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:

compose.loki.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   loki:
-    image: grafana/loki:3.7.2
+    image: grafana/loki:3.7.9
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - proxy
@@ -23,6 +23,7 @@ services:
       - STACK_NAME
       - LOKI_STORAGE_FILESYSTEM
       - LOKI_STORAGE_S3
+      - LOKI_TSDB_FROM_DATE
     healthcheck:
       test: ["CMD", "/usr/bin/loki", "-health"]
       interval: 10s

compose.prometheus.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v3.15.1
+    image: prom/prometheus:v3.16.0
     secrets:
       - basic_auth
     volumes:
@@ -28,7 +28,7 @@ services:
       start_period: 30s
     deploy:
       restart_policy:
-        condition: unless-stopped
+        condition: any
       labels:
         - "traefik.enable=true"
         - "traefik.swarm.network=proxy"

compose.smartctl.yml

@@ -5,9 +5,9 @@ services:
     image: "prometheuscommunity/smartctl-exporter:v0.15.0"
     volumes:
     - "/var/lib/smartmontools/json:/debug"
+    - "/dev:/dev:ro"
     command:
     - "--smartctl.fake-data"
-    - "--smartctl.interval=1h"
     networks:
     - "proxy"
     deploy:

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: grafana/alloy:v1.20.1
+    image: grafana/alloy:v1.21.3
     hostname: "${DOMAIN}"
     configs:
       - source: config_alloy
@@ -50,10 +50,10 @@ services:
       start_period: 30s
     deploy:
       restart_policy:
-        condition: unless-stopped
+        condition: any
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1"
+        - "coop-cloud.${STACK_NAME}.version=2.4.0+v1.21.3"
 configs:
   config_alloy:
     template_driver: golang

loki.yml.tmpl

@@ -56,12 +56,26 @@ schema_config:
       index:
         prefix: index_
         period: 24h
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+    - from: {{ env "LOKI_TSDB_FROM_DATE" }}
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+{{ end }}
 
 storage_config:
   boltdb_shipper:
     active_index_directory: /loki/boltdb-shipper-active
     cache_location: /loki/boltdb-shipper-cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+{{ end }}
   filesystem:
     directory: /loki/chunks
 {{ end }}
@@ -74,6 +88,15 @@ schema_config:
     index:
       prefix: index_
       period: 24h
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+  - from: {{ env "LOKI_TSDB_FROM_DATE" }}
+    store: tsdb
+    object_store: aws
+    schema: v13
+    index:
+      prefix: index_
+      period: 24h
+{{ end }}
 
 storage_config:
  boltdb_shipper:
@@ -81,6 +104,11 @@ storage_config:
    cache_location: /loki/index_cache
    resync_interval: 5s
    shared_store: aws
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+ tsdb_shipper:
+   active_index_directory: /loki/tsdb-index
+   cache_location: /loki/tsdb-cache
+{{ end }}
 
  aws:
    endpoint: {{ env "LOKI_AWS_ENDPOINT" }}
@@ -99,7 +127,7 @@ storage_config:
 
 
 compactor:
-  working_directory: /loki/boltdb-shipper-compactor
+  working_directory: /loki/compactor
   compaction_interval: 10m
   retention_enabled: true
   retention_delete_delay: 2h

release/2.3.0+v1.21.3

@@ -0,0 +1,2 @@
+- opt-in migration of Loki's index store from deprecated boltdb-shipper to TSDB via LOKI_TSDB_FROM_DATE
+- alloy and prometheus containers restart policy from on-failure to any

release/2.3.2+v1.21.3

@@ -0,0 +1 @@
+smartctl exporter works again!

release/next

@@ -1 +0,0 @@
-- alloy and prometheus containers restart policy from on-failure to unless-stopped