feat: opt-in Loki index store migration from deprecated boltdb-shipper to TSDB

boltdb-shipper is deprecated and will be removed in Loki 4.0.
Setting LOKI_TSDB_FROM_DATE adds a second schema_config period (store:
tsdb) from that date onward, alongside the existing boltdb-shipper
period, so logs before the cutover stay queryable under the old store
while new logs are indexed with TSDB.

.env.sample

@@ -79,6 +79,12 @@ SECRET_BASIC_AUTH_VERSION=v1
 # LOKI_RETENTION_PERIOD=744h
 # LOKI_STORAGE_FILESYSTEM=1
 #
+# Opt in to migrating the deprecated boltdb-shipper index store to TSDB, from this date
+# onward (YYYY-MM-DD). MUST be a FUTURE date, e.g. the day after you deploy this: a past
+# or same-day date makes both stores claim that day's data, breaking queries for it.
+# Logs from before this date are still ingested and queried from boltdb-shipper.
+# LOKI_TSDB_FROM_DATE=2026-11-01
+#
 ## S3 Storage
 # LOKI_STORAGE_S3=1
 # LOKI_AWS_ENDPOINT=https://minio.autonomic.zone
@@ -133,6 +139,3 @@ SECRET_BASIC_AUTH_VERSION=v1
 
 # Node memory usage alert will trigger when memory usage is above the given number in percent
 #ALERT_NODE_MEMORY_USAGE=85
-
-# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored
-#ALERT_TRAEFIK_APDEX=0.5

README.md

@@ -130,6 +130,18 @@ $ echo '{
 $ systemctl restart docker.service
 ```
 
+### Upgrading: Loki TSDB migration
+
+Loki's index store defaults to `boltdb-shipper`, which is deprecated upstream in favour of
+TSDB. To migrate an existing deployment without losing access to already-ingested logs, set
+`LOKI_TSDB_FROM_DATE` in your `.env` (see `.env.sample`) to a date on or after the day after
+your deploy, then `abra app deploy`. Logs from before that date stay on boltdb-shipper and
+remain queryable as before; only logs from that date onward are indexed with TSDB.
+
+**The date must be in the future when you deploy it, never today or earlier**: both stores share the same
+per-day table, so a past or same-day date makes them collide over that day's data and
+breaks querying it.
+
 ### Setup Push Gateway
 
 1. Enable in the env file by uncommenting the following lines:
@@ -174,7 +186,6 @@ It is possible to enable the following alerts, by uncommenting the corresponding
 
 - node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
 - node memory usage: `ALERT_NODE_MEMORY_USAGE`
-- traefik apdex: `ALERT_TRAEFIK_APDEX`
 
 ## smart monitoring
 

abra.sh

@@ -5,12 +5,11 @@ export GF_STACKS_DASH_VERSION=v3
 export GF_TRAEFIK_DASH_VERSION=v3
 export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
-export LOKI_YML_VERSION=v3
+export LOKI_YML_VERSION=v4
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
-export GF_ALERTS_NODE_VERSION=v3
-export GF_ALERTS_TRAEFIK_VERSION=v1
+export GF_ALERTS_NODE_VERSION=v4
 export CONFIG_ALLOY_VERSION=v3
 
 # migrates secrets from old names to new names by reading values from the

alerts/node.yml.tmpl

@@ -87,7 +87,7 @@ groups:
           datasourceUid: PBFA97CFB590B2093
           model:
             editorMode: code
-            expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100
+            expr: 100 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100)
             instant: true
             intervalMs: 1000
             legendFormat: __auto

alerts/traefik.yml.tmpl

@@ -1,81 +0,0 @@
-apiVersion: 1
-
-# List of alert rule UIDs that should be deleted
-deleteRules:
-  {{ if not (env "ALERT_TRAEFIK_APDEX") }}
-  - orgId: 1
-    uid: coopcloud_traefik_apdex
-  {{ end }}
-
-groups:
-- orgId: 1
-  name: traefik
-  folder: recipes
-  interval: 5m
-  rules:
-    {{ if (env "ALERT_TRAEFIK_APDEX") }}
-    - uid: coopcloud_traefik_apdex
-      title: Apdex
-      condition: B
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            # apdex = (satisfied + tolerating / 2) / total, which reduces to
-            # (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is
-            # cumulative and therefore already contains the satisfied requests.
-            # services with less than 10 requests per 10m are ignored.
-            expr: |-
-              ((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) +
-               sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 /
-               sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service))
-              and on(service)
-              (
-                sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10
-              )
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: B
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - {{ env "ALERT_TRAEFIK_APDEX" }}
-                    type: lt
-                  operator:
-                    type: and
-                  query:
-                    params: []
-                  reducer:
-                    params: []
-                    type: avg
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: B
-            type: threshold
-      noDataState: OK
-      execErrState: Error
-      for: 10m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}})
-      isPaused: false
-    {{ end }}

compose.grafana.yml

@@ -24,8 +24,6 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
-      - source: gf_alerts_traefik
-        target: /etc/grafana/provisioning/alerting/traefik.yml
     networks:
       - proxy
       - internal
@@ -36,7 +34,6 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_LEFT
       - ALERT_NODE_MEMORY_USAGE
-      - ALERT_TRAEFIK_APDEX
     deploy:
       labels:
         - "traefik.enable=true"
@@ -80,10 +77,6 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
-  gf_alerts_traefik:
-    template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION}
-    file: alerts/traefik.yml.tmpl
 
 volumes:
   grafana-data:

compose.loki.yml

@@ -22,6 +22,7 @@ services:
       - STACK_NAME
       - LOKI_STORAGE_FILESYSTEM
       - LOKI_STORAGE_S3
+      - LOKI_TSDB_FROM_DATE
     deploy:
       restart_policy:
         condition: on-failure

loki.yml.tmpl

@@ -56,12 +56,26 @@ schema_config:
       index:
         prefix: index_
         period: 24h
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+    - from: {{ env "LOKI_TSDB_FROM_DATE" }}
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+{{ end }}
 
 storage_config:
   boltdb_shipper:
     active_index_directory: /loki/boltdb-shipper-active
     cache_location: /loki/boltdb-shipper-cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+{{ end }}
   filesystem:
     directory: /loki/chunks
 {{ end }}
@@ -74,6 +88,15 @@ schema_config:
     index:
       prefix: index_
       period: 24h
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+  - from: {{ env "LOKI_TSDB_FROM_DATE" }}
+    store: tsdb
+    object_store: aws
+    schema: v13
+    index:
+      prefix: index_
+      period: 24h
+{{ end }}
 
 storage_config:
  boltdb_shipper:
@@ -81,6 +104,11 @@ storage_config:
    cache_location: /loki/index_cache
    resync_interval: 5s
    shared_store: aws
+{{ if env "LOKI_TSDB_FROM_DATE" }}
+ tsdb_shipper:
+   active_index_directory: /loki/tsdb-index
+   cache_location: /loki/tsdb-cache
+{{ end }}
 
  aws:
    endpoint: {{ env "LOKI_AWS_ENDPOINT" }}
@@ -99,7 +127,7 @@ storage_config:
 
 
 compactor:
-  working_directory: /loki/boltdb-shipper-compactor
+  working_directory: /loki/compactor
   compaction_interval: 10m
   retention_enabled: true
   retention_delete_delay: 2h

release/next

@@ -0,0 +1 @@
+- opt-in migration of Loki's index store from deprecated boltdb-shipper to TSDB via LOKI_TSDB_FROM_DATE