diff --git a/.env.sample b/.env.sample index 5e5ef78..fc30d73 100644 --- a/.env.sample +++ b/.env.sample @@ -81,6 +81,12 @@ SECRET_BASIC_AUTH_VERSION=v1 # LOKI_RETENTION_PERIOD=744h # LOKI_STORAGE_FILESYSTEM=1 # +# Opt in to migrating the deprecated boltdb-shipper index store to TSDB, from this date +# onward (YYYY-MM-DD). MUST be a FUTURE date, e.g. the day after you deploy this: a past +# or same-day date makes both stores claim that day's data, breaking queries for it. +# Logs from before this date are still ingested and queried from boltdb-shipper. +# LOKI_TSDB_FROM_DATE=2026-11-01 +# ## S3 Storage # LOKI_STORAGE_S3=1 # LOKI_AWS_ENDPOINT=https://minio.autonomic.zone diff --git a/README.md b/README.md index 902a131..b8cb644 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,18 @@ $ echo '{ $ systemctl restart docker.service ``` +### Upgrading: Loki TSDB migration + +Loki's index store defaults to `boltdb-shipper`, which is deprecated upstream in favour of +TSDB. To migrate an existing deployment without losing access to already-ingested logs, set +`LOKI_TSDB_FROM_DATE` in your `.env` (see `.env.sample`) to a date on or after the day after +your deploy, then `abra app deploy`. Logs from before that date stay on boltdb-shipper and +remain queryable as before; only logs from that date onward are indexed with TSDB. + +**The date must be in the future when you deploy it, never today or earlier**: both stores share the same +per-day table, so a past or same-day date makes them collide over that day's data and +breaks querying it. + ### Setup Push Gateway 1. Enable in the env file by uncommenting the following lines: diff --git a/compose.loki.yml b/compose.loki.yml index 0e4dc73..bd6e937 100644 --- a/compose.loki.yml +++ b/compose.loki.yml @@ -23,6 +23,7 @@ services: - STACK_NAME - LOKI_STORAGE_FILESYSTEM - LOKI_STORAGE_S3 + - LOKI_TSDB_FROM_DATE healthcheck: test: ["CMD", "/usr/bin/loki", "-health"] interval: 10s diff --git a/loki.yml.tmpl b/loki.yml.tmpl index 2d74f1d..f3dded7 100644 --- a/loki.yml.tmpl +++ b/loki.yml.tmpl @@ -56,12 +56,26 @@ schema_config: index: prefix: index_ period: 24h +{{ if env "LOKI_TSDB_FROM_DATE" }} + - from: {{ env "LOKI_TSDB_FROM_DATE" }} + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +{{ end }} storage_config: boltdb_shipper: active_index_directory: /loki/boltdb-shipper-active cache_location: /loki/boltdb-shipper-cache cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space +{{ if env "LOKI_TSDB_FROM_DATE" }} + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache +{{ end }} filesystem: directory: /loki/chunks {{ end }} @@ -74,6 +88,15 @@ schema_config: index: prefix: index_ period: 24h +{{ if env "LOKI_TSDB_FROM_DATE" }} + - from: {{ env "LOKI_TSDB_FROM_DATE" }} + store: tsdb + object_store: aws + schema: v13 + index: + prefix: index_ + period: 24h +{{ end }} storage_config: boltdb_shipper: @@ -81,6 +104,11 @@ storage_config: cache_location: /loki/index_cache resync_interval: 5s shared_store: aws +{{ if env "LOKI_TSDB_FROM_DATE" }} + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache +{{ end }} aws: endpoint: {{ env "LOKI_AWS_ENDPOINT" }} @@ -99,7 +127,7 @@ storage_config: compactor: - working_directory: /loki/boltdb-shipper-compactor + working_directory: /loki/compactor compaction_interval: 10m retention_enabled: true retention_delete_delay: 2h diff --git a/release/next b/release/next index cdd836d..c095073 100644 --- a/release/next +++ b/release/next @@ -1 +1,2 @@ -- alloy and prometheus containers restart policy from on-failure to unless-stopped \ No newline at end of file +- opt-in migration of Loki's index store from deprecated boltdb-shipper to TSDB via LOKI_TSDB_FROM_DATE +- alloy and prometheus containers restart policy from on-failure to unless-stopped