Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 renovate-bot
|
f47a0b7c14 |
-20
@@ -75,18 +75,10 @@ SECRET_BASIC_AUTH_VERSION=v1
|
||||
#
|
||||
# COMPOSE_FILE="$COMPOSE_FILE:compose.loki.yml"
|
||||
#
|
||||
# LOKI_LOG_LEVEL=warn
|
||||
#
|
||||
# Set to 0 to disable retention
|
||||
# LOKI_RETENTION_PERIOD=744h
|
||||
# LOKI_STORAGE_FILESYSTEM=1
|
||||
#
|
||||
# Opt in to migrating the deprecated boltdb-shipper index store to TSDB, from this date
|
||||
# onward (YYYY-MM-DD). MUST be a FUTURE date, e.g. the day after you deploy this: a past
|
||||
# or same-day date makes both stores claim that day's data, breaking queries for it.
|
||||
# Logs from before this date are still ingested and queried from boltdb-shipper.
|
||||
# LOKI_TSDB_FROM_DATE=2026-11-01
|
||||
#
|
||||
## S3 Storage
|
||||
# LOKI_STORAGE_S3=1
|
||||
# LOKI_AWS_ENDPOINT=https://minio.autonomic.zone
|
||||
@@ -141,15 +133,3 @@ SECRET_BASIC_AUTH_VERSION=v1
|
||||
|
||||
# Node memory usage alert will trigger when memory usage is above the given number in percent
|
||||
#ALERT_NODE_MEMORY_USAGE=85
|
||||
|
||||
# Loki ingestion alert will trigger when a host ships fewer than the given number of log entries per second
|
||||
#ALERT_LOKI_INGESTION_STALLED=0.01
|
||||
|
||||
# Loki write error alert will trigger when log pushes fail at more than the given rate per second
|
||||
#ALERT_LOKI_WRITE_ERRORS=0
|
||||
|
||||
# Loki WAL disk alert will trigger when the filesystem holding the WAL is above the given number in percent. Loki throttles all writes at 90
|
||||
#ALERT_LOKI_WAL_DISK_USAGE=85
|
||||
|
||||
# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored
|
||||
#ALERT_TRAEFIK_APDEX=0.5
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
/.envrc
|
||||
*~
|
||||
@@ -130,18 +130,6 @@ $ echo '{
|
||||
$ systemctl restart docker.service
|
||||
```
|
||||
|
||||
### Upgrading: Loki TSDB migration
|
||||
|
||||
Loki's index store defaults to `boltdb-shipper`, which is deprecated upstream in favour of
|
||||
TSDB. To migrate an existing deployment without losing access to already-ingested logs, set
|
||||
`LOKI_TSDB_FROM_DATE` in your `.env` (see `.env.sample`) to a date on or after the day after
|
||||
your deploy, then `abra app deploy`. Logs from before that date stay on boltdb-shipper and
|
||||
remain queryable as before; only logs from that date onward are indexed with TSDB.
|
||||
|
||||
**The date must be in the future when you deploy it, never today or earlier**: both stores share the same
|
||||
per-day table, so a past or same-day date makes them collide over that day's data and
|
||||
breaks querying it.
|
||||
|
||||
### Setup Push Gateway
|
||||
|
||||
1. Enable in the env file by uncommenting the following lines:
|
||||
@@ -186,7 +174,6 @@ It is possible to enable the following alerts, by uncommenting the corresponding
|
||||
|
||||
- node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
|
||||
- node memory usage: `ALERT_NODE_MEMORY_USAGE`
|
||||
- traefik apdex: `ALERT_TRAEFIK_APDEX`
|
||||
|
||||
## smart monitoring
|
||||
|
||||
|
||||
@@ -5,12 +5,11 @@ export GF_STACKS_DASH_VERSION=v3
|
||||
export GF_TRAEFIK_DASH_VERSION=v3
|
||||
export GF_BACKUP_DASH_VERSION=v1
|
||||
export GF_CUSTOM_INI_VERSION=v4
|
||||
export LOKI_YML_VERSION=v4
|
||||
export LOKI_YML_VERSION=v3
|
||||
export PROMETHEUS_YML_VERSION=v2
|
||||
export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
|
||||
export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
|
||||
export GF_ALERTS_NODE_VERSION=v5
|
||||
export GF_ALERTS_LOKI_VERSION=v5
|
||||
export GF_ALERTS_NODE_VERSION=v4
|
||||
export CONFIG_ALLOY_VERSION=v3
|
||||
|
||||
# migrates secrets from old names to new names by reading values from the
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
# List of alert rule UIDs that should be deleted
|
||||
deleteRules:
|
||||
{{ if not (env "ALERT_LOKI_INGESTION_STALLED") }}
|
||||
- orgId: 1
|
||||
uid: coopcloud_loki_ingestion_stalled
|
||||
{{ end }}
|
||||
{{ if not (env "ALERT_LOKI_WRITE_ERRORS") }}
|
||||
- orgId: 1
|
||||
uid: coopcloud_loki_write_errors
|
||||
{{ end }}
|
||||
{{ if not (env "ALERT_LOKI_WAL_DISK_USAGE") }}
|
||||
- orgId: 1
|
||||
uid: coopcloud_loki_wal_disk_usage
|
||||
{{ end }}
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: loki
|
||||
folder: monitoring
|
||||
interval: 5m
|
||||
rules:
|
||||
{{ if (env "ALERT_LOKI_INGESTION_STALLED") }}
|
||||
- uid: coopcloud_loki_ingestion_stalled
|
||||
title: Loki Ingestion Stalled
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 1800
|
||||
to: 0
|
||||
datasourceUid: PBFA97CFB590B2093
|
||||
model:
|
||||
editorMode: code
|
||||
expr: sum by (instance) (rate(loki_write_sent_entries_total[15m]))
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
relativeTimeRange:
|
||||
from: 1800
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- {{ env "ALERT_LOKI_INGESTION_STALLED" }}
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- C
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
noDataState: Alerting
|
||||
execErrState: Error
|
||||
for: 15m
|
||||
annotations:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Loki received no logs from {{`{{ $labels.instance }}`}} for 15m ({{`{{ printf "%.4f" (index $values "A").Value }}`}} entries/sec, threshold {{ env "ALERT_LOKI_INGESTION_STALLED" }})
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
{{ if (env "ALERT_LOKI_WRITE_ERRORS") }}
|
||||
- uid: coopcloud_loki_write_errors
|
||||
title: Loki Write Errors
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 1200
|
||||
to: 0
|
||||
datasourceUid: PBFA97CFB590B2093
|
||||
model:
|
||||
editorMode: code
|
||||
expr: sum by (instance, status_code) (rate(loki_write_request_duration_seconds_count{status_code!~"2.."}[10m]))
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
relativeTimeRange:
|
||||
from: 1200
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- {{ env "ALERT_LOKI_WRITE_ERRORS" }}
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- C
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
annotations:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Loki is rejecting log pushes from {{`{{ $labels.instance }}`}} with HTTP {{`{{ $labels.status_code }}`}} ({{`{{ printf "%.2f" (index $values "A").Value }}`}} failed pushes/sec)
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
{{ if (env "ALERT_LOKI_WAL_DISK_USAGE") }}
|
||||
- uid: coopcloud_loki_wal_disk_usage
|
||||
title: Loki WAL Disk Usage
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: PBFA97CFB590B2093
|
||||
model:
|
||||
editorMode: code
|
||||
expr: max by (instance) (loki_ingester_wal_disk_usage_percent * 100)
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: C
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}
|
||||
type: gt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params:
|
||||
- C
|
||||
reducer:
|
||||
params: []
|
||||
type: last
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: C
|
||||
type: threshold
|
||||
noDataState: NoData
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
annotations:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Loki WAL disk {{`{{ printf "%.1f" (index $values "A").Value }}`}}% full on {{`{{ $labels.instance }}`}} (threshold {{ env "ALERT_LOKI_WAL_DISK_USAGE" }}%, writes stop at 90%)
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
@@ -71,7 +71,8 @@ groups:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Less than {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
|
||||
labels: {}
|
||||
labels:
|
||||
"": ""
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
{{ if (env "ALERT_NODE_MEMORY_USAGE") }}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
# List of alert rule UIDs that should be deleted
|
||||
deleteRules:
|
||||
{{ if not (env "ALERT_TRAEFIK_APDEX") }}
|
||||
- orgId: 1
|
||||
uid: coopcloud_traefik_apdex
|
||||
{{ end }}
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: traefik
|
||||
folder: recipes
|
||||
interval: 5m
|
||||
rules:
|
||||
{{ if (env "ALERT_TRAEFIK_APDEX") }}
|
||||
- uid: coopcloud_traefik_apdex
|
||||
title: Apdex
|
||||
condition: B
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: PBFA97CFB590B2093
|
||||
model:
|
||||
editorMode: code
|
||||
# apdex = (satisfied + tolerating / 2) / total, which reduces to
|
||||
# (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is
|
||||
# cumulative and therefore already contains the satisfied requests.
|
||||
# services with less than 10 requests per 10m are ignored.
|
||||
expr: |-
|
||||
((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) +
|
||||
sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 /
|
||||
sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service))
|
||||
and on(service)
|
||||
(
|
||||
sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10
|
||||
)
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: B
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- {{ env "ALERT_TRAEFIK_APDEX" }}
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: []
|
||||
reducer:
|
||||
params: []
|
||||
type: avg
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: B
|
||||
type: threshold
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
annotations:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}})
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
@@ -1,7 +1,7 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
docker-events:
|
||||
image: docker:29.8.1-cli
|
||||
image: docker:29.9.2-cli
|
||||
command: >
|
||||
docker events
|
||||
--format '{{json .}}'
|
||||
|
||||
+1
-17
@@ -24,10 +24,6 @@ services:
|
||||
target: /var/lib/grafana/dashboards/backup.json
|
||||
- source: gf_alerts_node
|
||||
target: /etc/grafana/provisioning/alerting/node.yml
|
||||
- source: gf_alerts_loki
|
||||
target: /etc/grafana/provisioning/alerting/loki.yml
|
||||
- source: gf_alerts_traefik
|
||||
target: /etc/grafana/provisioning/alerting/traefik.yml
|
||||
networks:
|
||||
- proxy
|
||||
- internal
|
||||
@@ -38,10 +34,6 @@ services:
|
||||
- GF_INSTALL_PLUGINS
|
||||
- ALERT_NODE_DISK_SPACE_LEFT
|
||||
- ALERT_NODE_MEMORY_USAGE
|
||||
- ALERT_LOKI_INGESTION_STALLED
|
||||
- ALERT_LOKI_WRITE_ERRORS
|
||||
- ALERT_LOKI_WAL_DISK_USAGE
|
||||
- ALERT_TRAEFIK_APDEX
|
||||
deploy:
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
@@ -56,7 +48,7 @@ services:
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 300s
|
||||
start_period: 10s
|
||||
|
||||
configs:
|
||||
gf_custom_ini:
|
||||
@@ -85,14 +77,6 @@ configs:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
|
||||
file: alerts/node.yml.tmpl
|
||||
gf_alerts_loki:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_loki_${GF_ALERTS_LOKI_VERSION}
|
||||
file: alerts/loki.yml.tmpl
|
||||
gf_alerts_traefik:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION}
|
||||
file: alerts/traefik.yml.tmpl
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
|
||||
+1
-11
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
loki:
|
||||
image: grafana/loki:3.7.9
|
||||
image: grafana/loki:3.7.2
|
||||
command: -config.file=/etc/loki/local-config.yaml
|
||||
networks:
|
||||
- proxy
|
||||
@@ -19,17 +19,9 @@ services:
|
||||
- LOKI_AWS_REGION
|
||||
- LOKI_BUCKET_NAMES
|
||||
- LOKI_RETENTION_PERIOD
|
||||
- LOKI_LOG_LEVEL
|
||||
- STACK_NAME
|
||||
- LOKI_STORAGE_FILESYSTEM
|
||||
- LOKI_STORAGE_S3
|
||||
- LOKI_TSDB_FROM_DATE
|
||||
healthcheck:
|
||||
test: ["CMD", "/usr/bin/loki", "-health"]
|
||||
interval: 10s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
@@ -42,8 +34,6 @@ services:
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.tls=true"
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
|
||||
- "prometheus.io/scrape=true"
|
||||
- "prometheus.io/port=3100"
|
||||
|
||||
|
||||
configs:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.15.1
|
||||
image: prom/prometheus:v3.15.0
|
||||
secrets:
|
||||
- basic_auth
|
||||
volumes:
|
||||
@@ -28,7 +28,7 @@ services:
|
||||
start_period: 30s
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: unless-stopped
|
||||
condition: on-failure
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.swarm.network=proxy"
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ services:
|
||||
start_period: 30s
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: unless-stopped
|
||||
condition: on-failure
|
||||
labels:
|
||||
- "backupbot.backup=${ENABLE_BACKUPS:-true}"
|
||||
- "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1"
|
||||
|
||||
+3
-31
@@ -15,7 +15,7 @@ ruler:
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
log_level: {{ or (env "LOKI_LOG_LEVEL") "warn" }}
|
||||
log_level: warn
|
||||
|
||||
distributor:
|
||||
ring:
|
||||
@@ -56,26 +56,12 @@ schema_config:
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
{{ if env "LOKI_TSDB_FROM_DATE" }}
|
||||
- from: {{ env "LOKI_TSDB_FROM_DATE" }}
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
{{ end }}
|
||||
|
||||
storage_config:
|
||||
boltdb_shipper:
|
||||
active_index_directory: /loki/boltdb-shipper-active
|
||||
cache_location: /loki/boltdb-shipper-cache
|
||||
cache_ttl: 24h # Can be increased for faster performance over longer query periods, uses more disk space
|
||||
{{ if env "LOKI_TSDB_FROM_DATE" }}
|
||||
tsdb_shipper:
|
||||
active_index_directory: /loki/tsdb-index
|
||||
cache_location: /loki/tsdb-cache
|
||||
{{ end }}
|
||||
filesystem:
|
||||
directory: /loki/chunks
|
||||
{{ end }}
|
||||
@@ -88,15 +74,6 @@ schema_config:
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
{{ if env "LOKI_TSDB_FROM_DATE" }}
|
||||
- from: {{ env "LOKI_TSDB_FROM_DATE" }}
|
||||
store: tsdb
|
||||
object_store: aws
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
{{ end }}
|
||||
|
||||
storage_config:
|
||||
boltdb_shipper:
|
||||
@@ -104,11 +81,6 @@ storage_config:
|
||||
cache_location: /loki/index_cache
|
||||
resync_interval: 5s
|
||||
shared_store: aws
|
||||
{{ if env "LOKI_TSDB_FROM_DATE" }}
|
||||
tsdb_shipper:
|
||||
active_index_directory: /loki/tsdb-index
|
||||
cache_location: /loki/tsdb-cache
|
||||
{{ end }}
|
||||
|
||||
aws:
|
||||
endpoint: {{ env "LOKI_AWS_ENDPOINT" }}
|
||||
@@ -127,7 +99,7 @@ storage_config:
|
||||
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
working_directory: /loki/boltdb-shipper-compactor
|
||||
compaction_interval: 10m
|
||||
retention_enabled: true
|
||||
retention_delete_delay: 2h
|
||||
@@ -155,4 +127,4 @@ frontend:
|
||||
|
||||
table_manager:
|
||||
retention_deletes_enabled: false
|
||||
retention_period: 0s
|
||||
retention_period: 0s
|
||||
@@ -1,2 +0,0 @@
|
||||
- opt-in migration of Loki's index store from deprecated boltdb-shipper to TSDB via LOKI_TSDB_FROM_DATE
|
||||
- alloy and prometheus containers restart policy from on-failure to unless-stopped
|
||||
Reference in New Issue
Block a user