Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 p4u1
|
034e635631 |
@@ -133,6 +133,3 @@ SECRET_BASIC_AUTH_VERSION=v1
|
||||
|
||||
# Node memory usage alert will trigger when memory usage is above the given number in percent
|
||||
#ALERT_NODE_MEMORY_USAGE=85
|
||||
|
||||
# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored
|
||||
#ALERT_TRAEFIK_APDEX=0.5
|
||||
|
||||
@@ -174,7 +174,6 @@ It is possible to enable the following alerts, by uncommenting the corresponding
|
||||
|
||||
- node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
|
||||
- node memory usage: `ALERT_NODE_MEMORY_USAGE`
|
||||
- traefik apdex: `ALERT_TRAEFIK_APDEX`
|
||||
|
||||
## smart monitoring
|
||||
|
||||
|
||||
@@ -7,10 +7,9 @@ export GF_BACKUP_DASH_VERSION=v1
|
||||
export GF_CUSTOM_INI_VERSION=v4
|
||||
export LOKI_YML_VERSION=v3
|
||||
export PROMETHEUS_YML_VERSION=v2
|
||||
export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
|
||||
export MATRIX_ALERTMANAGER_CONFIG_VERSION=v2
|
||||
export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
|
||||
export GF_ALERTS_NODE_VERSION=v3
|
||||
export GF_ALERTS_TRAEFIK_VERSION=v1
|
||||
export CONFIG_ALLOY_VERSION=v3
|
||||
|
||||
# migrates secrets from old names to new names by reading values from the
|
||||
|
||||
@@ -54,21 +54,35 @@ templating:
|
||||
{{ else if .Alert.Labels.alertname }}
|
||||
{{ .Alert.Labels.alertname }}
|
||||
{{ end }}
|
||||
{{ if .Alert.Labels.rulename }}
|
||||
→ <strong>{{ .Alert.Labels.rulename }}</strong>
|
||||
{{ end }}
|
||||
{{ if .Alert.Labels.grafana_folder }}
|
||||
<em>({{ .Alert.Labels.grafana_folder }})</em>
|
||||
{{ end }}
|
||||
>>
|
||||
{{ if .Alert.Labels.severity }}
|
||||
{{ .Alert.Labels.severity | ToUpper }}:
|
||||
{{ .Alert.Labels.severity | ToUpper }}:
|
||||
{{ end }}
|
||||
{{ if .Alert.Annotations.description }}
|
||||
{{ .Alert.Annotations.description }}
|
||||
{{ else if .Alert.Annotations.summary }}
|
||||
{{ .Alert.Annotations.summary }}
|
||||
{{ else if .Alert.Labels.rulename }}
|
||||
Query {{ .Alert.Labels.ref_id }} returned no data (datasource {{ .Alert.Labels.datasource_uid }})
|
||||
{{ end }}
|
||||
{{ if .Alert.Labels.instance }}
|
||||
on {{ .Alert.Labels.instance }}
|
||||
{{ end }}
|
||||
>>
|
||||
{{ if .Alert.Annotations.runbook }}
|
||||
<a href="{{ .Alert.Annotations.runbook }}">Runbook</a> |
|
||||
<a href="{{ .Alert.Annotations.runbook }}">Runbook</a> |
|
||||
{{ end }}
|
||||
{{ if .Alert.Annotations.dashboard }}
|
||||
<a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> |
|
||||
<a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> |
|
||||
{{ end }}
|
||||
{{ if .GeneratorURL }}
|
||||
<a href="{{ .GeneratorURL }}">Rule</a> |
|
||||
{{ end }}
|
||||
<a href="{{ .SilenceURL }}">Silence</a>
|
||||
</p>`}}'
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
# List of alert rule UIDs that should be deleted
|
||||
deleteRules:
|
||||
{{ if not (env "ALERT_TRAEFIK_APDEX") }}
|
||||
- orgId: 1
|
||||
uid: coopcloud_traefik_apdex
|
||||
{{ end }}
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: traefik
|
||||
folder: recipes
|
||||
interval: 5m
|
||||
rules:
|
||||
{{ if (env "ALERT_TRAEFIK_APDEX") }}
|
||||
- uid: coopcloud_traefik_apdex
|
||||
title: Apdex
|
||||
condition: B
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: PBFA97CFB590B2093
|
||||
model:
|
||||
editorMode: code
|
||||
# apdex = (satisfied + tolerating / 2) / total, which reduces to
|
||||
# (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is
|
||||
# cumulative and therefore already contains the satisfied requests.
|
||||
# services with less than 10 requests per 10m are ignored.
|
||||
expr: |-
|
||||
((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) +
|
||||
sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 /
|
||||
sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service))
|
||||
and on(service)
|
||||
(
|
||||
sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10
|
||||
)
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: B
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- {{ env "ALERT_TRAEFIK_APDEX" }}
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: []
|
||||
reducer:
|
||||
params: []
|
||||
type: avg
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: B
|
||||
type: threshold
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
annotations:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}})
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
+1
-8
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
grafana:
|
||||
image: grafana/grafana:13.2.3
|
||||
image: grafana/grafana:13.0.6
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana:rw
|
||||
secrets:
|
||||
@@ -24,8 +24,6 @@ services:
|
||||
target: /var/lib/grafana/dashboards/backup.json
|
||||
- source: gf_alerts_node
|
||||
target: /etc/grafana/provisioning/alerting/node.yml
|
||||
- source: gf_alerts_traefik
|
||||
target: /etc/grafana/provisioning/alerting/traefik.yml
|
||||
networks:
|
||||
- proxy
|
||||
- internal
|
||||
@@ -36,7 +34,6 @@ services:
|
||||
- GF_INSTALL_PLUGINS
|
||||
- ALERT_NODE_DISK_SPACE_LEFT
|
||||
- ALERT_NODE_MEMORY_USAGE
|
||||
- ALERT_TRAEFIK_APDEX
|
||||
deploy:
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
@@ -80,10 +77,6 @@ configs:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
|
||||
file: alerts/node.yml.tmpl
|
||||
gf_alerts_traefik:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION}
|
||||
file: alerts/traefik.yml.tmpl
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.15.0
|
||||
image: prom/prometheus:v3.12.0
|
||||
secrets:
|
||||
- basic_auth
|
||||
volumes:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
smartctl:
|
||||
image: "prometheuscommunity/smartctl-exporter:v0.15.0"
|
||||
image: "prometheuscommunity/smartctl-exporter:v0.14.0"
|
||||
volumes:
|
||||
- "/var/lib/smartmontools/json:/debug"
|
||||
command:
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ version: "3.8"
|
||||
|
||||
services:
|
||||
app:
|
||||
image: grafana/alloy:v1.20.1
|
||||
image: grafana/alloy:v1.18.1
|
||||
hostname: "${DOMAIN}"
|
||||
configs:
|
||||
- source: config_alloy
|
||||
@@ -53,7 +53,7 @@ services:
|
||||
condition: on-failure
|
||||
labels:
|
||||
- "backupbot.backup=${ENABLE_BACKUPS:-true}"
|
||||
- "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1"
|
||||
- "coop-cloud.${STACK_NAME}.version=2.2.0+v1.18.1"
|
||||
configs:
|
||||
config_alloy:
|
||||
template_driver: golang
|
||||
|
||||
Reference in New Issue
Block a user