9 Commits

16 changed files with 282 additions and 73 deletions

View File

@ -12,6 +12,9 @@ ENABLE_BACKUPS=true
# COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
# LOKI_PUSH_URL=https://loki.monitoring.example.org/loki/api/v1/push
## Expose node and cadvisor ports instead of traefik
# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
# Monitoring Server
#
## Prometheus
@ -43,6 +46,8 @@ ENABLE_BACKUPS=true
# COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
# GF_SERVER_ROOT_URL=https://monitoring.example.com
# SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
## Seperate domain for Grafana
#GRAFANA_DOMAIN=grafana.example.com
#
## Single-Sign-On with OIDC
# OIDC_ENABLED=1
@ -65,6 +70,13 @@ ENABLE_BACKUPS=true
# SECRET_GRAFANA_SMTP_PASSWORD_VERSION=v1
#
## Grafana Matrix Contact Point (optional)
#COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
#SECRET_MATRIX_ACCESS_TOKEN_VERSION=v1
#GF_MATRIX_USER_ID="<user-id>"
#GF_MATRIX_ROOM_ID="<room-id>"
#GF_MATRIX_HOMESERVER_URL="<homeserver-url>"
# ALerts
#ALERT_BACKUP_FAILED_ENABLED=true
#ALERT_BACKUP_MISSING_ENABLED=true

View File

@ -36,7 +36,7 @@ Where gathering.org is the node you want to gather metrics from.
SECRET_USERSFILE_VERSION=v1
```
- Generate userslist with httpasswd hashed password
`abra app secret insert traefik.gathering.org userslist v1 'admin:<hashed-secret>'`
`abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
- `abra app deploy -f traefik`
1. `abra app new monitoring-ng`
@ -47,6 +47,13 @@ Where gathering.org is the node you want to gather metrics from.
- cadvisor.monitoring.gathering.org
- node.monitoring.gathering.org
### Expose node and cadvisor via ports instead of traefik
In case you have no traefik running on the machine, you can expose the ports directly by uncommenting the following line:
```
# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
```
## Setup Metrics Browser
@ -85,7 +92,6 @@ Where gathering.org is the node you want to gather metrics from.
| Cadvisor | traefik basic-auth | cadvisor.monitoring.example.org |
| Node Exporter | traefik basic-auth | node.monitoring.example.org |
### Logging from a docker host to loki server without anything else
```
@ -130,6 +136,26 @@ After that you need to add the `pushgateway.${DOMAIN}` to the scare config.
THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal
## Adding Matrix as Alert Contact point
1. Enable the [matrix-alertmanager-receiver](https://github.com/metio/matrix-alertmanager-receiver/):
```
COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
```
2. Insert the matrix access token secret:
```
abra app secret insert monitoring.marx.klasse-methode.it matrix_access_token v1
```
3. Set required configurations:
```
GF_MATRIX_USER_ID=
GF_MATRIX_ROOM_ID=
GF_MATRIX_HOME_SERVER_URL=
```
4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`
## alerts
It is possible to enable the following alerts, by setting the corresponding env variable to `true`:
@ -138,3 +164,5 @@ It is possible to enable the following alerts, by setting the corresponding env
- backupbot not successfull: `ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED`
- node disk space: `ALERT_NODE_DISK_SPACE_ENABLED`
- node memory usage: `ALERT_NODE_MEMORY_USAGE_ENABLED`

View File

@ -10,6 +10,8 @@ export GRAFANA_CUSTOM_INI_VERSION=v4
export PROMTAIL_YML_VERSION=v3
export LOKI_YML_VERSION=v2
export PROMETHEUS_YML_VERSION=v2
export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
# creates a default prometheus scrape config for a given node
add_node(){

View File

@ -0,0 +1,74 @@
# configuration of the HTTP server
http:
## address: 127.0.0.1 # bind address for this service. Can be left unspecified to bind on all interfaces
port: 12345 # port used by this service
alerts-path-prefix: /alerts # URL path for the webhook receiver called by an Alertmanager. Defaults to /alerts
metrics-path: /metrics # URL path to collect metrics. Defaults to /metrics
metrics-enabled: true # Whether to enable metrics or not. Defaults to false
# basic-username: alertmanager # Username for basic authentication. Defaults to alertmanager
# basic-password: secret # If set, the alerts endpoint expects basic-auth credentials with the configured username and password
# configuration for the Matrix connection
matrix:
homeserver-url: "{{ env "GF_MATRIX_HOMESERVER_URL" }}"
user-id: "{{ env "GF_MATRIX_USER_ID" }}"
access-token: "{{ secret "matrix_access_token" }}"
room-mapping:
matrixroom: "{{ env "GF_MATRIX_ROOM_ID" }}"
templating:
# mapping of ExternalURL values
external-url-mapping:
# key is the original value taken from the Alertmanager payload
# value is the mapped value which will be available as '.ExternalURL' in templates
"http://alertmanager:9093": https://alertmanager.example.com
# mapping of GeneratorURL values
generator-url-mapping:
# key is the original value taken from the Alertmanager payload
# value is the mapped value which will be available as '.GeneratorURL' in templates
"http://prometheus:8080": https://prometheus.example.com
# computation of arbitrary values based on matching alert annotations, labels, or status
# values will be evaluated top to bottom, last entry wins
computed-values:
- values: # always set 'color' to 'yellow'
color: yellow
- values: # set 'color' to 'orange' when alert label 'severity' is 'warning'
color: orange
when-matching-labels:
severity: warning
- values: # set 'color' to 'red' when alert label 'severity' is 'critical'
color: red
when-matching-labels:
severity: critical
- values: # set 'color' to 'green' when alert status is 'resolved'
color: green
when-matching-status: resolved
# template for alerts in status 'firing'
firing-template: '{{`
<p>
<strong><font color="{{ .ComputedValues.color }}">{{ .Alert.Status | ToUpper }}</font></strong>
{{ if .Alert.Labels.name }}
{{ .Alert.Labels.name }}
{{ else if .Alert.Labels.alertname }}
{{ .Alert.Labels.alertname }}
{{ end }}
>>
{{ if .Alert.Labels.severity }}
{{ .Alert.Labels.severity | ToUpper }}:
{{ end }}
{{ if .Alert.Annotations.description }}
{{ .Alert.Annotations.description }}
{{ else if .Alert.Annotations.summary }}
{{ .Alert.Annotations.summary }}
{{ end }}
>>
{{ if .Alert.Annotations.runbook }}
<a href="{{ .Alert.Annotations.runbook }}">Runbook</a> |
{{ end }}
{{ if .Alert.Annotations.dashboard }}
<a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> |
{{ end }}
<a href="{{ .SilenceURL }}">Silence</a>
</p>`}}'

13
compose.expose-ports.yml Normal file
View File

@ -0,0 +1,13 @@
---
version: "3.8"
services:
app:
ports:
- "9100:9100"
deploy:
cadvisor:
ports:
- "9101:8080"
deploy:

View File

@ -48,8 +48,9 @@ services:
deploy:
labels:
- "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
- "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
- "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
- "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"

View File

@ -27,6 +27,7 @@ services:
condition: on-failure
labels:
- "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-loki.loadbalancer.server.port=3100"
- "traefik.http.routers.${STACK_NAME}-loki.rule=Host(`loki.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-loki.entrypoints=web-secure"

View File

@ -0,0 +1,28 @@
version: '3.8'
services:
matrix-alertmanager-receiver:
image: metio/matrix-alertmanager-receiver:2025.2.9
secrets:
- matrix_access_token
configs:
- source: matrix-alertmanager-receiver-config
target: /etc/matrix-alertmanager-receiver/config.yml
networks:
- internal
environment:
- GF_MATRIX_USER_ID
- GF_MATRIX_ROOM_ID
- GF_MATRIX_HOMESERVER_URL
command: "--config-path=/etc/matrix-alertmanager-receiver/config.yml"
configs:
matrix-alertmanager-receiver-config:
template_driver: golang
name: ${STACK_NAME}_mar_config_${MATRIX_ALERTMANAGER_CONFIG_VERSION}
file: alertmanager-matrix-config.yml.tmpl
secrets:
matrix_access_token:
external: true
name: ${STACK_NAME}_matrix_access_token_${SECRET_MATRIX_ACCESS_TOKEN_VERSION}

View File

@ -24,12 +24,12 @@ services:
condition: on-failure
labels:
- "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-prometheus.loadbalancer.server.port=9090"
- "traefik.http.routers.${STACK_NAME}-prometheus.rule=Host(`prometheus.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
- "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
- "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
- "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
configs:
prometheus_yml:

View File

@ -17,6 +17,7 @@ services:
condition: on-failure
labels:
- "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-pushgateway.loadbalancer.server.port=9191"
- "traefik.http.routers.${STACK_NAME}-pushgateway.rule=Host(`pushgateway.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-pushgateway.entrypoints=web-secure"

View File

@ -32,13 +32,14 @@ services:
labels:
- "backupbot.backup=${ENABLE_BACKUPS:-true}"
- "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
- "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure"
- "traefik.http.routers.${STACK_NAME}-node.tls=true"
- "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
- "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
- "coop-cloud.${STACK_NAME}.version=1.3.0+v1.8.1"
- "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
- "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
cadvisor:
@ -63,6 +64,7 @@ services:
condition: on-failure
labels:
- "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080"
- "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure"

View File

@ -216,7 +216,7 @@
"datasourceUid": "PBFA97CFB590B2093",
"model": {
"editorMode": "code",
"expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100",
"expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
"instant": true,
"intervalMs": 1000,
"legendFormat": "__auto",

View File

@ -93,7 +93,6 @@
},
"hideTimeOverride": true,
"id": 2,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
@ -107,10 +106,12 @@
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto"
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -172,7 +173,6 @@
"y": 0
},
"id": 1,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
@ -186,10 +186,12 @@
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto"
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -251,7 +253,6 @@
},
"hideTimeOverride": true,
"id": 4,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
@ -265,10 +266,12 @@
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto"
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -335,9 +338,10 @@
"y": 0
},
"id": 8,
"links": [],
"maxDataPoints": 100,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
@ -348,9 +352,10 @@
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"text": {}
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -405,13 +410,12 @@
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"pointradius": 5,
"points": false,
"renderer": "flot",
@ -507,7 +511,6 @@
},
"hideTimeOverride": true,
"id": 3,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
@ -521,10 +524,12 @@
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto"
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -585,7 +590,6 @@
},
"hideTimeOverride": true,
"id": 9,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
@ -599,10 +603,12 @@
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto"
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -671,9 +677,10 @@
},
"hideTimeOverride": true,
"id": 11,
"links": [],
"maxDataPoints": 100,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
@ -684,9 +691,10 @@
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"text": {}
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"targets": [
{
"datasource": {
@ -713,7 +721,39 @@
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "left",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [
{
@ -747,33 +787,42 @@
},
"unit": "percent"
},
"overrides": []
"overrides": [
{
"matcher": {
"id": "byType",
"options": "time"
},
"properties": [
{
"id": "custom.axisPlacement",
"value": "hidden"
}
]
}
]
},
"gridPos": {
"h": 4,
"w": 2.6666666666666665,
"h": 6,
"w": 6,
"x": 0,
"y": 8
},
"id": 10,
"links": [],
"maxDataPoints": 100,
"maxPerRow": 12,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"textMode": "auto"
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"repeat": "node_id",
"repeatDirection": "h",
"targets": [
@ -782,18 +831,20 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "",
"legendFormat": "{{mountpoint}}",
"range": true,
"refId": "A",
"step": 20
}
],
"title": "Available Disk Space $node_id",
"type": "stat"
"type": "timeseries"
},
{
"aliasColors": {},
@ -811,7 +862,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 12
"y": 14
},
"hiddenSeries": false,
"id": 14,
@ -830,13 +881,12 @@
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"pointradius": 5,
"points": false,
"renderer": "flot",
@ -900,6 +950,7 @@
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
@ -913,6 +964,7 @@
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
@ -961,12 +1013,11 @@
},
"gridPos": {
"h": 7,
"w": 2.6666666666666665,
"w": 6,
"x": 0,
"y": 19
"y": 21
},
"id": 15,
"links": [],
"maxPerRow": 12,
"options": {
"legend": {
@ -1074,7 +1125,7 @@
"h": 7,
"w": 24,
"x": 0,
"y": 26
"y": 28
},
"hiddenSeries": false,
"id": 16,
@ -1091,13 +1142,12 @@
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "10.0.2",
"pluginVersion": "10.4.14",
"pointradius": 5,
"points": false,
"renderer": "flot",
@ -1178,7 +1228,7 @@
"h": 7,
"w": 12,
"x": 0,
"y": 33
"y": 35
},
"hiddenSeries": false,
"id": 18,
@ -1195,7 +1245,6 @@
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
@ -1281,7 +1330,7 @@
"h": 7,
"w": 12,
"x": 12,
"y": 33
"y": 35
},
"hiddenSeries": false,
"id": 19,
@ -1300,7 +1349,6 @@
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
@ -1376,7 +1424,7 @@
"h": 7,
"w": 18,
"x": 0,
"y": 40
"y": 42
},
"hiddenSeries": false,
"id": 12,
@ -1397,7 +1445,6 @@
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
@ -1499,10 +1546,9 @@
"h": 7,
"w": 6,
"x": 18,
"y": 40
"y": 42
},
"id": 7,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "none",
@ -1600,10 +1646,9 @@
"h": 7,
"w": 24,
"x": 0,
"y": 47
"y": 49
},
"id": 17,
"links": [],
"options": {
"legend": {
"calcs": [],
@ -1658,7 +1703,7 @@
"h": 9,
"w": 24,
"x": 0,
"y": 54
"y": 56
},
"id": 30,
"options": {
@ -1688,8 +1733,7 @@
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"schemaVersion": 39,
"tags": [
"swarmprom",
"prometheus",
@ -1836,6 +1880,6 @@
"timezone": "",
"title": "Docker Swarm Nodes",
"uid": "BPlb-Sgik",
"version": 24,
"version": 7,
"weekStart": ""
}

1
release/1.4.0+v1.8.1 Normal file
View File

@ -0,0 +1 @@
Adds an optional GRAFANA_DOMAIN

1
release/1.5.0+v1.8.1 Normal file
View File

@ -0,0 +1 @@
Adds an optional matrix contact point for grafana

1
release/1.6.0+v1.8.1 Normal file
View File

@ -0,0 +1 @@
Adds option to expose ports for node and cadvisor service