Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 fauno
|
d7bb30d346
|
||
|
fauno
|
f9820da529
|
+6
-9
@@ -53,13 +53,6 @@ SECRET_BASIC_AUTH_VERSION=v1
|
||||
# Not needed if you just want to read local log files — use SYSLOG_FILES instead.
|
||||
# SYSLOG=1
|
||||
# COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml"
|
||||
#
|
||||
# Scrape Docker-Events via official docker-cli image using docker socket
|
||||
# COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
|
||||
|
||||
|
||||
# Monitor physical disks health
|
||||
# COMPOSE_FILE="$COMPOSE_FILE:compose.smartctl.yml"
|
||||
|
||||
# Monitoring Server
|
||||
#
|
||||
@@ -134,5 +127,9 @@ SECRET_BASIC_AUTH_VERSION=v1
|
||||
# Node memory usage alert will trigger when memory usage is above the given number in percent
|
||||
#ALERT_NODE_MEMORY_USAGE=85
|
||||
|
||||
# Traefik apdex alert will trigger when the apdex score of a service drops below the given number. Services receiving fewer than 10 requests per 10m are ignored
|
||||
#ALERT_TRAEFIK_APDEX=0.5
|
||||
# Tell Traefik to keep access logs (could be very verbose). See Access
|
||||
# logs on Traefik recipe.
|
||||
#ALLOY_ACCESS_LOGS=true
|
||||
#PROMETHEUS_ACCESS_LOGS=true
|
||||
#LOKI_ACCESS_LOGS=true
|
||||
#GRAFANA_ACCESS_LOGS=true
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
* @coop-cloud/monitoring-ng-maintainers
|
||||
@@ -1,35 +0,0 @@
|
||||
# Monitoring NG Recipe Maintenance
|
||||
|
||||
All contributions should be made via a pull request. This is to ensure a
|
||||
certain quality and consistency, that others can rely on.
|
||||
|
||||
## Maintainer Responsibilities
|
||||
|
||||
A recipe maintainer has the following responsibilities:
|
||||
|
||||
- Respond to pull requests / issues within two weeks
|
||||
- Make image security updates within a week
|
||||
- Make image major updates every three months
|
||||
|
||||
In order to fullfill these responsibilities a recipe maintainer:
|
||||
|
||||
- Has to watch the repository (to get notifications)
|
||||
- Needs to make sure renovate is configured properly
|
||||
|
||||
## Pull Requests
|
||||
|
||||
A pull request can be merged if it is approved by at least one maintainer. For
|
||||
pull requests opened by a maintainer they need to be approved by another
|
||||
maintainer. Even though it is okay to merge a pull request with one approval, it
|
||||
is always better if all maintainers looked at the pull request and approved it.
|
||||
|
||||
## Become a maintainer
|
||||
|
||||
Everyone can apply to be a recipe maintainer:
|
||||
1. Watch the repository to always get updates
|
||||
2. Simply add your self to the list in the [README.md](./README.md) and open a new pull request with the change.
|
||||
3. Once the pull request gets merged you will be added to the
|
||||
[monitoring-ng maintainers
|
||||
team](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers).
|
||||
4. Join the room
|
||||
[#cc-|-monitoring-ng-maintenance:matrix.org](#cc-|-monitoring-ng-maintenance:matrix.org) and chat to other maintainers.
|
||||
@@ -7,7 +7,7 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou
|
||||
Metrics and logs are collected by [Grafana Alloy](https://grafana.com/docs/alloy/latest/) and pushed to a central Prometheus/Loki (via `remote_write`/`loki push`). Every `monitoring-ng` instance runs its own Alloy, whether or not it also runs the central stack.
|
||||
|
||||
<!-- metadata -->
|
||||
- **Maintainer**: [@coop-cloud/monitoring-ng-maintainers](https://git.coopcloud.tech/org/coop-cloud/teams/monitoring-ng-maintainers)
|
||||
|
||||
- **Category**: Apps
|
||||
- **Status**: 2, beta
|
||||
- **Image**: [`grafana/grafana`](https://hub.docker.com/r/grafana/grafana), 4, upstream
|
||||
@@ -54,14 +54,6 @@ This is what a gathering host pushes into. It also runs its own Alloy, so it mon
|
||||
|
||||
## Additional features
|
||||
|
||||
### Scrape docker events
|
||||
|
||||
Just uncomment the following line in your .env to collect docker events using
|
||||
the official docker-cli image and the docker socket:
|
||||
```
|
||||
COMPOSE_FILE="$COMPOSE_FILE:compose.docker-events.yml"
|
||||
```
|
||||
|
||||
### Discovering metrics from other apps
|
||||
|
||||
Alloy auto-discovers and scrapes other Docker Swarm services running on the same host, on the `proxy` network, that opt in via labels. No manual scrape config needed. On the app's `compose.yml`:
|
||||
@@ -174,12 +166,3 @@ It is possible to enable the following alerts, by uncommenting the corresponding
|
||||
|
||||
- node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
|
||||
- node memory usage: `ALERT_NODE_MEMORY_USAGE`
|
||||
- traefik apdex: `ALERT_TRAEFIK_APDEX`
|
||||
|
||||
## smart monitoring
|
||||
|
||||
To be able monitor hard drive health data, you need to configure
|
||||
`smartd` to run on the host system, and also the
|
||||
`collect-smartctl-json.sh` script provided here (via cronjob or as
|
||||
a `smartd` hook). This is a limitation on Docker Swarm, which prevents
|
||||
the `smartctl_exporter` from running on privileged mode.
|
||||
|
||||
@@ -10,8 +10,7 @@ export PROMETHEUS_YML_VERSION=v2
|
||||
export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
|
||||
export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
|
||||
export GF_ALERTS_NODE_VERSION=v3
|
||||
export GF_ALERTS_TRAEFIK_VERSION=v1
|
||||
export CONFIG_ALLOY_VERSION=v3
|
||||
export CONFIG_ALLOY_VERSION=v2
|
||||
|
||||
# migrates secrets from old names to new names by reading values from the
|
||||
# running containers on the server and re-inserting them under the new names.
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
apiVersion: 1
|
||||
|
||||
# List of alert rule UIDs that should be deleted
|
||||
deleteRules:
|
||||
{{ if not (env "ALERT_TRAEFIK_APDEX") }}
|
||||
- orgId: 1
|
||||
uid: coopcloud_traefik_apdex
|
||||
{{ end }}
|
||||
|
||||
groups:
|
||||
- orgId: 1
|
||||
name: traefik
|
||||
folder: recipes
|
||||
interval: 5m
|
||||
rules:
|
||||
{{ if (env "ALERT_TRAEFIK_APDEX") }}
|
||||
- uid: coopcloud_traefik_apdex
|
||||
title: Apdex
|
||||
condition: B
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: PBFA97CFB590B2093
|
||||
model:
|
||||
editorMode: code
|
||||
# apdex = (satisfied + tolerating / 2) / total, which reduces to
|
||||
# (le=0.3 + le=1.2) / 2 / total because the le=1.2 bucket is
|
||||
# cumulative and therefore already contains the satisfied requests.
|
||||
# services with less than 10 requests per 10m are ignored.
|
||||
expr: |-
|
||||
((sum(rate(traefik_service_request_duration_seconds_bucket{le="0.3",code="200"}[5m])) by (service) +
|
||||
sum(rate(traefik_service_request_duration_seconds_bucket{le="1.2",code="200"}[5m])) by (service)) / 2 /
|
||||
sum(rate(traefik_service_request_duration_seconds_count{code="200"}[5m])) by (service))
|
||||
and on(service)
|
||||
(
|
||||
sum(increase(traefik_service_request_duration_seconds_count{code="200"}[10m])) by (service) >= 10
|
||||
)
|
||||
instant: true
|
||||
intervalMs: 1000
|
||||
legendFormat: __auto
|
||||
maxDataPoints: 43200
|
||||
range: false
|
||||
refId: A
|
||||
- refId: B
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator:
|
||||
params:
|
||||
- {{ env "ALERT_TRAEFIK_APDEX" }}
|
||||
type: lt
|
||||
operator:
|
||||
type: and
|
||||
query:
|
||||
params: []
|
||||
reducer:
|
||||
params: []
|
||||
type: avg
|
||||
type: query
|
||||
datasource:
|
||||
type: __expr__
|
||||
uid: __expr__
|
||||
expression: A
|
||||
intervalMs: 1000
|
||||
maxDataPoints: 43200
|
||||
refId: B
|
||||
type: threshold
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
for: 10m
|
||||
annotations:
|
||||
description: ""
|
||||
runbook_url: ""
|
||||
summary: Apdex for {{`{{ $labels.service }}`}} is below {{ env "ALERT_TRAEFIK_APDEX" }} (currently at {{`{{ printf "%.2f" (index $values "A").Value }}`}})
|
||||
isPaused: false
|
||||
{{ end }}
|
||||
@@ -1,6 +0,0 @@
|
||||
[Unit]
|
||||
Description=Collect SMART data
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/collect-smartctl-json.sh
|
||||
@@ -1,61 +0,0 @@
|
||||
#! /bin/bash
|
||||
# Adapted from https://github.com/prometheus-community/smartctl_exporter/blob/master/collect-smartctl-json.sh
|
||||
|
||||
# Data directory to dump smartctl output
|
||||
# This directory will be created if it doesn't exist
|
||||
data_dir="/var/lib/smartmontools/json"
|
||||
|
||||
# The original script used --xall but that doesn't work
|
||||
# This matches the command in readSMARTctl()
|
||||
smartctl_args="--json --info --health --attributes --tolerance=verypermissive \
|
||||
--nocheck=standby --format=brief --log=error"
|
||||
|
||||
# Ignore this devices
|
||||
smartctl_ignore_dev_regex="^(/dev/bus)"
|
||||
|
||||
# Determine the json query tool to use
|
||||
if command -v jq >/dev/null; then
|
||||
json_tool="jq"
|
||||
json_args="--raw-output"
|
||||
elif command -v yq >/dev/null; then
|
||||
json_tool="yq"
|
||||
json_args="--unwrapScalar"
|
||||
else
|
||||
echo -e "One of 'yq' or 'jq' is required. Please try again after \
|
||||
installing one of them"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! "${UID}" -eq 0 ]] && ! command -v sudo >/dev/null; then
|
||||
# Not root and sudo doesn't exist
|
||||
echo "sudo does not exist. Please run this as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SUDO="sudo"
|
||||
if [[ "${UID}" -eq 0 ]]; then
|
||||
# Don't use sudo if root
|
||||
SUDO=""
|
||||
fi
|
||||
|
||||
[[ ! -d "${data_dir}" ]] && mkdir --parents "${data_dir}"
|
||||
|
||||
if [[ $# -ne 0 ]]; then
|
||||
devices="${1}"
|
||||
else
|
||||
devices="$(smartctl --scan --json | "${json_tool}" "${json_args}" \
|
||||
".devices[].name | select(test(\"${smartctl_ignore_dev_regex}\") | not)")"
|
||||
mapfile -t devices <<< "${devices[@]}"
|
||||
fi
|
||||
|
||||
for device in "${devices[@]}"
|
||||
do
|
||||
echo -n "Collecting data for '${device}'..."
|
||||
# shellcheck disable=SC2086
|
||||
data="$($SUDO smartctl ${smartctl_args} ${device})"
|
||||
# Accommodate a smartmontools pre-7.3 bug
|
||||
data=${data#" Pending defect count:"}
|
||||
device_name="$(basename "${device}")"
|
||||
echo -e "\tSaving to ${device_name}.json"
|
||||
echo "${data}" > "${data_dir}/${device_name}.json"
|
||||
done
|
||||
@@ -1,9 +0,0 @@
|
||||
[Unit]
|
||||
Description=Collect SMART data
|
||||
|
||||
[Timer]
|
||||
OnCalendar=hourly
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -14,3 +14,4 @@ services:
|
||||
- "traefik.http.routers.${STACK_NAME}-alloy.tls=true"
|
||||
- "traefik.http.routers.${STACK_NAME}-alloy.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||
- "traefik.http.routers.${STACK_NAME}-alloy.middlewares=basicauth@file"
|
||||
- "traefik.http.routers.${STACK_NAME}-alloy.observability.accesslogs=${ALLOY_ACCESS_LOGS:-false}"
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
docker-events:
|
||||
image: docker:29.8.1-cli
|
||||
command: >
|
||||
docker events
|
||||
--format '{{json .}}'
|
||||
--filter type=container
|
||||
--filter event=start
|
||||
--filter event=die
|
||||
--filter event=kill
|
||||
--filter event=oom
|
||||
--filter event=restart
|
||||
--filter event=stop
|
||||
--filter event=health_status
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
+2
-8
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
grafana:
|
||||
image: grafana/grafana:13.2.3
|
||||
image: grafana/grafana:13.0.6
|
||||
volumes:
|
||||
- grafana-data:/var/lib/grafana:rw
|
||||
secrets:
|
||||
@@ -24,8 +24,6 @@ services:
|
||||
target: /var/lib/grafana/dashboards/backup.json
|
||||
- source: gf_alerts_node
|
||||
target: /etc/grafana/provisioning/alerting/node.yml
|
||||
- source: gf_alerts_traefik
|
||||
target: /etc/grafana/provisioning/alerting/traefik.yml
|
||||
networks:
|
||||
- proxy
|
||||
- internal
|
||||
@@ -36,7 +34,6 @@ services:
|
||||
- GF_INSTALL_PLUGINS
|
||||
- ALERT_NODE_DISK_SPACE_LEFT
|
||||
- ALERT_NODE_MEMORY_USAGE
|
||||
- ALERT_TRAEFIK_APDEX
|
||||
deploy:
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
@@ -46,6 +43,7 @@ services:
|
||||
- "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
|
||||
- "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
|
||||
- "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||
- "traefik.http.routers.${STACK_NAME}-grafana.observability.accesslogs=${GRAFANA_ACCESS_LOGS:-false}"
|
||||
healthcheck:
|
||||
test: "wget -q http://localhost:3000/healthz -O/dev/null"
|
||||
interval: 5s
|
||||
@@ -80,10 +78,6 @@ configs:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_node_${GF_ALERTS_NODE_VERSION}
|
||||
file: alerts/node.yml.tmpl
|
||||
gf_alerts_traefik:
|
||||
template_driver: golang
|
||||
name: ${STACK_NAME}_gf_alerts_traefik_${GF_ALERTS_TRAEFIK_VERSION}
|
||||
file: alerts/traefik.yml.tmpl
|
||||
|
||||
volumes:
|
||||
grafana-data:
|
||||
|
||||
@@ -34,6 +34,7 @@ services:
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.tls=true"
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
|
||||
- "traefik.http.routers.${STACK_NAME}-loki.observability.accesslogs=${LOKI_ACCESS_LOGS:-false}"
|
||||
|
||||
|
||||
configs:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.15.0
|
||||
image: prom/prometheus:v3.12.0
|
||||
secrets:
|
||||
- basic_auth
|
||||
volumes:
|
||||
@@ -38,6 +38,7 @@ services:
|
||||
- "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
|
||||
- "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
|
||||
- "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
|
||||
- "traefik.http.routers.${STACK_NAME}-prometheus.observability.accesslogs=${PROMETHEUS_ACCESS_LOGS:-false}"
|
||||
|
||||
configs:
|
||||
prometheus_yml:
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
---
|
||||
version: "3.8"
|
||||
services:
|
||||
smartctl:
|
||||
image: "prometheuscommunity/smartctl-exporter:v0.15.0"
|
||||
volumes:
|
||||
- "/var/lib/smartmontools/json:/debug"
|
||||
command:
|
||||
- "--smartctl.fake-data"
|
||||
- "--smartctl.interval=1h"
|
||||
networks:
|
||||
- "proxy"
|
||||
deploy:
|
||||
labels:
|
||||
- "prometheus.io/scrape=true"
|
||||
- "prometheus.io/port=9633"
|
||||
- "prometheus.io/path=/metrics"
|
||||
+4
-4
@@ -3,7 +3,7 @@ version: "3.8"
|
||||
|
||||
services:
|
||||
app:
|
||||
image: grafana/alloy:v1.20.1
|
||||
image: grafana/alloy:v1.18.1
|
||||
hostname: "${DOMAIN}"
|
||||
configs:
|
||||
- source: config_alloy
|
||||
@@ -46,14 +46,14 @@ services:
|
||||
test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/12345 && printf 'GET /-/ready HTTP/1.0\r\nHost: localhost\r\n\r\n' >&3 && head -1 <&3 | grep -q 200"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
start_period: 30s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: on-failure
|
||||
labels:
|
||||
- "backupbot.backup=${ENABLE_BACKUPS:-true}"
|
||||
- "coop-cloud.${STACK_NAME}.version=2.3.0+v1.20.1"
|
||||
- "coop-cloud.${STACK_NAME}.version=2.0.0+v1.18.1"
|
||||
configs:
|
||||
config_alloy:
|
||||
template_driver: golang
|
||||
|
||||
@@ -52,32 +52,7 @@ prometheus.scrape "default" {
|
||||
prometheus.exporter.cadvisor.docker.targets,
|
||||
)
|
||||
|
||||
forward_to = [prometheus.relabel.container_meta.receiver]
|
||||
}
|
||||
|
||||
prometheus.relabel "container_meta" {
|
||||
forward_to = [prometheus.remote_write.prometheus.receiver]
|
||||
|
||||
// remove sha tail: nginx:1.31.1@sha256:608a... -> nginx:1.31.1
|
||||
rule {
|
||||
source_labels = ["image"]
|
||||
regex = "([^@]+)@sha256:.*"
|
||||
target_label = "image"
|
||||
replacement = "$1"
|
||||
}
|
||||
// split image in name and tag
|
||||
rule {
|
||||
source_labels = ["image"]
|
||||
regex = "(.+):[^:/]+"
|
||||
target_label = "image_name"
|
||||
replacement = "$1"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["image"]
|
||||
regex = ".+:([^:/]+)"
|
||||
target_label = "image_tag"
|
||||
replacement = "$1"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.remote_write "prometheus" {
|
||||
@@ -315,35 +290,7 @@ loki.source.docker "docker" {
|
||||
loki.source.journal "journal" {
|
||||
path = "/rootfs/var/log/journal"
|
||||
labels = { job = "{{ env "DOMAIN" }}" }
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
forward_to = [loki.process.journal.receiver]
|
||||
}
|
||||
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_syslog_identifier"]
|
||||
target_label = "ident"
|
||||
}
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "priority"
|
||||
}
|
||||
}
|
||||
|
||||
// drop network db stats (15-20% of kernel logs)
|
||||
loki.process "journal" {
|
||||
forward_to = [loki.write.loki.receiver]
|
||||
|
||||
stage.drop {
|
||||
expression = ".*NetworkDB stats.*"
|
||||
drop_counter_reason = "networkdb_stats"
|
||||
}
|
||||
}
|
||||
{{ end }}
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
alloy: split image label in "image_name" and "image_tag", relabel journald logs and drop NetworkDB stats
|
||||
@@ -1 +0,0 @@
|
||||
allow scraping of docker events via new compose.docker-events.yml
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:recommended"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user