From f2310f2b868f977b33542a8171067fc779061451 Mon Sep 17 00:00:00 2001 From: Danny Groenewegen Date: Sat, 13 Jun 2026 22:00:05 +0200 Subject: [PATCH 1/8] improve alloy config and match with main branch - Restrict Alloy UI to loopback - Narrow volume mounts: drop /dev, reduce /var/run to docker.sock:ro - Replace HTTP scrape of :12345 with prometheus.exporter.self Match with main branch (node-exporter / promtail / cadvisor): - Add docker_only and explicit enabled_metrics to cadvisor exporter - Match node-exporter collector config - Match promtail relabeling (container_name, container_id, stack_namespace, service_name) and external hostname label - Add SYSLOG_FILES option to tail /var/log/*log (matches promtail) - Fix journal path and syslog listener address --- .env.sample | 14 ++++++-- compose.yml | 5 ++- config.alloy.tmpl | 83 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 13 deletions(-) diff --git a/.env.sample b/.env.sample index b68f7c4..7902e29 100644 --- a/.env.sample +++ b/.env.sample @@ -9,10 +9,18 @@ SECRET_BASIC_AUTH_VERSION=v1 # Enable this to send logs to a Loki server, adapt DOMAIN if server is # remote # LOKI_PUSH_URL=https://loki.$DOMAIN/loki/api/v1/push -# Enable this on SystemD hosts to read logs +# Enable on systemd hosts to read logs from the journal # JOURNALD=1 -# Enable this on syslogd hosts and configure the syslogd to send logs to -# Alloy on port 514/tcp +# +# Enable on non-systemd hosts (Alpine, older Debian/Ubuntu) to tail +# /var/log/*log files (syslog, auth.log, kern.log, etc.) that a local +# syslogd writes. No syslogd reconfiguration needed. +# SYSLOG_FILES=1 +# +# Enable to receive syslog messages over the network on port 514/tcp. +# Use for remote devices that push syslog to this host, or for a +# local syslogd configured to forward over the network. +# Not needed if you just want to read local log files — use SYSLOG_FILES instead. # SYSLOG=1 # COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml" diff --git a/compose.yml b/compose.yml index b5f4fa1..ecd70d4 100644 --- a/compose.yml +++ b/compose.yml @@ -10,15 +10,14 @@ services: target: /etc/alloy/config.alloy volumes: - /:/rootfs:ro - - /var/run:/var/run:rw - - /var/run/docker.sock:/var/run/docker.sock + - /var/run/docker.sock:/var/run/docker.sock:ro - /sys:/sys:ro - /var/lib/docker:/var/lib/docker:ro - - /dev:/dev:ro - alloy-data:/var/lib/alloy/data command: - "run" - "--storage.path=/var/lib/alloy/data" + - "--server.http.listen-addr=127.0.0.1:12345" - "/etc/alloy/config.alloy" networks: - internal diff --git a/config.alloy.tmpl b/config.alloy.tmpl index 2083ce6..3def201 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -9,19 +9,35 @@ discovery.docker "linux" { {{ if ne (env "PROMETHEUS_REMOTE_WRITE_URL") "" }} prometheus.exporter.cadvisor "docker" { + docker_only = true + enabled_metrics = ["cpu", "cpuLoad", "disk", "diskIO", "memory", "network", "process"] } prometheus.exporter.unix "default" { include_exporter_metrics = true rootfs_path = "/rootfs" + procfs_path = "/rootfs/proc" + sysfs_path = "/rootfs/sys" + + disable_collectors = ["ipvs"] + + filesystem { + fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$" + mount_points_exclude = "^/(sys|proc|dev|host|etc)($|/)" + mount_timeout = "5s" + } + + netclass { ignored_devices = "^(veth.*)$" } + netdev { device_exclude = "^(veth.*)$" } } +prometheus.exporter.self "alloy" {} + prometheus.scrape "default" { + scrape_interval = "120s" + targets = array.concat( - [{ - job = "alloy", - __address__ = "127.0.0.1:12345", - }], + prometheus.exporter.self.alloy.targets, prometheus.exporter.unix.default.targets, prometheus.exporter.cadvisor.docker.targets, ) @@ -42,21 +58,73 @@ prometheus.remote_write "prometheus" { {{ end }} {{ if ne (env "LOKI_PUSH_URL") "" }} +discovery.relabel "docker" { + targets = discovery.docker.linux.targets + + rule { + source_labels = ["__meta_docker_container_name"] + target_label = "container_name" + } + rule { + source_labels = ["__meta_docker_container_id"] + target_label = "container_id" + } + rule { + source_labels = ["__meta_docker_container_label_com_docker_stack_namespace"] + target_label = "stack_namespace" + } + rule { + source_labels = ["__meta_docker_container_label_com_docker_swarm_service_name"] + target_label = "service_name" + } + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } +} + loki.source.docker "docker" { host = "unix:///var/run/docker.sock" - targets = discovery.docker.linux.targets + targets = discovery.relabel.docker.output labels = {"app" = "docker"} forward_to = [loki.write.loki.receiver] } +// JOURNALD: reads the systemd journal binary log directly. +// Use on systemd hosts (most modern Linux distros). Requires no syslogd. {{ if eq (env "JOURNALD") "1" }} loki.source.journal "journal" { - path = "/var/log/journal" + path = "/rootfs/var/log/journal" labels = { job = "{{ env "DOMAIN" }}" } forward_to = [loki.write.loki.receiver] } {{ end }} +// SYSLOG_FILES: tails all /var/log/*log files (syslog, auth.log, kern.log, etc.). +// Use on non-systemd hosts where a syslogd writes to /var/log. +{{ if eq (env "SYSLOG_FILES") "1" }} +local.file_match "syslog_files" { + path_targets = [{ __path__ = "/rootfs/var/log/*log" }] +} + +loki.source.file "syslog_files" { + targets = local.file_match.syslog_files.targets + forward_to = [loki.process.syslog_files.receiver] +} + +loki.process "syslog_files" { + stage.static_labels { + values = { job = "syslog" } + } + forward_to = [loki.write.loki.receiver] +} +{{ end }} + +// SYSLOG: opens a network syslog listener on port 514. +// Use when a remote device or a local syslogd configured to +// forward over the network sends logs to this host. +// Requires compose.syslog.yml to publish port 514 to the host. +// This is NOT needed for reading local log files — use SYSLOG_FILES instead. {{ if eq (env "SYSLOG") "1" }} loki.relabel "syslog" { rule { @@ -69,7 +137,7 @@ loki.relabel "syslog" { loki.source.syslog "syslog" { listener { - address = "[::1]:514" + address = "[::]:514" label_structured_data = true labels = { component = "loki.source.syslog" } } @@ -88,5 +156,6 @@ loki.write "loki" { password = "{{ secret "basic_auth" }}" } } + external_labels = { hostname = "{{ env "DOMAIN" }}" } } {{ end }} -- 2.49.0 From e247677433eac66959ade307a68a0f5bcce58e7f Mon Sep 17 00:00:00 2001 From: Danny Groenewegen Date: Sun, 14 Jun 2026 21:04:21 +0200 Subject: [PATCH 2/8] feat: scrape metrics from containers via Docker label discovery Containers opt in with prometheus.io/scrape=true and optionally set prometheus.io/port, prometheus.io/path, and prometheus.io/auth=basic. --- config.alloy.tmpl | 84 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/config.alloy.tmpl b/config.alloy.tmpl index 3def201..d050502 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -55,6 +55,90 @@ prometheus.remote_write "prometheus" { } } } + +// Scrape Prometheus metrics from other containers on this host. +// Containers opt in via Docker labels: +// prometheus.io/scrape=true required: enable scraping +// prometheus.io/port=9090 optional: port exposing /metrics (defaults to first exposed port) +// prometheus.io/path=/metrics optional: path to metrics endpoint (default: /metrics) +// prometheus.io/auth=basic optional: use basic auth with the shared basic_auth secret +// +// Uses docker_gwbridge — the host-local bridge network Docker attaches all +// Swarm containers to for outbound connectivity. Alloy can reach any container +// on the same host via this network without needing to join each stack's +// overlay network. +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" + match_first_network = false +} + +discovery.relabel "metrics" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_network_name"] + regex = "docker_gwbridge" + action = "keep" + } + + rule { + source_labels = ["__meta_docker_container_label_prometheus_io_scrape"] + regex = "true" + action = "keep" + } + + rule { + source_labels = ["__address__", "__meta_docker_container_label_prometheus_io_port"] + regex = `(.+):\d+;(\d+)` + target_label = "__address__" + replacement = "$1:$2" + } + + rule { + source_labels = ["__meta_docker_container_label_prometheus_io_path"] + regex = `(.+)` + target_label = "__metrics_path__" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_swarm_service_name"] + target_label = "job" + } +} + +discovery.relabel "metrics_noauth" { + targets = discovery.relabel.metrics.output + rule { + source_labels = ["__meta_docker_container_label_prometheus_io_auth"] + regex = "^$" + action = "keep" + } +} + +discovery.relabel "metrics_basicauth" { + targets = discovery.relabel.metrics.output + rule { + source_labels = ["__meta_docker_container_label_prometheus_io_auth"] + regex = "basic" + action = "keep" + } +} + +prometheus.scrape "containers" { + scrape_interval = "120s" + targets = discovery.relabel.metrics_noauth.output + forward_to = [prometheus.remote_write.prometheus.receiver] +} + +prometheus.scrape "containers_basicauth" { + scrape_interval = "120s" + targets = discovery.relabel.metrics_basicauth.output + forward_to = [prometheus.remote_write.prometheus.receiver] + basic_auth { + username = "admin" + password = "{{ secret "basic_auth" }}" + } +} {{ end }} {{ if ne (env "LOKI_PUSH_URL") "" }} -- 2.49.0 From 64cb07a4a25e6201d5cbcb59e5e887792c9fb45e Mon Sep 17 00:00:00 2001 From: f Date: Tue, 16 Jun 2026 20:03:17 -0300 Subject: [PATCH 3/8] feat: bearer auth support --- config.alloy.tmpl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/config.alloy.tmpl b/config.alloy.tmpl index d050502..3bff4ce 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -124,6 +124,15 @@ discovery.relabel "metrics_basicauth" { } } +discovery.relabel "metrics_bearerauth" { + targets = discovery.relabel.metrics.output + rule { + source_labels = ["__meta_docker_container_label_prometheus_io_auth"] + regex = "bearer" + action = "keep" + } +} + prometheus.scrape "containers" { scrape_interval = "120s" targets = discovery.relabel.metrics_noauth.output @@ -139,6 +148,13 @@ prometheus.scrape "containers_basicauth" { password = "{{ secret "basic_auth" }}" } } + +prometheus.scrape "containers_bearerauth" { + scrape_interval = "120s" + targets = discovery.relabel.metrics_bearerauth.output + forward_to = [prometheus.remote_write.prometheus.receiver] + bearer_token = "{{ secret "basic_auth" }}" +} {{ end }} {{ if ne (env "LOKI_PUSH_URL") "" }} -- 2.49.0 From fa761799874ee7f129eec5959deaeff2b76c53d1 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 16 Jun 2026 21:20:29 -0300 Subject: [PATCH 4/8] feat: enable alloy web ui --- compose.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/compose.yml b/compose.yml index ecd70d4..5e77608 100644 --- a/compose.yml +++ b/compose.yml @@ -17,9 +17,10 @@ services: command: - "run" - "--storage.path=/var/lib/alloy/data" - - "--server.http.listen-addr=127.0.0.1:12345" + - "--server.http.listen-addr=0.0.0.0:12345" - "/etc/alloy/config.alloy" networks: + - proxy - internal secrets: - basic_auth @@ -28,8 +29,15 @@ services: condition: on-failure labels: - "backupbot.backup=${ENABLE_BACKUPS:-true}" - - "traefik.enable=false" - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1" + - "traefik.enable=true" + - "traefik.swarm.network=proxy" + - "traefik.http.services.${STACK_NAME}-alloy.loadbalancer.server.port=12345" + - "traefik.http.routers.${STACK_NAME}-alloy.rule=Host(`alloy.${DOMAIN}`)" + - "traefik.http.routers.${STACK_NAME}-alloy.entrypoints=web-secure" + - "traefik.http.routers.${STACK_NAME}-alloy.tls=true" + - "traefik.http.routers.${STACK_NAME}-alloy.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.routers.${STACK_NAME}-alloy.middlewares=basicauth@file" configs: config_alloy: template_driver: golang -- 2.49.0 From 1970061ff864c426839cc1c569146bc375f75dfe Mon Sep 17 00:00:00 2001 From: f Date: Tue, 16 Jun 2026 21:59:50 -0300 Subject: [PATCH 5/8] feat: live debugging alloy --- .env.sample | 2 ++ config.alloy.tmpl | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.env.sample b/.env.sample index 7902e29..6689006 100644 --- a/.env.sample +++ b/.env.sample @@ -6,6 +6,8 @@ DOMAIN=monitoring-ng.example.com ENABLE_BACKUPS=true SECRET_BASIC_AUTH_VERSION=v1 +# Enable Live Debugging +LIVE_DEBUGGING=false # Enable this to send logs to a Loki server, adapt DOMAIN if server is # remote # LOKI_PUSH_URL=https://loki.$DOMAIN/loki/api/v1/push diff --git a/config.alloy.tmpl b/config.alloy.tmpl index 3bff4ce..9fc6347 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -3,6 +3,10 @@ logging { format = "logfmt" } +livedebugging { + enabled = {{ env "LIVE_DEBUGGING" }} +} + discovery.docker "linux" { host = "unix:///var/run/docker.sock" } -- 2.49.0 From d085c66d68916ac182b15c69249ccf9f810141ba Mon Sep 17 00:00:00 2001 From: f Date: Tue, 16 Jun 2026 22:00:42 -0300 Subject: [PATCH 6/8] fix: needed labels come from docker swarm --- config.alloy.tmpl | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/config.alloy.tmpl b/config.alloy.tmpl index 9fc6347..aa6f898 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -66,46 +66,41 @@ prometheus.remote_write "prometheus" { // prometheus.io/port=9090 optional: port exposing /metrics (defaults to first exposed port) // prometheus.io/path=/metrics optional: path to metrics endpoint (default: /metrics) // prometheus.io/auth=basic optional: use basic auth with the shared basic_auth secret -// -// Uses docker_gwbridge — the host-local bridge network Docker attaches all -// Swarm containers to for outbound connectivity. Alloy can reach any container -// on the same host via this network without needing to join each stack's -// overlay network. -discovery.docker "containers" { - host = "unix:///var/run/docker.sock" - match_first_network = false +discovery.dockerswarm "swarm" { + host = "unix:///var/run/docker.sock" + role = "services" } discovery.relabel "metrics" { - targets = discovery.docker.containers.targets + targets = discovery.dockerswarm.swarm.targets rule { - source_labels = ["__meta_docker_network_name"] + source_labels = ["__meta_dockerswarm_network_name"] regex = "docker_gwbridge" action = "keep" } rule { - source_labels = ["__meta_docker_container_label_prometheus_io_scrape"] + source_labels = ["__meta_dockerswarm_service_label_prometheus_io_scrape"] regex = "true" action = "keep" } rule { - source_labels = ["__address__", "__meta_docker_container_label_prometheus_io_port"] + source_labels = ["__address__", "__meta_dockerswarm_service_label_prometheus_io_port"] regex = `(.+):\d+;(\d+)` target_label = "__address__" replacement = "$1:$2" } rule { - source_labels = ["__meta_docker_container_label_prometheus_io_path"] + source_labels = ["__meta_dockerswarm_service_label_prometheus_io_path"] regex = `(.+)` target_label = "__metrics_path__" } rule { - source_labels = ["__meta_docker_container_label_com_docker_swarm_service_name"] + source_labels = ["__meta_dockerswarm_service_name"] target_label = "job" } } @@ -113,7 +108,7 @@ discovery.relabel "metrics" { discovery.relabel "metrics_noauth" { targets = discovery.relabel.metrics.output rule { - source_labels = ["__meta_docker_container_label_prometheus_io_auth"] + source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"] regex = "^$" action = "keep" } @@ -122,7 +117,7 @@ discovery.relabel "metrics_noauth" { discovery.relabel "metrics_basicauth" { targets = discovery.relabel.metrics.output rule { - source_labels = ["__meta_docker_container_label_prometheus_io_auth"] + source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"] regex = "basic" action = "keep" } @@ -131,7 +126,7 @@ discovery.relabel "metrics_basicauth" { discovery.relabel "metrics_bearerauth" { targets = discovery.relabel.metrics.output rule { - source_labels = ["__meta_docker_container_label_prometheus_io_auth"] + source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"] regex = "bearer" action = "keep" } -- 2.49.0 From 03227f19070a8fecabd19d31fc9fed2ba11904fa Mon Sep 17 00:00:00 2001 From: f Date: Tue, 16 Jun 2026 22:01:05 -0300 Subject: [PATCH 7/8] fix: send container metrics directly to prometheus --- config.alloy.tmpl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config.alloy.tmpl b/config.alloy.tmpl index aa6f898..be50eec 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -44,6 +44,7 @@ prometheus.scrape "default" { prometheus.exporter.self.alloy.targets, prometheus.exporter.unix.default.targets, prometheus.exporter.cadvisor.docker.targets, + discovery.docker.containers.targets, ) forward_to = [prometheus.remote_write.prometheus.receiver] @@ -60,6 +61,11 @@ prometheus.remote_write "prometheus" { } } +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" + match_first_network = false +} + // Scrape Prometheus metrics from other containers on this host. // Containers opt in via Docker labels: // prometheus.io/scrape=true required: enable scraping -- 2.49.0 From 23acf56637be039c5a3921ecc3bac0e66fe89459 Mon Sep 17 00:00:00 2001 From: f Date: Tue, 16 Jun 2026 22:01:16 -0300 Subject: [PATCH 8/8] fix: filter by proxy network --- abra.sh | 2 +- config.alloy.tmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/abra.sh b/abra.sh index cd2c005..86c77c9 100644 --- a/abra.sh +++ b/abra.sh @@ -11,7 +11,7 @@ export PROMETHEUS_YML_VERSION=v2 export MATRIX_ALERTMANAGER_CONFIG_VERSION=e export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a export GRAFANA_ALERTS_NODE_VERSION=v1c -export CONFIG_ALLOY_VERSION=v9 +export CONFIG_ALLOY_VERSION=v10 # creates a default prometheus scrape config for a given node add_node(){ diff --git a/config.alloy.tmpl b/config.alloy.tmpl index be50eec..33a8258 100644 --- a/config.alloy.tmpl +++ b/config.alloy.tmpl @@ -82,7 +82,7 @@ discovery.relabel "metrics" { rule { source_labels = ["__meta_dockerswarm_network_name"] - regex = "docker_gwbridge" + regex = "proxy" action = "keep" } -- 2.49.0