simon
e5f57c97f1
Reviewed-on: #30 Reviewed-by: ammaratef45 <ammaratef45@proton.me> Co-authored-by: Simon <s.thiessen@local-it.org>
409 lines
11 KiB
Cheetah
409 lines
11 KiB
Cheetah
logging {
|
|
level = "info"
|
|
format = "logfmt"
|
|
}
|
|
|
|
livedebugging {
|
|
enabled = {{ env "LIVE_DEBUGGING" }}
|
|
}
|
|
|
|
discovery.docker "linux" {
|
|
host = "unix:///var/run/docker.sock"
|
|
}
|
|
|
|
{{ if ne (env "PROMETHEUS_REMOTE_WRITE_URL") "" }}
|
|
prometheus.exporter.cadvisor "docker" {
|
|
docker_only = true
|
|
{{ if eq (env "CADVISOR_DISK_USAGE") "1" }}
|
|
enabled_metrics = ["cpu", "cpuLoad", "disk", "diskIO", "memory", "network"]
|
|
{{ else }}
|
|
enabled_metrics = ["cpu", "cpuLoad", "diskIO", "memory", "network"]
|
|
{{ end }}
|
|
// host-wide totals already come from prometheus.exporter.unix
|
|
disable_root_cgroup_stats = true
|
|
}
|
|
|
|
prometheus.exporter.unix "default" {
|
|
include_exporter_metrics = true
|
|
rootfs_path = "/rootfs"
|
|
procfs_path = "/rootfs/proc"
|
|
sysfs_path = "/rootfs/sys"
|
|
|
|
disable_collectors = ["ipvs"]
|
|
|
|
filesystem {
|
|
fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
|
|
mount_points_exclude = "^/(sys|proc|dev|host|etc)($|/)"
|
|
mount_timeout = "5s"
|
|
}
|
|
|
|
netclass { ignored_devices = "^(veth.*)$" }
|
|
netdev { device_exclude = "^(veth.*)$" }
|
|
}
|
|
|
|
prometheus.exporter.self "alloy" {}
|
|
|
|
prometheus.scrape "default" {
|
|
scrape_interval = "120s"
|
|
|
|
targets = array.concat(
|
|
prometheus.exporter.self.alloy.targets,
|
|
prometheus.exporter.unix.default.targets,
|
|
prometheus.exporter.cadvisor.docker.targets,
|
|
)
|
|
|
|
forward_to = [prometheus.relabel.container_meta.receiver]
|
|
}
|
|
|
|
prometheus.relabel "container_meta" {
|
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
|
|
|
// remove sha tail: nginx:1.31.1@sha256:608a... -> nginx:1.31.1
|
|
rule {
|
|
source_labels = ["image"]
|
|
regex = "([^@]+)@sha256:.*"
|
|
target_label = "image"
|
|
replacement = "$1"
|
|
}
|
|
// split image in name and tag
|
|
rule {
|
|
source_labels = ["image"]
|
|
regex = "(.+):[^:/]+"
|
|
target_label = "image_name"
|
|
replacement = "$1"
|
|
}
|
|
rule {
|
|
source_labels = ["image"]
|
|
regex = ".+:([^:/]+)"
|
|
target_label = "image_tag"
|
|
replacement = "$1"
|
|
}
|
|
}
|
|
|
|
prometheus.remote_write "prometheus" {
|
|
endpoint {
|
|
url = "{{ env "PROMETHEUS_REMOTE_WRITE_URL" }}"
|
|
|
|
basic_auth {
|
|
username = "{{ env "WRITE_BASIC_AUTH_USERNAME" }}"
|
|
password = "{{ secret "basic_auth" }}"
|
|
}
|
|
}
|
|
}
|
|
|
|
// Scrape Prometheus metrics from other containers on this host.
|
|
// Containers opt in via Docker labels:
|
|
// prometheus.io/scrape=true required: enable scraping
|
|
// prometheus.io/port=9090 optional: port exposing /metrics (defaults to 80 if not set)
|
|
// prometheus.io/path=/metrics optional: path to metrics endpoint (default: /metrics)
|
|
// prometheus.io/auth=basic optional: use basic auth with the scrape_auth secret (see compose.scrape-auth.yml)
|
|
// prometheus.io/auth=bearer optional: use bearer auth with the scrape_auth secret (see compose.scrape-auth.yml)
|
|
discovery.dockerswarm "swarm" {
|
|
host = "unix:///var/run/docker.sock"
|
|
// "tasks" not "services": dnsrr-mode services (e.g. traefik) have no VIP
|
|
// and are invisible to the "services" role
|
|
role = "tasks"
|
|
}
|
|
|
|
discovery.relabel "metrics" {
|
|
targets = discovery.dockerswarm.swarm.targets
|
|
|
|
// skip old task history, only scrape currently-running tasks
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_task_desired_state"]
|
|
regex = "running"
|
|
action = "keep"
|
|
}
|
|
|
|
// only scrape hosts running on this host within the swam
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_node_id"]
|
|
regex = "{{ env "NODE_ID" }}"
|
|
action = "keep"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_network_name"]
|
|
regex = "proxy"
|
|
action = "keep"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_prometheus_io_scrape"]
|
|
regex = "true"
|
|
action = "keep"
|
|
}
|
|
|
|
// default to port 80 when prometheus.io/port isn't set
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_prometheus_io_port"]
|
|
regex = "^$"
|
|
target_label = "__meta_dockerswarm_service_label_prometheus_io_port"
|
|
replacement = "80"
|
|
}
|
|
|
|
// a task with multiple published ports produces one target per port;
|
|
// this unifies all of them to the single port above, so duplicates
|
|
// collapse at scrape time instead of scraping every port
|
|
rule {
|
|
source_labels = ["__address__", "__meta_dockerswarm_service_label_prometheus_io_port"]
|
|
regex = `(.+):\d+;(\d+)`
|
|
target_label = "__address__"
|
|
replacement = "$1:$2"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_prometheus_io_path"]
|
|
regex = `(.+)`
|
|
target_label = "__metrics_path__"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_name"]
|
|
target_label = "job"
|
|
}
|
|
|
|
// task IDs (and the default address-derived instance label) change on
|
|
// every redeploy; service+slot is stable across redeploys of the same
|
|
// replica, so data stays continuous instead of restarting each deploy
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_name", "__meta_dockerswarm_task_slot"]
|
|
separator = "."
|
|
target_label = "instance"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_task_slot"]
|
|
target_label = "task_slot"
|
|
}
|
|
|
|
// coop-cloud's STACK_NAME is the domain with "." replaced by "_"
|
|
// Derive a readable dotted domain label from it. RE2 has no global
|
|
// replace, so this is done by chaining multiple replacements. Each
|
|
// rule swaps the first remaining _ for a "." until none are left.
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_com_docker_stack_namespace"]
|
|
target_label = "domain"
|
|
}
|
|
rule {
|
|
source_labels = ["domain"]
|
|
regex = `([^_]*)_(.*)`
|
|
target_label = "domain"
|
|
replacement = "$1.$2"
|
|
}
|
|
rule {
|
|
source_labels = ["domain"]
|
|
regex = `([^_]*)_(.*)`
|
|
target_label = "domain"
|
|
replacement = "$1.$2"
|
|
}
|
|
rule {
|
|
source_labels = ["domain"]
|
|
regex = `([^_]*)_(.*)`
|
|
target_label = "domain"
|
|
replacement = "$1.$2"
|
|
}
|
|
rule {
|
|
source_labels = ["domain"]
|
|
regex = `([^_]*)_(.*)`
|
|
target_label = "domain"
|
|
replacement = "$1.$2"
|
|
}
|
|
rule {
|
|
source_labels = ["domain"]
|
|
regex = `([^_]*)_(.*)`
|
|
target_label = "domain"
|
|
replacement = "$1.$2"
|
|
}
|
|
rule {
|
|
source_labels = ["domain"]
|
|
regex = `([^_]*)_(.*)`
|
|
target_label = "domain"
|
|
replacement = "$1.$2"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "metrics_noauth" {
|
|
targets = discovery.relabel.metrics.output
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"]
|
|
regex = "^$"
|
|
action = "keep"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "metrics_basicauth" {
|
|
targets = discovery.relabel.metrics.output
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"]
|
|
regex = "basic"
|
|
action = "keep"
|
|
}
|
|
}
|
|
|
|
discovery.relabel "metrics_bearerauth" {
|
|
targets = discovery.relabel.metrics.output
|
|
rule {
|
|
source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"]
|
|
regex = "bearer"
|
|
action = "keep"
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "containers" {
|
|
scrape_interval = "120s"
|
|
targets = discovery.relabel.metrics_noauth.output
|
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
|
}
|
|
|
|
{{ if ne (env "SECRET_SCRAPE_AUTH_VERSION") "" }}
|
|
prometheus.scrape "containers_basicauth" {
|
|
scrape_interval = "120s"
|
|
targets = discovery.relabel.metrics_basicauth.output
|
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
|
basic_auth {
|
|
username = "{{ env "SCRAPE_BASIC_AUTH_USERNAME" }}"
|
|
password = "{{ secret "scrape_auth" }}"
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "containers_bearerauth" {
|
|
scrape_interval = "120s"
|
|
targets = discovery.relabel.metrics_bearerauth.output
|
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
|
bearer_token = "{{ secret "scrape_auth" }}"
|
|
}
|
|
{{ end }}
|
|
{{ end }}
|
|
|
|
{{ if ne (env "LOKI_PUSH_URL") "" }}
|
|
discovery.relabel "docker" {
|
|
targets = discovery.docker.linux.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_docker_container_name"]
|
|
target_label = "container_name"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_id"]
|
|
target_label = "container_id"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_stack_namespace"]
|
|
target_label = "stack_namespace"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_label_com_docker_swarm_service_name"]
|
|
target_label = "service_name"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_docker_container_log_stream"]
|
|
target_label = "stream"
|
|
}
|
|
}
|
|
|
|
loki.source.docker "docker" {
|
|
host = "unix:///var/run/docker.sock"
|
|
targets = discovery.relabel.docker.output
|
|
labels = {"app" = "docker"}
|
|
forward_to = [loki.write.loki.receiver]
|
|
}
|
|
|
|
// JOURNALD: reads the systemd journal binary log directly.
|
|
// Use on systemd hosts (most modern Linux distros). Requires no syslogd.
|
|
{{ if eq (env "JOURNALD") "1" }}
|
|
loki.source.journal "journal" {
|
|
path = "/rootfs/var/log/journal"
|
|
labels = { job = "{{ env "DOMAIN" }}" }
|
|
relabel_rules = loki.relabel.journal.rules
|
|
forward_to = [loki.process.journal.receiver]
|
|
}
|
|
|
|
loki.relabel "journal" {
|
|
forward_to = []
|
|
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
target_label = "unit"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_syslog_identifier"]
|
|
target_label = "ident"
|
|
}
|
|
rule {
|
|
source_labels = ["__journal_priority_keyword"]
|
|
target_label = "priority"
|
|
}
|
|
}
|
|
|
|
// drop network db stats (15-20% of kernel logs)
|
|
loki.process "journal" {
|
|
forward_to = [loki.write.loki.receiver]
|
|
|
|
stage.drop {
|
|
expression = ".*NetworkDB stats.*"
|
|
drop_counter_reason = "networkdb_stats"
|
|
}
|
|
}
|
|
{{ end }}
|
|
|
|
// SYSLOG_FILES: tails all /var/log/*log files (syslog, auth.log, kern.log, etc.).
|
|
// Use on non-systemd hosts where a syslogd writes to /var/log.
|
|
{{ if eq (env "SYSLOG_FILES") "1" }}
|
|
local.file_match "syslog_files" {
|
|
path_targets = [{ __path__ = "/rootfs/var/log/*log" }]
|
|
}
|
|
|
|
loki.source.file "syslog_files" {
|
|
targets = local.file_match.syslog_files.targets
|
|
forward_to = [loki.process.syslog_files.receiver]
|
|
}
|
|
|
|
loki.process "syslog_files" {
|
|
stage.static_labels {
|
|
values = { job = "syslog" }
|
|
}
|
|
forward_to = [loki.write.loki.receiver]
|
|
}
|
|
{{ end }}
|
|
|
|
// SYSLOG: opens a network syslog listener on port 514.
|
|
// Use when a remote device or a local syslogd configured to
|
|
// forward over the network sends logs to this host.
|
|
// Requires compose.syslog.yml to publish port 514 to the host.
|
|
// This is NOT needed for reading local log files — use SYSLOG_FILES instead.
|
|
{{ if eq (env "SYSLOG") "1" }}
|
|
loki.relabel "syslog" {
|
|
rule {
|
|
action = "labelmap"
|
|
regex = "__syslog_(.+)"
|
|
}
|
|
|
|
forward_to = []
|
|
}
|
|
|
|
loki.source.syslog "syslog" {
|
|
listener {
|
|
address = "[::]:514"
|
|
label_structured_data = true
|
|
labels = { component = "loki.source.syslog" }
|
|
}
|
|
|
|
relabel_rules = loki.relabel.syslog.rules
|
|
forward_to = [loki.write.loki.receiver]
|
|
}
|
|
{{ end }}
|
|
|
|
loki.write "loki" {
|
|
endpoint {
|
|
url = "{{ env "LOKI_PUSH_URL" }}"
|
|
|
|
basic_auth {
|
|
username = "{{ env "WRITE_BASIC_AUTH_USERNAME" }}"
|
|
password = "{{ secret "basic_auth" }}"
|
|
}
|
|
}
|
|
external_labels = { hostname = "{{ env "DOMAIN" }}" }
|
|
}
|
|
{{ end }}
|