feat: access log coop-cloud/traefik#126

Reviewed-on: #24
Reviewed-by: fauno <fauno@sutty.coop.ar>

.env.sample

@@ -6,13 +6,23 @@ DOMAIN=monitoring-ng.example.com
 ENABLE_BACKUPS=true
 
 SECRET_BASIC_AUTH_VERSION=v1
+# Enable Live Debugging
+LIVE_DEBUGGING=false
 # Enable this to send logs to a Loki server, adapt DOMAIN if server is
 # remote
 # LOKI_PUSH_URL=https://loki.$DOMAIN/loki/api/v1/push
-# Enable this on SystemD hosts to read logs
+# Enable on systemd hosts to read logs from the journal
 # JOURNALD=1
-# Enable this on syslogd hosts and configure the syslogd to send logs to
-# Alloy on port 514/tcp
+#
+# Enable on non-systemd hosts (Alpine, older Debian/Ubuntu) to tail
+# /var/log/*log files (syslog, auth.log, kern.log, etc.) that a local
+# syslogd writes. No syslogd reconfiguration needed.
+# SYSLOG_FILES=1
+#
+# Enable to receive syslog messages over the network on port 514/tcp.
+# Use for remote devices that push syslog to this host, or for a 
+# local syslogd configured to forward over the network.
+# Not needed if you just want to read local log files — use SYSLOG_FILES instead.
 # SYSLOG=1
 # COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml"
 
@@ -82,9 +92,16 @@ SECRET_BASIC_AUTH_VERSION=v1
 #GF_MATRIX_ROOM_ID="<room-id>"
 #GF_MATRIX_HOMESERVER_URL="<homeserver-url>"
 
-# ALerts
-#ALERT_BACKUP_FAILED_ENABLED=true
-#ALERT_BACKUP_MISSING_ENABLED=true
-#ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED=true
-#ALERT_NODE_DISK_SPACE_ENABLED=true
-#ALERT_NODE_MEMORY_USAGE_ENABLED=true
+## ALerts
+
+# Node disk space alert will trigger when free disk space left is below the given number in percent
+#ALERT_NODE_DISK_SPACE_LEFT=10
+
+# Node memory usage alert will trigger when memory usage is above the given number in percent
+#ALERT_NODE_MEMORY_USAGE=85
+
+# Tell Traefik to keep access logs (could be very verbose)
+ALLOY_ACCESS_LOGS=false
+PROMETHEUS_ACCESS_LOGS=false
+LOKI_ACCESS_LOGS=false
+GRAFANA_ACCESS_LOGS=false

README.md

@@ -18,47 +18,30 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou
 
 <!-- endmetadata -->
 
-## Setup a Metrics Gathering
+## Setup Metrics Gathering
 
 Where gathering.org is the node you want to gather metrics from.
 
 1. Configure DNS
-  - monitoring.gathering.org
   - cadvisor.monitoring.gathering.org
   - node.monitoring.gathering.org
-1. Configure Traefik to use BasicAuth
-  * `abra app config traefik.gathering.org`
-  uncomment
-  ```
-  # BASIC_AUTH
-  COMPOSE_FILE="$COMPOSE_FILE:compose.basicauth.yml"
-  BASIC_AUTH=1
-  SECRET_USERSFILE_VERSION=v1
-  ```
-  - Generate userslist with httpasswd hashed password
-    `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
-    make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
-  - `abra app deploy -f traefik`
-1. `abra app new monitoring-ng`
-1. `abra app config monitoring.gathering.org`
-   for gathering only the main `compose.yml` is needed, nothing more.
-1. `abra app deploy monitoring.gathering.org`
-1. check that endpoints are up and basic-auth works
+2. [Configure Traefik to use BasicAuth](https://git.coopcloud.tech/coop-cloud/traefik#configuring-wildcard-ssl-using-dns)
+3. `abra app new monitoring-ng`
+4. `abra app config monitoring.gathering.org` (for gathering only the main `compose.yml` is needed, nothing more.)
+5. `abra app deploy monitoring.gathering.org`
+6. check that endpoints are up and basic-auth works
   - cadvisor.monitoring.gathering.org
   - node.monitoring.gathering.org
 
 ## Setup Metrics Browser
 
+This builds upon [Setup Metrics Gathering](#setup-metrics-grathering) so make sure you did that first.
 
 1. Configure DNS
    - monitoring.example.org
-   - prometheus.monitoring.example.org
-   - loki.monitoring.example.org
 2. Setup monitoring stack
-   - `abra app new monitoring-ng`
-   - `abra app config monitoring.example.org`
-      Uncomment all the stuff
-   - `abra app secret insert monitoring.example.org basic_auth v1 <secret>`
+   - `abra app config monitoring.example.org` Uncomment prometheus, loki and grafana
+   - `abra app secret insert monitoring.example.org basic_auth v1 <password>`
       this needs the plaintext traefik basic-auth secret, not the hashed one!
    - `abra app secret ls monitoring.example.org`
    - `abra app deploy monitoring.example.org`
@@ -149,13 +132,9 @@ GF_MATRIX_HOME_SERVER_URL=
 ```
 4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`
 
-## alerts
-
-It is possible to enable the following alerts, by setting the corresponding env variable to `true`:
-- backupbot failed: `ALERT_BACKUP_FAILED_ENABLED`
-- backupbot missing: `ALERT_BACKUP_MISSING_ENABLED`
-- backupbot not successfull: `ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED`
-- node disk space: `ALERT_NODE_DISK_SPACE_ENABLED`
-- node memory usage: `ALERT_NODE_MEMORY_USAGE_ENABLED`
+## Alerts
 
+It is possible to enable the following alerts, by uncommenting the corresponding env variable:
 
+- node disk space: `ALERT_NODE_DISK_SPACE_LEFT`
+- node memory usage: `ALERT_NODE_MEMORY_USAGE`

abra.sh

@@ -8,10 +8,10 @@ export GF_BACKUP_DASH_VERSION=v1
 export GF_CUSTOM_INI_VERSION=v4
 export LOKI_YML_VERSION=v3
 export PROMETHEUS_YML_VERSION=v2
-export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
-export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
-export GRAFANA_ALERTS_NODE_VERSION=v1c
-export CONFIG_ALLOY_VERSION=v9
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
+export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
+export GRAFANA_ALERTS_NODE_VERSION=v2
+export CONFIG_ALLOY_VERSION=v10
 
 # creates a default prometheus scrape config for a given node
 add_node(){

alerts/node.yml.tmpl

@@ -2,13 +2,13 @@ apiVersion: 1
 
 # List of alert rule UIDs that should be deleted
 deleteRules:
-  {{ if ne (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
+  {{ if not (env "ALERT_NODE_DISK_SPACE_LEFT") }}
   - orgId: 1
-    uid: bds8bhxu97pxca
+    uid: coopcloud_node_disk_space_left
   {{ end }}
-  {{ if ne (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
+  {{ if not (env "ALERT_NODE_MEMORY_USAGE") }}
   - orgId: 1
-    uid: ads8cswmly96oa
+    uid: coopcloud_node_memory_usage
   {{ end }}
 
 groups:
@@ -17,8 +17,8 @@ groups:
   folder: node
   interval: 5m
   rules:
-    {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-    - uid: bds8bhxu97pxca
+    {{ if (env "ALERT_NODE_DISK_SPACE_LEFT") }}
+    - uid: coopcloud_node_disk_space_left
       title: Node Disk Space
       condition: C
       data:
@@ -45,7 +45,7 @@ groups:
             conditions:
                 - evaluator:
                     params:
-                        - 10
+                        - {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}
                     type: lt
                   operator:
                     type: and
@@ -70,13 +70,13 @@ groups:
       annotations:
         description: ""
         runbook_url: ""
-        summary: Less than 10% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
+        summary: Less than {{ env "ALERT_NODE_DISK_SPACE_LEFT" }}% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
       labels:
         "": ""
       isPaused: false
     {{ end }}
-    {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-    - uid: ads8cswmly96oa
+    {{ if (env "ALERT_NODE_MEMORY_USAGE")  }}
+    - uid: coopcloud_node_memory_usage
       title: Node Memory Usage
       condition: C
       data:
@@ -103,7 +103,7 @@ groups:
             conditions:
                 - evaluator:
                     params:
-                        - 85
+                        - {{ env "ALERT_NODE_MEMORY_USAGE"  }}
                     type: gt
                   operator:
                     type: and
@@ -126,6 +126,6 @@ groups:
       execErrState: Error
       for: 5m
       annotations:
-        summary: Memory usage is above 85% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
+        summary: Memory usage is above {{ env "ALERT_NODE_MEMORY_USAGE"  }}% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
       isPaused: false
     {{ end }}

compose.basic-auth.yml

@@ -0,0 +1,7 @@
+---
+version: "3.8"
+
+secrets:
+  basic_auth:
+    external: true
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.grafana.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:12.4.3
+    image: grafana/grafana:13.0.3
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:
@@ -32,8 +32,8 @@ services:
       - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/gf_adminpasswd
       - GF_SECURITY_ALLOW_EMBEDDING
       - GF_INSTALL_PLUGINS
-      - ALERT_NODE_DISK_SPACE_ENABLED
-      - ALERT_NODE_MEMORY_USAGE_ENABLED
+      - ALERT_NODE_DISK_SPACE_LEFT
+      - ALERT_NODE_MEMORY_USAGE
     deploy:
       labels:
         - "traefik.enable=true"
@@ -43,6 +43,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-grafana.observability.accesslogs=${GRAFANA_ACCESS_LOGS:-false}"
     healthcheck:
       test: "wget -q http://localhost:3000/healthz -O/dev/null"
       interval: 5s

compose.loki.yml

@@ -34,6 +34,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}-loki.tls=true"
         - "traefik.http.routers.${STACK_NAME}-loki.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-loki.middlewares=basicauth@file"
+        - "traefik.http.routers.${STACK_NAME}-loki.observability.accesslogs=${LOKI_ACCESS_LOGS:-false}"
 
 
 configs:

compose.prometheus.yml

@@ -16,7 +16,6 @@ services:
       - "--web.console.libraries=/usr/share/prometheus/console_libraries"
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME}"
-      - "--enable-feature=remote-write-receiver"
       - "--web.enable-remote-write-receiver"
     networks:
       - proxy
@@ -33,6 +32,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
+        - "traefik.http.routers.${STACK_NAME}-prometheus.observability.accesslogs=${PROMETHEUS_ACCESS_LOGS:-false}"
 
 configs:
   prometheus_yml:

compose.yml

@@ -3,24 +3,24 @@ version: "3.8"
 
 services:
   app:
-    image: grafana/alloy:v1.16.1
+    image: grafana/alloy:v1.17.0
     hostname: "${DOMAIN}"
     configs:
       - source: config_alloy
         target: /etc/alloy/config.alloy
     volumes:
       - /:/rootfs:ro
-      - /var/run:/var/run:rw
-      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/run/docker.sock:/var/run/docker.sock:ro
       - /sys:/sys:ro
       - /var/lib/docker:/var/lib/docker:ro
-      - /dev:/dev:ro
       - alloy-data:/var/lib/alloy/data
     command:
       - "run"
       - "--storage.path=/var/lib/alloy/data"
+      - "--server.http.listen-addr=0.0.0.0:12345"
       - "/etc/alloy/config.alloy"
     networks:
+      - proxy
       - internal
     secrets:
       - basic_auth
@@ -29,8 +29,16 @@ services:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "traefik.enable=false"
         - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
+        - "traefik.enable=true"
+        - "traefik.swarm.network=proxy"
+        - "traefik.http.services.${STACK_NAME}-alloy.loadbalancer.server.port=12345"
+        - "traefik.http.routers.${STACK_NAME}-alloy.rule=Host(`alloy.${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-alloy.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-alloy.tls=true"
+        - "traefik.http.routers.${STACK_NAME}-alloy.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-alloy.middlewares=basicauth@file"
+        - "traefik.http.routers.${STACK_NAME}-alloy.observability.accesslogs=${ALLOY_ACCESS_LOGS:-false}"
 configs:
   config_alloy:
     template_driver: golang

config.alloy.tmpl

@@ -3,27 +3,48 @@ logging {
   format = "logfmt"
 }
 
+livedebugging {
+  enabled = {{ env "LIVE_DEBUGGING" }}
+}
+
 discovery.docker "linux" {
   host = "unix:///var/run/docker.sock"
 }
 
 {{ if ne (env "PROMETHEUS_REMOTE_WRITE_URL") "" }}
 prometheus.exporter.cadvisor "docker" {
+  docker_only = true
+  enabled_metrics = ["cpu", "cpuLoad", "disk", "diskIO", "memory", "network", "process"]
 }
 
 prometheus.exporter.unix "default" {
   include_exporter_metrics = true
   rootfs_path = "/rootfs"
+  procfs_path = "/rootfs/proc"
+  sysfs_path = "/rootfs/sys"
+
+  disable_collectors = ["ipvs"]
+
+  filesystem {
+    fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
+    mount_points_exclude = "^/(sys|proc|dev|host|etc)($|/)"
+    mount_timeout        = "5s"
+  }
+
+  netclass { ignored_devices = "^(veth.*)$" }
+  netdev   { device_exclude  = "^(veth.*)$" }
 }
 
+prometheus.exporter.self "alloy" {}
+
 prometheus.scrape "default" {
+  scrape_interval = "120s"
+
   targets = array.concat(
-    [{
-      job         = "alloy",
-      __address__ = "127.0.0.1:12345",
-    }],
+    prometheus.exporter.self.alloy.targets,
     prometheus.exporter.unix.default.targets,
     prometheus.exporter.cadvisor.docker.targets,
+    discovery.docker.containers.targets,
   )
 
   forward_to = [prometheus.remote_write.prometheus.receiver]
@@ -39,24 +60,176 @@ prometheus.remote_write "prometheus" {
     }
   }
 }
+
+discovery.docker "containers" {
+  host                = "unix:///var/run/docker.sock"
+  match_first_network = false
+}
+
+// Scrape Prometheus metrics from other containers on this host.
+// Containers opt in via Docker labels:
+//   prometheus.io/scrape=true        required: enable scraping
+//   prometheus.io/port=9090          optional: port exposing /metrics (defaults to first exposed port)
+//   prometheus.io/path=/metrics      optional: path to metrics endpoint (default: /metrics)
+//   prometheus.io/auth=basic         optional: use basic auth with the shared basic_auth secret
+discovery.dockerswarm "swarm" {
+  host = "unix:///var/run/docker.sock"
+  role = "services"
+}
+
+discovery.relabel "metrics" {
+  targets = discovery.dockerswarm.swarm.targets
+
+  rule {
+    source_labels = ["__meta_dockerswarm_network_name"]
+    regex         = "proxy"
+    action        = "keep"
+  }
+
+  rule {
+    source_labels = ["__meta_dockerswarm_service_label_prometheus_io_scrape"]
+    regex         = "true"
+    action        = "keep"
+  }
+
+  rule {
+    source_labels = ["__address__", "__meta_dockerswarm_service_label_prometheus_io_port"]
+    regex         = `(.+):\d+;(\d+)`
+    target_label  = "__address__"
+    replacement   = "$1:$2"
+  }
+
+  rule {
+    source_labels = ["__meta_dockerswarm_service_label_prometheus_io_path"]
+    regex         = `(.+)`
+    target_label  = "__metrics_path__"
+  }
+
+  rule {
+    source_labels = ["__meta_dockerswarm_service_name"]
+    target_label  = "job"
+  }
+}
+
+discovery.relabel "metrics_noauth" {
+  targets = discovery.relabel.metrics.output
+  rule {
+    source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"]
+    regex         = "^$"
+    action        = "keep"
+  }
+}
+
+discovery.relabel "metrics_basicauth" {
+  targets = discovery.relabel.metrics.output
+  rule {
+    source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"]
+    regex         = "basic"
+    action        = "keep"
+  }
+}
+
+discovery.relabel "metrics_bearerauth" {
+  targets = discovery.relabel.metrics.output
+  rule {
+    source_labels = ["__meta_dockerswarm_service_label_prometheus_io_auth"]
+    regex         = "bearer"
+    action        = "keep"
+  }
+}
+
+prometheus.scrape "containers" {
+  scrape_interval = "120s"
+  targets         = discovery.relabel.metrics_noauth.output
+  forward_to      = [prometheus.remote_write.prometheus.receiver]
+}
+
+prometheus.scrape "containers_basicauth" {
+  scrape_interval = "120s"
+  targets         = discovery.relabel.metrics_basicauth.output
+  forward_to      = [prometheus.remote_write.prometheus.receiver]
+  basic_auth {
+    username = "admin"
+    password = "{{ secret "basic_auth" }}"
+  }
+}
+
+prometheus.scrape "containers_bearerauth" {
+  scrape_interval = "120s"
+  targets         = discovery.relabel.metrics_bearerauth.output
+  forward_to      = [prometheus.remote_write.prometheus.receiver]
+  bearer_token    = "{{ secret "basic_auth" }}"
+}
 {{ end }}
 
 {{ if ne (env "LOKI_PUSH_URL") "" }}
+discovery.relabel "docker" {
+  targets = discovery.docker.linux.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    target_label  = "container_name"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_id"]
+    target_label  = "container_id"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_stack_namespace"]
+    target_label  = "stack_namespace"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_swarm_service_name"]
+    target_label  = "service_name"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_log_stream"]
+    target_label  = "stream"
+  }
+}
+
 loki.source.docker "docker" {
   host       = "unix:///var/run/docker.sock"
-  targets    = discovery.docker.linux.targets
+  targets    = discovery.relabel.docker.output
   labels     = {"app" = "docker"}
   forward_to = [loki.write.loki.receiver]
 }
 
+// JOURNALD: reads the systemd journal binary log directly.
+// Use on systemd hosts (most modern Linux distros). Requires no syslogd.
 {{ if eq (env "JOURNALD") "1" }}
 loki.source.journal "journal" {
-  path = "/var/log/journal"
+  path = "/rootfs/var/log/journal"
   labels = { job = "{{ env "DOMAIN" }}" }
   forward_to = [loki.write.loki.receiver]
 }
 {{ end }}
 
+// SYSLOG_FILES: tails all /var/log/*log files (syslog, auth.log, kern.log, etc.).
+// Use on non-systemd hosts where a syslogd writes to /var/log.
+{{ if eq (env "SYSLOG_FILES") "1" }}
+local.file_match "syslog_files" {
+  path_targets = [{ __path__ = "/rootfs/var/log/*log" }]
+}
+
+loki.source.file "syslog_files" {
+  targets    = local.file_match.syslog_files.targets
+  forward_to = [loki.process.syslog_files.receiver]
+}
+
+loki.process "syslog_files" {
+  stage.static_labels {
+    values = { job = "syslog" }
+  }
+  forward_to = [loki.write.loki.receiver]
+}
+{{ end }}
+
+// SYSLOG: opens a network syslog listener on port 514.
+// Use when a remote device or a local syslogd configured to 
+// forward over the network sends logs to this host.
+// Requires compose.syslog.yml to publish port 514 to the host.
+// This is NOT needed for reading local log files — use SYSLOG_FILES instead.
 {{ if eq (env "SYSLOG") "1" }}
 loki.relabel "syslog" {
   rule {
@@ -69,7 +242,7 @@ loki.relabel "syslog" {
 
 loki.source.syslog "syslog" {
   listener {
-    address = "[::1]:514"
+    address = "[::]:514"
     label_structured_data = true
     labels = { component = "loki.source.syslog" }
   }
@@ -88,5 +261,6 @@ loki.write "loki" {
       password = "{{ secret "basic_auth" }}"
     }
   }
+  external_labels = { hostname = "{{ env "DOMAIN" }}" }
 }
 {{ end }}

grafana-alerts.json.tmpl

@@ -1,315 +0,0 @@
-{
-  "apiVersion": 1,
-  "groups": [
-    {
-      "orgId": 1,
-      "name": "backupbot",
-      "folder": "node",
-      "interval": "1m",
-      "rules": [
-        {{ if eq (env "ALERT_BACKUP_FAILED_ENABLED") "true" }}
-        {
-          "uid": "de8e5xxup7t34a",
-          "title": "Backup Failed",
-          "condition": "C",
-          "data": [
-            {
-              "refId": "A",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "PBFA97CFB590B2093",
-              "model": {
-                "disableTextWrap": false,
-                "editorMode": "builder",
-                "expr": "backup",
-                "fullMetaSearch": false,
-                "includeNullMetadata": true,
-                "instant": true,
-                "intervalMs": 1000,
-                "legendFormat": "__auto",
-                "maxDataPoints": 43200,
-                "range": false,
-                "refId": "A",
-                "useBackend": false
-              }
-            },
-            {
-              "refId": "C",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "__expr__",
-              "model": {
-                "conditions": [
-                  {
-                    "evaluator": { "params": [0], "type": "lt" },
-                    "operator": { "type": "and" },
-                    "query": { "params": ["C"] },
-                    "reducer": { "params": [], "type": "last" },
-                    "type": "query"
-                  }
-                ],
-                "datasource": { "type": "__expr__", "uid": "__expr__" },
-                "expression": "A",
-                "intervalMs": 1000,
-                "maxDataPoints": 43200,
-                "refId": "C",
-                "type": "threshold"
-              }
-            }
-          ],
-          "noDataState": "NoData",
-          "execErrState": "Error",
-          "for": "1m",
-          "isPaused": false
-        },
-        {{ end }}
-        {{ if eq (env "ALERT_BACKUP_MISSING_ENABLED") "true" }}
-        {
-          "uid": "ce8e65uddcwe8d",
-          "title": "Backup Missing",
-          "condition": "B",
-          "data": [
-            {
-              "refId": "A",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "PBFA97CFB590B2093",
-              "model": {
-                "disableTextWrap": false,
-                "editorMode": "builder",
-                "expr": "rate(backup[24h])",
-                "fullMetaSearch": false,
-                "includeNullMetadata": true,
-                "instant": true,
-                "intervalMs": 1000,
-                "legendFormat": "__auto",
-                "maxDataPoints": 43200,
-                "range": false,
-                "refId": "A",
-                "useBackend": false
-              }
-            },
-            {
-              "refId": "B",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "__expr__",
-              "model": {
-                "conditions": [
-                  {
-                    "evaluator": { "params": [0, 0], "type": "within_range" },
-                    "operator": { "type": "and" },
-                    "query": { "params": ["C"] },
-                    "reducer": { "params": [], "type": "last" },
-                    "type": "query"
-                  }
-                ],
-                "datasource": { "type": "__expr__", "uid": "__expr__" },
-                "expression": "A",
-                "intervalMs": 1000,
-                "maxDataPoints": 43200,
-                "refId": "B",
-                "type": "threshold"
-              }
-            }
-          ],
-          "noDataState": "NoData",
-          "execErrState": "Error",
-          "for": "5m",
-          "isPaused": false
-        },
-        {{ end }}
-        {{ if eq (env "ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED") "true" }}
-        {
-            "uid": "de8e6bc92a8lcc",
-            "title": "Backup Not Successfull",
-            "condition": "B",
-            "data": [
-                {
-                    "refId": "A",
-                    "relativeTimeRange": {
-                        "from": 60,
-                        "to": 0
-                    },
-                    "datasourceUid": "PBFA97CFB590B2093",
-                    "model": {
-                        "disableTextWrap": false,
-                        "editorMode": "builder",
-                        "expr": "backup",
-                        "fullMetaSearch": false,
-                        "includeNullMetadata": true,
-                        "instant": true,
-                        "intervalMs": 1000,
-                        "legendFormat": "__auto",
-                        "maxDataPoints": 43200,
-                        "range": false,
-                        "refId": "A",
-                        "useBackend": false
-                    }
-                },
-                {
-                    "refId": "B",
-                    "relativeTimeRange": {
-                        "from": 60,
-                        "to": 0
-                    },
-                    "datasourceUid": "__expr__",
-                    "model": {
-                        "conditions": [
-                            {
-                                "evaluator": {
-                                    "params": [
-                                        0
-                                    ],
-                                    "type": "gt"
-                                },
-                                "operator": {
-                                    "type": "and"
-                                },
-                                "query": {
-                                    "params": [
-                                        "C"
-                                    ]
-                                },
-                                "reducer": {
-                                    "params": [],
-                                    "type": "last"
-                                },
-                                "type": "query"
-                            }
-                        ],
-                        "datasource": {
-                            "type": "__expr__",
-                            "uid": "__expr__"
-                        },
-                        "expression": "A",
-                        "intervalMs": 1000,
-                        "maxDataPoints": 43200,
-                        "refId": "B",
-                        "type": "threshold"
-                    }
-                }
-            ],
-            "noDataState": "NoData",
-            "execErrState": "Error",
-            "for": "20m",
-            "annotations": {
-                "summary": "Backup did not finish within 20 minutes"
-            },
-            "labels": {},
-            "isPaused": false
-        }
-        {{ end }}
-      ]
-    },
-    {
-      "orgId": 1,
-      "name": "node",
-      "folder": "node",
-      "interval": "5m",
-      "rules": [
-        {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-        {
-          "uid": "bds8bhxu97pxca",
-          "title": "Node Disk Space",
-          "condition": "C",
-          "data": [
-            {
-              "refId": "A",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "PBFA97CFB590B2093",
-              "model": {
-                "editorMode": "code",
-                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
-                "instant": true,
-                "intervalMs": 1000,
-                "legendFormat": "__auto",
-                "maxDataPoints": 43200,
-                "range": false,
-                "refId": "A"
-              }
-            },
-            {
-              "refId": "C",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "__expr__",
-              "model": {
-                "conditions": [
-                  {
-                    "evaluator": { "params": [10], "type": "lt" },
-                    "operator": { "type": "and" },
-                    "query": { "params": ["C"] },
-                    "reducer": { "params": [], "type": "last" },
-                    "type": "query"
-                  }
-                ],
-                "datasource": { "type": "__expr__", "uid": "__expr__" },
-                "expression": "A",
-                "intervalMs": 1000,
-                "maxDataPoints": 43200,
-                "refId": "C",
-                "type": "threshold"
-              }
-            }
-          ],
-          "noDataState": "NoData",
-          "execErrState": "Error",
-          "for": "5m",
-          "annotations": {},
-          "labels": {},
-          "isPaused": false
-        },
-        {{ end }}
-        {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-        {
-          "uid": "ads8cswmly96oa",
-          "title": "Node Memory Usage",
-          "condition": "C",
-          "data": [
-            {
-              "refId": "A",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "PBFA97CFB590B2093",
-              "model": {
-                "editorMode": "code",
-                "expr": "(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
-                "instant": true,
-                "intervalMs": 1000,
-                "legendFormat": "__auto",
-                "maxDataPoints": 43200,
-                "range": false,
-                "refId": "A"
-              }
-            },
-            {
-              "refId": "C",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "__expr__",
-              "model": {
-                "conditions": [
-                  {
-                    "evaluator": { "params": [90], "type": "gt" },
-                    "operator": { "type": "and" },
-                    "query": { "params": ["C"] },
-                    "reducer": { "params": [], "type": "last" },
-                    "type": "query"
-                  }
-                ],
-                "datasource": { "type": "__expr__", "uid": "__expr__" },
-                "expression": "A",
-                "intervalMs": 1000,
-                "maxDataPoints": 43200,
-                "refId": "C",
-                "type": "threshold"
-              }
-            }
-          ],
-          "noDataState": "NoData",
-          "execErrState": "Error",
-          "for": "5m",
-          "annotations": {},
-          "labels": {},
-          "isPaused": false
-        }
-        {{ end }}
-      ]
-    }
-  ]
-}
-