fix backup missing alert & bump Not successful alert to 60m

.env.sample

@@ -1,24 +1,19 @@
 TYPE=monitoring-ng
 LETS_ENCRYPT_ENV=production
 COMPOSE_FILE=compose.yml
-DOMAIN=monitoring-ng.example.com
+DOMAIN=monitoring.example.com
-#TIMEOUT=120
+TIMEOUT=120
 ENABLE_BACKUPS=true
 
-SECRET_BASIC_AUTH_VERSION=v1
+## Enable this secret for Promtail / Prometheus
-# Enable this to send logs to a Loki server, adapt DOMAIN if server is
+# SECRET_BASIC_AUTH_VERSION=v1
-# remote
+#
-# LOKI_PUSH_URL=https://loki.$DOMAIN/loki/api/v1/push
+# Promtail (Gathering Logs)
-# Enable this on SystemD hosts to read logs
+# COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
-# JOURNALD=1
+# LOKI_PUSH_URL=https://loki.monitoring.example.org/loki/api/v1/push
-# Enable this on syslogd hosts and configure the syslogd to send logs to
-# Alloy on port 514/tcp
-# SYSLOG=1
-# COMPOSE_FILE="$COMPOSE_FILE:compose.syslog.yml"
 
-# Enable this to send metrics to a Prometheus server, adapt DOMAIN if
+## Expose node and cadvisor ports instead of traefik
-# server is remote
+# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
-# PROMETHEUS_REMOTE_WRITE_URL=https://prometheus.$DOMAIN/api/v1/write
 
 # Monitoring Server
 #
@@ -44,18 +39,19 @@ SECRET_BASIC_AUTH_VERSION=v1
 # LOKI_AWS_REGION=eu-west-1
 # LOKI_ACCESS_KEY_ID=bush-debrief-approval-robust-scraggly-molecule
 # LOKI_BUCKET_NAMES=loki
-# SECRET_LOKI_AWS_KEY_VERSION=v1
+# SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION=v1
 #
 ## Grafana
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
 # GF_SERVER_ROOT_URL=https://monitoring.example.com
-# SECRET_GF_ADMINPASSWD_VERSION=v1
+# SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
+## Seperate domain for Grafana
+#GRAFANA_DOMAIN=grafana.example.com
 #
 ## Single-Sign-On with OIDC
-# COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
 # OIDC_ENABLED=1
-# SECRET_GF_OIDC_SECRET_VERSION=v1
+# SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION=v1
 # OIDC_CLIENT_ID=grafana
 # OIDC_AUTH_URL="https://authentik.example.com/application/o/authorize/"
 # OIDC_API_URL="https://authentik.example.com/application/o/userinfo/"
@@ -66,18 +62,17 @@ SECRET_BASIC_AUTH_VERSION=v1
 # GF_INSTALL_PLUGINS=grafana-piechart-panel
 #
 ## grafana SMTP configuration (optional)
-# COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"
 # GF_SMTP_HOST=changeme
 # GF_SMTP_USER=changme
 # GF_SMTP_ENABLED=true
 # GF_SMTP_FROM_ADDRESS=grafana@example.com
 # GF_SMTP_SKIP_VERIFY=false
-# SECRET_GF_SMTP_PASSWD_VERSION=v1
+# SECRET_GRAFANA_SMTP_PASSWORD_VERSION=v1
 #
 
 ## Grafana Matrix Contact Point (optional)
 #COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
-#SECRET_MATRIX_TOKEN_VERSION=v1
+#SECRET_MATRIX_ACCESS_TOKEN_VERSION=v1
 #GF_MATRIX_USER_ID="<user-id>"
 #GF_MATRIX_ROOM_ID="<room-id>"
 #GF_MATRIX_HOMESERVER_URL="<homeserver-url>"
@@ -88,15 +83,3 @@ SECRET_BASIC_AUTH_VERSION=v1
 #ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED=true
 #ALERT_NODE_DISK_SPACE_ENABLED=true
 #ALERT_NODE_MEMORY_USAGE_ENABLED=true
-
-# Forgejo metrics
-# SECRET_FORGEJO_METRICS_TOKEN_VERSION=v1
-# FORGEJO_METRICS_HOSTNAME=
-# FORGEJO_INSECURE_SKIP_VERIFY=false
-# COMPOSE_FILE="$COMPOSE_FILE:compose.forgejo.yml"
-
-# Garage metrics
-# SECRET_GARAGE_METRICS_TOKEN_VERSION=v1
-# GARAGE_METRICS_HOSTNAME=
-# GARAGE_INSECURE_SKIP_VERIFY=false
-# COMPOSE_FILE="$COMPOSE_FILE:compose.garage.yml"

README.md

@@ -1,8 +1,8 @@
 # monitoring-ng
 
 Yet another monitoring stack ...
-This time its a all-in-one grafana/prometheus/loki/alloy stack.
+This time its a all-in-one grafana/prometheus/loki/node_exporter/cadvisor/promtail stack.
-It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-cloud/monitoring-lite) stack, but has everything in one recipe included now. So you can deploy monitoring instances to only gather metrics / logs (alloy) and also deploy instances with the full monitoring stack (grafana/prometheus/loki) with the same recipe and just different .env configuration.
+It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-cloud/monitoring-lite) stack, but has everything in one recipe included now. So you can deploy monitoring instances to only gather metrics / logs (node_exporter/cadvisor/promtail) and also deploy instances with the full monitoring stack (grafana/prometheus/loki) with the same recipe and just different .env configuration.
 
 
 <!-- metadata -->
@@ -47,6 +47,13 @@ Where gathering.org is the node you want to gather metrics from.
   - cadvisor.monitoring.gathering.org
   - node.monitoring.gathering.org
 
+### Expose node and cadvisor via ports instead of traefik
+
+In case you have no traefik running on the machine, you can expose the ports directly by uncommenting the following line:
+```
+# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
+```
+
 ## Setup Metrics Browser
 
 
@@ -138,7 +145,7 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
 
 2. Insert the matrix access token secret:
 ```
-abra app secret insert monitoring.marx.klasse-methode.it matrix_token v1
+abra app secret insert monitoring.marx.klasse-methode.it matrix_access_token v1
 ```
 
 3. Set required configurations:

abra.sh

@@ -1,122 +1,27 @@
 export ENTRYPOINT_VERSION=v1
-export GF_DATASOURCES_VERSION=v1
+export GRAFANA_DATASOURCES_YML_VERSION=v1
-export GF_DASHBOARDS_VERSION=v2
+export GRAFANA_DASHBOARDS_YML_VERSION=v2
-export GF_SWARM_DASH_VERSION=v2
+export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v2
-export GF_STACKS_DASH_VERSION=v2
+export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v2
-export GF_TRAEFIK_DASH_VERSION=v2
+export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v2
-export GF_BACKUP_DASH_VERSION=v1
+export GRAFANA_BACKUP_DASHBOARD_JSON_VERSION=v1
-export GF_CUSTOM_INI_VERSION=v4
+export GRAFANA_ALERTS_JSON_VERSION=v3
-export LOKI_YML_VERSION=v3
+export GRAFANA_CUSTOM_INI_VERSION=v4
+export PROMTAIL_YML_VERSION=v3
+export LOKI_YML_VERSION=v2
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
-export GRAFANA_ALERTS_NODE_VERSION=v1c
-export CONFIG_ALLOY_VERSION=v9
 
 # creates a default prometheus scrape config for a given node
 add_node(){
   name=$1
-  add_domain "$name" "metrics.traefik.$name"
+  add_domain "$name" "$name:8082"
   add_domain "$name" "node.monitoring.$name"
   add_domain "$name" "cadvisor.monitoring.$name"
   cat "/prometheus/scrape_configs/$name.yml"
 }
 
-# migrates secrets from old names to new names by reading values from the
-# running containers on the server and re-inserting them under the new names.
-# preview changes: abra app cmd --local <app> migrate_secret_names
-# execute changes: abra app cmd --local <app> migrate_secret_names execute
-migrate_secret_names() {
-  if ! command -v jq &> /dev/null; then
-    echo "jq is required on your local machine to migrate secret names"
-    echo "It could not be found in your PATH, please install jq to proceed."
-    echo "For example: On a debian/ubuntu system, run `apt install jq`"
-    exit 1
-  fi
-
-  # Hardcoded migration mappings: old_secret_name|new_secret_name
-  MIGRATIONS="
-grafana_admin_password|gf_adminpasswd
-grafana_smtp_password|gf_smtp_passwd
-grafana_oidc_client_secret|gf_oidc_secret
-matrix_access_token|matrix_token
-loki_aws_secret_access_key|loki_aws_key
-"
-
-  # Determine which server the app is deployed on
-  SERVER=$(abra app ls -m | jq -r --arg domain "$APP_NAME" '[.[].apps[] | select(.domain == $domain) | .server] | first' 2>/dev/null)
-
-  if [ -z "$SERVER" ]; then
-    echo "Error: could not determine server for app '$APP_NAME'"
-    exit 1
-  fi
-
-  # Build a lookup table of all secrets currently mounted in this stack.
-  # Each line: <secretID> <containerID> <secretName>
-  LOOKUP=$(ssh "$SERVER" "
-    docker stack services ${STACK_NAME} --format '{{.Name}}' | while read svc; do
-      CID=\$(docker ps --no-trunc -q --filter \"name=\${svc}\" | head -1)
-      docker service inspect \"\$svc\" --format '{{json .Spec.TaskTemplate.ContainerSpec.Secrets}}' | \
-        jq -r --arg cid \"\$CID\" '.[]? | .SecretID + \" \" + \$cid + \" \" + .SecretName'
-    done | sort -k3 -r
-  " 2>/dev/null)
-
-  echo "Secret migration plan for: $APP_NAME (server: $SERVER)"
-  echo ""
-  printf "  %-24s %-8s %s\n" "OLD NAME" "FOUND" "ACTION"
-  printf "  %-24s %-8s %s\n" "--------" "-----" "------"
-
-  # Check each old name against the lookup table and display the plan
-  ANY_FOUND=false
-  while IFS='|' read -r OLD_NAME NEW_NAME; do
-    [ -z "$OLD_NAME" ] && continue
-    MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
-    if [ -n "$MATCH" ]; then
-      printf "  %-24s %-8s %s\n" "$OLD_NAME" "yes" "recreate as '$NEW_NAME' version V1"
-      ANY_FOUND=true
-    else
-      printf "  %-24s %-8s %s\n" "$OLD_NAME" "no" "nothing (not found on server)"
-    fi
-  done <<< "$MIGRATIONS"
-
-  echo ""
-
-  if [ "$ANY_FOUND" = false ]; then
-    echo "No old secrets found on server. Nothing to migrate."
-    return 0
-  fi
-
-  if [ "$1" != "execute" ]; then
-    echo "To apply the above changes, run:"
-    echo "  abra app cmd --local $APP_NAME migrate_secret_names execute"
-    return 0
-  fi
-
-  # read each found secret from its container and re-insert with the new name
-  while IFS='|' read -r OLD_NAME NEW_NAME; do
-    [ -z "$OLD_NAME" ] && continue
-
-    MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
-    [ -z "$MATCH" ] && continue
-
-    SECRET_ID=$(echo "$MATCH" | awk '{print $1}')
-    CID=$(echo "$MATCH" | awk '{print $2}')
-    SECRET_VALUE=$(ssh "$SERVER" "cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null || sudo cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null")
-
-    if [ -z "$SECRET_VALUE" ]; then
-      echo "Error: could not read value for '$OLD_NAME', skipping"
-      continue
-    fi
-
-    echo "Migrating: '$OLD_NAME' -> '$NEW_NAME' (v1)"
-    printf '%s' "$SECRET_VALUE" | abra app secret insert -C "$APP_NAME" "$NEW_NAME" v1
-    
-  done <<< "$MIGRATIONS"
-
-  echo ""
-  echo "Done."
-}
-
 # adds a domain to a scrape config or creates a new one
 add_domain(){
   name=$1

alertmanager-matrix-config.yml.tmpl

@@ -12,7 +12,7 @@ http:
 matrix:
   homeserver-url: "{{ env "GF_MATRIX_HOMESERVER_URL" }}"
   user-id: "{{ env "GF_MATRIX_USER_ID" }}"
-  access-token: "{{ secret "matrix_token" }}"
+  access-token: "{{ secret "matrix_access_token" }}"
   room-mapping:
     matrixroom: "{{ env "GF_MATRIX_ROOM_ID" }}"
 

alerts/node.yml.tmpl

@@ -1,131 +0,0 @@
-apiVersion: 1
-
-# List of alert rule UIDs that should be deleted
-deleteRules:
-  {{ if ne (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-  - orgId: 1
-    uid: bds8bhxu97pxca
-  {{ end }}
-  {{ if ne (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-  - orgId: 1
-    uid: ads8cswmly96oa
-  {{ end }}
-
-groups:
-- orgId: 1
-  name: node
-  folder: node
-  interval: 5m
-  rules:
-    {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-    - uid: bds8bhxu97pxca
-      title: Node Disk Space
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: (node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"}) * 100
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - 10
-                    type: lt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 5m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Less than 10% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
-      labels:
-        "": ""
-      isPaused: false
-    {{ end }}
-    {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-    - uid: ads8cswmly96oa
-      title: Node Memory Usage
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - 85
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 5m
-      annotations:
-        summary: Memory usage is above 85% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
-      isPaused: false
-    {{ end }}

compose.expose-ports.yml

@@ -0,0 +1,13 @@
+---
+version: "3.8"
+
+services:
+  app:
+    ports:
+      - "9100:9100"
+    deploy:
+
+  cadvisor:
+    ports:
+      - "9101:8080"
+    deploy:

compose.forgejo.yml

@@ -1,10 +0,0 @@
----
-version: "3.8"
-services:
-  app:
-    secrets:
-      - forgejo_token
-secrets:
-  forgejo_token:
-    external: true
-    name: ${STACK_NAME}_forgejo_token_${SECRET_FORGEJO_METRICS_TOKEN_VERSION}

compose.garage.yml

@@ -1,10 +0,0 @@
----
-version: "3.8"
-services:
-  app:
-    secrets:
-      - garage_token
-secrets:
-  garage_token:
-    external: true
-    name: ${STACK_NAME}_garage_token_${SECRET_GARAGE_METRICS_TOKEN_VERSION}

compose.grafana-oidc.yml

@@ -1,17 +0,0 @@
-version: '3.8'
-
-services:
-  grafana:
-    secrets:
-      - gf_oidc_secret
-    environment:
-      - OIDC_API_URL
-      - OIDC_AUTH_URL
-      - OIDC_CLIENT_ID
-      - OIDC_ENABLED
-      - OIDC_TOKEN_URL
-
-secrets:
-  gf_oidc_secret:
-    external: true
-    name: ${STACK_NAME}_gf_oidc_secret_${SECRET_GF_OIDC_SECRET_VERSION}

compose.grafana-smtp.yml

@@ -1,18 +0,0 @@
-version: '3.8'
-
-services:
-  grafana:
-    secrets:
-      - gf_smtp_passwd
-    environment:
-      - GF_SMTP_HOST
-      - GF_SMTP_USER
-      - GF_SMTP_PASSWORD__FILE=/run/secrets/gf_smtp_passwd
-      - GF_SMTP_ENABLED
-      - GF_SMTP_FROM_ADDRESS
-      - GF_SMTP_SKIP_VERIFY
-
-secrets:
-  gf_smtp_passwd:
-    external: true
-    name: ${STACK_NAME}_gf_smtp_passwd_${SECRET_GF_SMTP_PASSWD_VERSION}

compose.grafana.yml

@@ -2,87 +2,104 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:12.4.3
+    image: grafana/grafana:10.4.14
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:
-      - gf_adminpasswd
+      - grafana_admin_password
+      - grafana_oidc_client_secret
+      - grafana_smtp_password
     configs:
-      - source: gf_custom_ini
+      - source: grafana_custom_ini
         target: /etc/grafana/grafana.ini
-      - source: gf_datasources
+      - source: grafana_datasources_yml
         target: /etc/grafana/provisioning/datasources/datasources.yml
-      - source: gf_dashboards
+      - source: grafana_dashboards_yml
         target: /etc/grafana/provisioning/dashboards/dashboards.yml
-      - source: gf_swarm_dash
+      - source: grafana_swarm_dashboard_json
         target: /var/lib/grafana/dashboards/docker-swarm-nodes.json
-      - source: gf_stacks_dash
+      - source: grafana_stacks_dashboard_json
         target: /var/lib/grafana/dashboards/docker-swarm-stacks.json
-      - source: gf_traefik_dash
+      - source: grafana_traefik_dashboard_json
         target: /var/lib/grafana/dashboards/traefik.json
-      - source: gf_backup_dash
+      - source: grafana_backup_dashboard_json
         target: /var/lib/grafana/dashboards/backup.json
-      - source: gf_alerts_node
+      - source: grafana_alerts_json
-        target: /etc/grafana/provisioning/alerting/node.yml
+        target: /var/lib/grafana/alerts/alerts.json
     networks:
       - proxy
       - internal
     environment:
       - GF_SERVER_ROOT_URL
-      - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/gf_adminpasswd
+      - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/grafana_admin_password
+      - GF_SMTP_HOST
+      - GF_SMTP_USER
+      - GF_SMTP_PASSWORD__FILE=/run/secrets/grafana_smtp_password
+      - GF_SMTP_ENABLED
+      - GF_SMTP_FROM_ADDRESS
+      - GF_SMTP_SKIP_VERIFY
       - GF_SECURITY_ALLOW_EMBEDDING
       - GF_INSTALL_PLUGINS
-      - ALERT_NODE_DISK_SPACE_ENABLED
+      - OIDC_API_URL
-      - ALERT_NODE_MEMORY_USAGE_ENABLED
+      - OIDC_AUTH_URL
+      - OIDC_CLIENT_ID
+      - OIDC_ENABLED
+      - OIDC_TOKEN_URL
     deploy:
       labels:
         - "traefik.enable=true"
-        - "traefik.swarm.network=proxy"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
-        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
     healthcheck:
-      test: "wget -q http://localhost:3000/healthz -O/dev/null"
+      test: "wget -q http://localhost:3000/ -O/dev/null"
       interval: 5s
       timeout: 10s
       retries: 3
       start_period: 10s
 
 configs:
-  gf_custom_ini:
+  grafana_custom_ini:
     template_driver: golang
-    name: ${STACK_NAME}_gf_custom_ini_${GF_CUSTOM_INI_VERSION}
+    name: ${STACK_NAME}_grafana_custom_ini_${GRAFANA_CUSTOM_INI_VERSION}
     file: grafana_custom.ini
-  gf_datasources:
+  grafana_datasources_yml:
-    name: ${STACK_NAME}_gf_datasources_${GF_DATASOURCES_VERSION}
+    name: ${STACK_NAME}_g_datasources_yml_${GRAFANA_DATASOURCES_YML_VERSION}
     file: grafana-datasources.yml
-  gf_dashboards:
+  grafana_dashboards_yml:
-    name: ${STACK_NAME}_gf_dashboards_${GF_DASHBOARDS_VERSION}
+    name: ${STACK_NAME}_g_dashboards_yml_${GRAFANA_DASHBOARDS_YML_VERSION}
     file: grafana-dashboards.yml
-  gf_swarm_dash:
+  grafana_swarm_dashboard_json:
-    name: ${STACK_NAME}_gf_swarm_dash_${GF_SWARM_DASH_VERSION}
+    name: ${STACK_NAME}_g_swarm_dashboard_json_${GRAFANA_SWARM_DASHBOARD_JSON_VERSION}
     file: grafana-swarm-dashboard.json
-  gf_stacks_dash:
+  grafana_stacks_dashboard_json:
-    name: ${STACK_NAME}_gf_stacks_dash_${GF_STACKS_DASH_VERSION}
+    name: ${STACK_NAME}_g_stacks_dashboard_json_${GRAFANA_STACKS_DASHBOARD_JSON_VERSION}
     file: grafana-stacks-dashboard.json
-  gf_traefik_dash:
+  grafana_traefik_dashboard_json:
-    name: ${STACK_NAME}_gf_traefik_dash_${GF_TRAEFIK_DASH_VERSION}
+    name: ${STACK_NAME}_g_traefik_dashboard_json_${GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION}
     file: grafana-traefik-dashboard.json
-  gf_backup_dash:
+  grafana_backup_dashboard_json:
-    name: ${STACK_NAME}_gf_backup_dash_${GF_BACKUP_DASH_VERSION}
+    name: ${STACK_NAME}_g_backup_dashboard_json_${GRAFANA_BACKUP_DASHBOARD_JSON_VERSION}
     file: grafana-backup-dashboard.json
-  gf_alerts_node:
+  grafana_alerts_json:
     template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_node_${GRAFANA_ALERTS_NODE_VERSION}
+    name: ${STACK_NAME}_g_alerts_json_${GRAFANA_ALERTS_JSON_VERSION}
-    file: alerts/node.yml.tmpl
+    file: grafana-alerts.json.tmpl
 
 volumes:
   grafana-data:
 
 
 secrets:
-  gf_adminpasswd:
+  grafana_admin_password:
     external: true
-    name: ${STACK_NAME}_gf_adminpasswd_${SECRET_GF_ADMINPASSWD_VERSION}
+    name: ${STACK_NAME}_grafana_admin_password_${SECRET_GRAFANA_ADMIN_PASSWORD_VERSION}
+  grafana_oidc_client_secret:
+    external: true
+    name: ${STACK_NAME}_grafana_oidc_client_secret_${SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION}
+  grafana_smtp_password:
+    external: true
+    name: ${STACK_NAME}_grafana_smtp_password_${SECRET_GRAFANA_SMTP_PASSWORD_VERSION}

compose.loki.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   loki:
-    image: grafana/loki:3.7.2
+    image: grafana/loki:2.9.11
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - proxy
@@ -12,7 +12,7 @@ services:
     volumes:
       - loki-data:/loki
     # secrets:
-    #   - loki_aws_key
+    #   - loki_aws_secret_access_key
     environment:
       - LOKI_ACCESS_KEY_ID
       - LOKI_AWS_ENDPOINT
@@ -27,7 +27,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
-        - "traefik.swarm.network=proxy"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-loki.loadbalancer.server.port=3100"
         - "traefik.http.routers.${STACK_NAME}-loki.rule=Host(`loki.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-loki.entrypoints=web-secure"
@@ -47,6 +47,6 @@ volumes:
   loki-data:
 
 # secrets:
-#   loki_aws_key:
+#   loki_aws_secret_access_key:
 #     external: true
-#     name: ${STACK_NAME}_loki_aws_key_${SECRET_LOKI_AWS_KEY_VERSION}
+#     name: ${STACK_NAME}_loki_aws_secret_access_key_${SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION}

compose.matrix-alertmanager-receiver.yml

@@ -2,9 +2,9 @@ version: '3.8'
 
 services:
   matrix-alertmanager-receiver:
-    image: metio/matrix-alertmanager-receiver:2026.2.25
+    image: metio/matrix-alertmanager-receiver:2025.2.9
     secrets:
-      - matrix_token
+      - matrix_access_token
     configs:
       - source: matrix-alertmanager-receiver-config
         target: /etc/matrix-alertmanager-receiver/config.yml
@@ -23,6 +23,6 @@ configs:
     file: alertmanager-matrix-config.yml.tmpl
 
 secrets:
-  matrix_token:
+  matrix_access_token:
     external: true
-    name: ${STACK_NAME}_matrix_token_${SECRET_MATRIX_TOKEN_VERSION}
+    name: ${STACK_NAME}_matrix_access_token_${SECRET_MATRIX_ACCESS_TOKEN_VERSION}

compose.prometheus.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v3.12.0
+    image: prom/prometheus:v2.55.1
     secrets:
       - basic_auth
     volumes:
@@ -16,8 +16,6 @@ services:
       - "--web.console.libraries=/usr/share/prometheus/console_libraries"
       - "--web.console.templates=/usr/share/prometheus/consoles"
       - "--storage.tsdb.retention.time=${PROMETHEUS_RETENTION_TIME}"
-      - "--enable-feature=remote-write-receiver"
-      - "--web.enable-remote-write-receiver"
     networks:
       - proxy
       - internal
@@ -26,13 +24,12 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
-        - "traefik.swarm.network=proxy"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-prometheus.loadbalancer.server.port=9090"
         - "traefik.http.routers.${STACK_NAME}-prometheus.rule=Host(`prometheus.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
 
 configs:
   prometheus_yml:

compose.promtail.yml

@@ -0,0 +1,30 @@
+version: "3.8"
+
+services:
+  promtail:
+    image: grafana/promtail:2.9.11
+    volumes:
+      - /var/log:/var/log:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    command: -config.file=/etc/promtail/config.yml
+    configs:
+      - source: promtail_yml
+        target: /etc/promtail/config.yml
+    networks:
+      - internal
+    secrets:
+      - basic_auth
+    environment:
+      - DOMAIN
+      - LOKI_PUSH_URL
+
+configs:
+  promtail_yml:
+    name: ${STACK_NAME}_promtail_yml_${PROMTAIL_YML_VERSION}
+    file: promtail.yml.tmpl
+    template_driver: golang
+
+secrets:
+  basic_auth:
+    external: true
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.pushgateway.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   pushgateway:
-    image: prom/pushgateway:v1.11.2
+    image: prom/pushgateway:v1.10.0
     command:
       - '--web.listen-address=:9191'
       - '--push.disable-consistency-check'
@@ -17,7 +17,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
-        - "traefik.swarm.network=proxy"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-pushgateway.loadbalancer.server.port=9191"
         - "traefik.http.routers.${STACK_NAME}-pushgateway.rule=Host(`pushgateway.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-pushgateway.entrypoints=web-secure"

compose.syslog.yml

@@ -1,6 +0,0 @@
----
-version: "3.8"
-services:
-  app:
-    ports:
-    - "514:514"

compose.yml

@@ -3,46 +3,89 @@ version: "3.8"
 
 services:
   app:
-    image: grafana/alloy:v1.16.1
+    image: prom/node-exporter:v1.8.1
-    hostname: "${DOMAIN}"
+    user: root
-    configs:
+    environment:
-      - source: config_alloy
+      - NODE_ID={{.Node.ID}}
-        target: /etc/alloy/config.alloy
     volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
       - /:/rootfs:ro
-      - /var/run:/var/run:rw
+      - /etc/hostname:/etc/nodename:ro
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /sys:/sys:ro
-      - /var/lib/docker:/var/lib/docker:ro
-      - /dev:/dev:ro
-      - alloy-data:/var/lib/alloy/data
     command:
-      - "run"
+      - "--path.sysfs=/host/sys"
-      - "--storage.path=/var/lib/alloy/data"
+      - "--path.procfs=/host/proc"
-      - "/etc/alloy/config.alloy"
+      - "--path.rootfs=/rootfs"
+      - "--collector.textfile.directory=/etc/node-exporter/"
+      - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)"
+      - "--no-collector.ipvs"
+    configs:
+      - source: entrypoint
+        target: /entrypoint.sh
     networks:
       - internal
-    secrets:
+      - proxy
-      - basic_auth
+    entrypoint: [ "/bin/sh", "-e", "/entrypoint.sh" ]
     deploy:
       restart_policy:
         condition: on-failure
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
-        - "traefik.enable=false"
+        - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
+        - "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
+        - "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-node.tls=true"
+        - "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
         - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
+        - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.49.2
+    command:
+      - "-logtostderr"
+      - "--enable_metrics=cpu,cpuLoad,disk,diskIO,process,memory,network"
+      # all possible metrics: advtcp,app,cpu,cpuLoad,cpu_topology,cpuset,disk,diskIO,hugetlb,memory,memory_numa,network,oom_event,percpu,perf_event,process,referenced_memory,resctrl,sched,tcp,udp.
+      - "--housekeeping_interval=120s"
+      - "--docker_only=true"
+    volumes:
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+      - /sys:/sys:ro
+      - /var/run:/var/run:ro
+      - /:/rootfs:ro
+    networks:
+      - internal
+      - proxy
+    deploy:
+      restart_policy:
+        condition: on-failure
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
+        - "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.tls=true"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-cadvisor.middlewares=basicauth@file"
+    healthcheck:
+      test: wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1
+      interval: 15s
+      timeout: 15s
+      retries: 5
+      start_period: 30s
+
 configs:
-  config_alloy:
+  entrypoint:
-    template_driver: golang
+    name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
-    name: ${STACK_NAME}_config_alloy_${CONFIG_ALLOY_VERSION}
+    file: node-exporter-entrypoint.sh
-    file: config.alloy.tmpl
+
+
+
 networks:
   proxy:
     external: true
   internal:
-volumes:
-  alloy-data:
-secrets:
-  basic_auth:
-    external: true
-    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

config.alloy.tmpl

@@ -1,123 +0,0 @@
-logging {
-  level = "info"
-  format = "logfmt"
-}
-
-discovery.docker "linux" {
-  host = "unix:///var/run/docker.sock"
-}
-
-{{ if ne (env "PROMETHEUS_REMOTE_WRITE_URL") "" }}
-prometheus.exporter.cadvisor "docker" {
-}
-
-prometheus.exporter.unix "default" {
-  include_exporter_metrics = true
-  rootfs_path = "/rootfs"
-}
-
-prometheus.scrape "default" {
-  targets = array.concat(
-    [{
-      job         = "alloy",
-      __address__ = "127.0.0.1:12345",
-    }],
-    prometheus.exporter.unix.default.targets,
-    prometheus.exporter.cadvisor.docker.targets,
-  )
-
-  forward_to = [prometheus.remote_write.prometheus.receiver]
-}
-
-{{ if ne (env "FORGEJO_METRICS_HOSTNAME") "" }}
-prometheus.scrape "forgejo" {
-  bearer_token = "{{ secret "forgejo_token" }}"
-  job_name = "forgejo"
-  scheme = "https"
-
-  targets = [{ __address__ = "{{ env "FORGEJO_METRICS_HOSTNAME" }}" }]
-  forward_to = [prometheus.remote_write.prometheus.receiver]
-
-  tls_config {
-    insecure_skip_verify = {{ env "FORGEJO_INSECURE_SKIP_VERIFY" }}
-    server_name = "{{ env "FORGEJO_METRICS_HOSTNAME" }}"
-  }
-}
-{{ end }}
-
-{{ if ne (env "GARAGE_METRICS_HOSTNAME") "" }}
-prometheus.scrape "garage" {
-  bearer_token = "{{ secret "garage_token" }}"
-  job_name = "garage"
-  scheme = "https"
-
-  targets = [{ __address__ = "{{ env "GARAGE_METRICS_HOSTNAME" }}:443" }]
-  forward_to = [prometheus.remote_write.prometheus.receiver]
-
-  tls_config {
-    insecure_skip_verify = {{ env "GARAGE_INSECURE_SKIP_VERIFY" }}
-  }
-}
-{{ end }}
-
-prometheus.remote_write "prometheus" {
-  endpoint {
-    url = "{{ env "PROMETHEUS_REMOTE_WRITE_URL" }}"
-
-    basic_auth {
-      username = "admin"
-      password = "{{ secret "basic_auth" }}"
-    }
-  }
-}
-{{ end }}
-
-{{ if ne (env "LOKI_PUSH_URL") "" }}
-loki.source.docker "docker" {
-  host       = "unix:///var/run/docker.sock"
-  targets    = discovery.docker.linux.targets
-  labels     = {"app" = "docker"}
-  forward_to = [loki.write.loki.receiver]
-}
-
-{{ if eq (env "JOURNALD") "1" }}
-loki.source.journal "journal" {
-  path = "/var/log/journal"
-  labels = { job = "{{ env "DOMAIN" }}" }
-  forward_to = [loki.write.loki.receiver]
-}
-{{ end }}
-
-{{ if eq (env "SYSLOG") "1" }}
-loki.relabel "syslog" {
-  rule {
-    action = "labelmap"
-    regex = "__syslog_(.+)"
-  }
-
-  forward_to = []
-}
-
-loki.source.syslog "syslog" {
-  listener {
-    address = "[::1]:514"
-    label_structured_data = true
-    labels = { component = "loki.source.syslog" }
-  }
-
-  relabel_rules = loki.relabel.syslog.rules
-  forward_to = [loki.write.loki.receiver]
-}
-{{ end }}
-
-loki.write "loki" {
-  endpoint {
-    url = "{{ env "LOKI_PUSH_URL" }}"
-
-    basic_auth {
-      username = "admin"
-      password = "{{ secret "basic_auth" }}"
-    }
-  }
-}
-{{ end }}

grafana-alerts.json.tmpl

@@ -68,50 +68,76 @@
           "condition": "B",
           "data": [
             {
-              "refId": "A",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "PBFA97CFB590B2093",
-              "model": {
-                "disableTextWrap": false,
-                "editorMode": "builder",
-                "expr": "rate(backup[24h])",
-                "fullMetaSearch": false,
-                "includeNullMetadata": true,
-                "instant": true,
-                "intervalMs": 1000,
-                "legendFormat": "__auto",
-                "maxDataPoints": 43200,
-                "range": false,
                 "refId": "A",
-                "useBackend": false
+                "relativeTimeRange": {
-              }
+                    "from": 28800,
+                    "to": 0
+                },
+                "datasourceUid": "PBFA97CFB590B2093",
+                "model": {
+                    "disableTextWrap": false,
+                    "editorMode": "code",
+                    "expr": "rate(backup[25h]) == 0",
+                    "fullMetaSearch": false,
+                    "includeNullMetadata": true,
+                    "instant": true,
+                    "intervalMs": 1000,
+                    "legendFormat": "__auto",
+                    "maxDataPoints": 43200,
+                    "range": false,
+                    "refId": "A",
+                    "useBackend": false
+                }
             },
             {
-              "refId": "B",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "__expr__",
-              "model": {
-                "conditions": [
-                  {
-                    "evaluator": { "params": [0, 0], "type": "within_range" },
-                    "operator": { "type": "and" },
-                    "query": { "params": ["C"] },
-                    "reducer": { "params": [], "type": "last" },
-                    "type": "query"
-                  }
-                ],
-                "datasource": { "type": "__expr__", "uid": "__expr__" },
-                "expression": "A",
-                "intervalMs": 1000,
-                "maxDataPoints": 43200,
                 "refId": "B",
-                "type": "threshold"
+                "relativeTimeRange": {
-              }
+                    "from": 28800,
+                    "to": 0
+                },
+                "datasourceUid": "__expr__",
+                "model": {
+                    "conditions": [
+                        {
+                            "evaluator": {
+                                "params": [
+                                    -1,
+                                    0
+                                ],
+                                "type": "gt"
+                            },
+                            "operator": {
+                                "type": "and"
+                            },
+                            "query": {
+                                "params": []
+                            },
+                            "reducer": {
+                                "params": [],
+                                "type": "avg"
+                            },
+                            "type": "query"
+                        }
+                    ],
+                    "datasource": {
+                        "name": "Expression",
+                        "type": "__expr__",
+                        "uid": "__expr__"
+                    },
+                    "expression": "A",
+                    "hide": false,
+                    "intervalMs": 1000,
+                    "maxDataPoints": 43200,
+                    "refId": "B",
+                    "type": "threshold"
+                }
             }
           ],
           "noDataState": "NoData",
           "execErrState": "Error",
           "for": "5m",
+          "annotations": {},
+          "labels": {},
           "isPaused": false
         },
         {{ end }}
@@ -188,9 +214,9 @@
             ],
             "noDataState": "NoData",
             "execErrState": "Error",
-            "for": "20m",
+            "for": "60m",
             "annotations": {
-                "summary": "Backup did not finish within 20 minutes"
+                "summary": "Backup did not finish within 60 minutes"
             },
             "labels": {},
             "isPaused": false
@@ -216,7 +242,7 @@
               "datasourceUid": "PBFA97CFB590B2093",
               "model": {
                 "editorMode": "code",
-                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
+                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100",
                 "instant": true,
                 "intervalMs": 1000,
                 "legendFormat": "__auto",

grafana-swarm-dashboard.json

@@ -93,6 +93,7 @@
       },
       "hideTimeOverride": true,
       "id": 2,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -106,12 +107,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -173,6 +172,7 @@
         "y": 0
       },
       "id": 1,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -186,12 +186,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -253,6 +251,7 @@
       },
       "hideTimeOverride": true,
       "id": 4,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -266,12 +265,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -338,10 +335,9 @@
         "y": 0
       },
       "id": 8,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -352,10 +348,9 @@
         },
         "showThresholdLabels": false,
         "showThresholdMarkers": true,
-        "sizing": "auto",
         "text": {}
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -410,12 +405,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null",
       "options": {
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -511,6 +507,7 @@
       },
       "hideTimeOverride": true,
       "id": 3,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -524,12 +521,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -590,6 +585,7 @@
       },
       "hideTimeOverride": true,
       "id": 9,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -603,12 +599,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -677,10 +671,9 @@
       },
       "hideTimeOverride": true,
       "id": 11,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -691,10 +684,9 @@
         },
         "showThresholdLabels": false,
         "showThresholdMarkers": true,
-        "sizing": "auto",
         "text": {}
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -721,39 +713,7 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "left",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
           },
           "mappings": [
             {
@@ -787,42 +747,33 @@
           },
           "unit": "percent"
         },
-        "overrides": [
+        "overrides": []
-          {
-            "matcher": {
-              "id": "byType",
-              "options": "time"
-            },
-            "properties": [
-              {
-                "id": "custom.axisPlacement",
-                "value": "hidden"
-              }
-            ]
-          }
-        ]
       },
       "gridPos": {
-        "h": 6,
+        "h": 4,
-        "w": 6,
+        "w": 2.6666666666666665,
         "x": 0,
         "y": 8
       },
       "id": 10,
+      "links": [],
       "maxDataPoints": 100,
+      "maxPerRow": 12,
       "options": {
-        "legend": {
+        "colorMode": "value",
-          "calcs": [],
+        "graphMode": "area",
-          "displayMode": "list",
+        "justifyMode": "auto",
-          "placement": "bottom",
+        "orientation": "horizontal",
-          "showLegend": true
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
         },
-        "tooltip": {
+        "textMode": "auto"
-          "mode": "single",
-          "sort": "none"
-        }
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "repeat": "node_id",
       "repeatDirection": "h",
       "targets": [
@@ -831,20 +782,18 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
-          "editorMode": "code",
           "exemplar": true,
-          "expr": "node_filesystem_free_bytes{fstype=\"ext4\"}  / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
+          "expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
-          "legendFormat": "{{mountpoint}}",
+          "legendFormat": "",
-          "range": true,
           "refId": "A",
           "step": 20
         }
       ],
       "title": "Available Disk Space $node_id",
-      "type": "timeseries"
+      "type": "stat"
     },
     {
       "aliasColors": {},
@@ -862,7 +811,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 14
+        "y": 12
       },
       "hiddenSeries": false,
       "id": 14,
@@ -881,12 +830,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -950,7 +900,6 @@
             "mode": "palette-classic"
           },
           "custom": {
-            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -964,7 +913,6 @@
               "tooltip": false,
               "viz": false
             },
-            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1013,11 +961,12 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 6,
+        "w": 2.6666666666666665,
         "x": 0,
-        "y": 21
+        "y": 19
       },
       "id": 15,
+      "links": [],
       "maxPerRow": 12,
       "options": {
         "legend": {
@@ -1125,7 +1074,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 28
+        "y": 26
       },
       "hiddenSeries": false,
       "id": 16,
@@ -1142,12 +1091,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1228,7 +1178,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 35
+        "y": 33
       },
       "hiddenSeries": false,
       "id": 18,
@@ -1245,6 +1195,7 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
@@ -1330,7 +1281,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 35
+        "y": 33
       },
       "hiddenSeries": false,
       "id": 19,
@@ -1349,6 +1300,7 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
@@ -1424,7 +1376,7 @@
         "h": 7,
         "w": 18,
         "x": 0,
-        "y": 42
+        "y": 40
       },
       "hiddenSeries": false,
       "id": 12,
@@ -1445,6 +1397,7 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null",
       "options": {
         "alertThreshold": true
@@ -1546,9 +1499,10 @@
         "h": 7,
         "w": 6,
         "x": 18,
-        "y": 42
+        "y": 40
       },
       "id": 7,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "none",
@@ -1646,9 +1600,10 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 49
+        "y": 47
       },
       "id": 17,
+      "links": [],
       "options": {
         "legend": {
           "calcs": [],
@@ -1703,7 +1658,7 @@
         "h": 9,
         "w": 24,
         "x": 0,
-        "y": 56
+        "y": 54
       },
       "id": 30,
       "options": {
@@ -1733,7 +1688,8 @@
     }
   ],
   "refresh": "",
-  "schemaVersion": 39,
+  "schemaVersion": 38,
+  "style": "dark",
   "tags": [
     "swarmprom",
     "prometheus",
@@ -1880,6 +1836,6 @@
   "timezone": "",
   "title": "Docker Swarm Nodes",
   "uid": "BPlb-Sgik",
-  "version": 7,
+  "version": 24,
   "weekStart": ""
-}
+}

grafana_custom.ini

@@ -21,7 +21,7 @@ tls_skip_verify_insecure = false
 allow_sign_up = true
 auto_login = true
 client_id = {{ env "OIDC_CLIENT_ID" }}
-client_secret = {{ secret "gf_oidc_secret" }}
+client_secret = {{ secret "grafana_oidc_client_secret" }}
 auth_url = {{ env "OIDC_AUTH_URL" }}
 token_url = {{ env "OIDC_TOKEN_URL" }}
 api_url = {{ env "OIDC_API_URL" }}

loki.yml.tmpl

@@ -34,6 +34,7 @@ ingester:
   max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
   chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
   chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
   wal:
     dir: "/tmp/wal"
 
@@ -52,7 +53,7 @@ schema_config:
     - from: 2020-10-24
       store: boltdb-shipper
       object_store: filesystem
-      schema: v13
+      schema: v11
       index:
         prefix: index_
         period: 24h
@@ -62,6 +63,7 @@ storage_config:
     active_index_directory: /loki/boltdb-shipper-active
     cache_location: /loki/boltdb-shipper-cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
   filesystem:
     directory: /loki/chunks
 {{ end }}
@@ -70,6 +72,7 @@ schema_config:
   configs:
   - from: 2020-11-25
     store: boltdb-shipper
+    object_store: aws
     schema: v11
     index:
       prefix: index_
@@ -86,7 +89,7 @@ storage_config:
    endpoint: {{ env "LOKI_AWS_ENDPOINT" }}
    region: {{ env "LOKI_AWS_REGION" }}
    access_key_id: {{ env "LOKI_ACCESS_KEY_ID" }}
-   secret_access_key: {{ secret "loki_aws_key" }}
+   secret_access_key: {{ secret "loki_aws_secret_access_key" }}
    bucketnames: {{ env "LOKI_BUCKET_NAMES" }}
    insecure: false
    sse_encryption: false
@@ -100,24 +103,19 @@ storage_config:
 
 compactor:
   working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
   compaction_interval: 10m
   retention_enabled: true
   retention_delete_delay: 2h
   retention_delete_worker_count: 150
-{{ if eq (env "LOKI_STORAGE_FILESYSTEM") "1" }}
-  delete_request_store: filesystem
-{{ end }}
-{{ if eq (env "LOKI_STORAGE_S3") "1" }}
-  delete_request_store: aws
-{{ end }}
 
 limits_config:
+  enforce_metric_name: false
   reject_old_samples: true
   reject_old_samples_max_age: 168h
   retention_period: {{ env "LOKI_RETENTION_PERIOD" }}
   split_queries_by_interval: 24h
   max_query_parallelism: 100
-  allow_structured_metadata: false
 
 query_scheduler:
   max_outstanding_requests_per_tenant: 4096
@@ -125,6 +123,9 @@ query_scheduler:
 frontend:
   max_outstanding_per_tenant: 4096
 
+chunk_store_config:
+  max_look_back_period: 0s
+
 table_manager:
   retention_deletes_enabled: false
   retention_period: 0s

node-exporter-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+NODE_NAME=$(cat /etc/nodename)
+
+mkdir -p /etc/node-exporter
+
+echo "node_meta{node_id=\"$NODE_ID\", container_label_com_docker_swarm_node_id=\"$NODE_ID\", node_name=\"$NODE_NAME\"} 1" > /etc/node-exporter/node-meta.prom
+
+set -- /bin/node_exporter "$@"
+
+exec "$@"

promtail.yml.tmpl

@@ -0,0 +1,37 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: {{ env "LOKI_PUSH_URL" }}
+    basic_auth:
+      username: admin
+      password: {{ secret "basic_auth" }}
+    external_labels:
+      hostname: {{ env "DOMAIN" }}
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
+
+- job_name: "docker"
+  docker_sd_configs:
+    - host: "unix:///var/run/docker.sock"
+      refresh_interval: "10s"
+  relabel_configs:
+    - source_labels: ['__meta_docker_container_name']
+      target_label: "container_name"
+    - source_labels: ['__meta_docker_container_id']
+      target_label: "container_id"
+    - source_labels: ['__meta_docker_container_label_com_docker_stack_namespace']
+      target_label: "stack_namespace"
+    - source_labels: ['__meta_docker_container_label_com_docker_swarm_service_name']
+      target_label: "service_name"

release/next

@@ -1,12 +0,0 @@
-1. OIDC was moved into a seperate compose file. If you have oidc configured you need to add the following line to you .env file:
-
-COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
-
-2. SMTP was moved into a seperate compose file. If you have smtp configured you need to add the following line to you .env file:
-
-COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"
-
-3. The scrape-config.example.yml file and add_node() command were updated to use a secure endpoint for the traefik metrics instead of http. This requires an updated Traefik recipe that publishes the metrics on https.
-
-4. Secret and config names were shortened to max 14 characters to prevent going over Docker's 64 character limit when STACK_NAME and VERSION are added to it.
-When upgrading, you need to reinsert the secrets with their shorter names. Run `abra app secret list <domain>` to see which secrets aren't created on the server (because their name was shortened) and run `abra app secret insert <domain> <secret_name> v1 <value>` to reinsert them with the shorter name. Or you can use the migrate_secret_names function in abra.sh to reinsert all existing secrets with their shorter name automatically: `abra app cmd --local <domain> migrate_secret_names`

scrape-config.example.yml

@@ -1,4 +1,4 @@
 - targets:
-  - 'metrics.traefik.example.org'
+  - 'example.org:8082'
   - 'node.monitoring.example.org'
   - 'cadvisor.monitoring.example.org'