docs: Update deploy docs and adds prometheus basic auth back

2025-11-05 15:45:17 +01:00
13 changed files with 107 additions and 327 deletions
--- a/.env.sample
+++ b/.env.sample
@ -1,8 +1,8 @@
 TYPE=monitoring-ng
 LETS_ENCRYPT_ENV=production
 COMPOSE_FILE=compose.yml
-DOMAIN=monitoring-ng.example.com
-#TIMEOUT=120
+DOMAIN=monitoring.example.com
+TIMEOUT=120
 ENABLE_BACKUPS=true

 ## Enable this secret for Promtail / Prometheus
@ -50,7 +50,6 @@ ENABLE_BACKUPS=true
 #GRAFANA_DOMAIN=grafana.example.com
 #
 ## Single-Sign-On with OIDC
-# COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
 # OIDC_ENABLED=1
 # SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION=v1
 # OIDC_CLIENT_ID=grafana
@ -63,7 +62,6 @@ ENABLE_BACKUPS=true
 # GF_INSTALL_PLUGINS=grafana-piechart-panel
 #
 ## grafana SMTP configuration (optional)
-# COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"
 # GF_SMTP_HOST=changeme
 # GF_SMTP_USER=changme
 # GF_SMTP_ENABLED=true
--- a/README.md
+++ b/README.md
@ -18,32 +18,18 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou

 <!-- endmetadata -->

-## Setup a Metrics Gathering
+## Setup Metrics Gathering

 Where gathering.org is the node you want to gather metrics from.

 1. Configure DNS
-  - monitoring.gathering.org
  - cadvisor.monitoring.gathering.org
  - node.monitoring.gathering.org
-1. Configure Traefik to use BasicAuth
-  * `abra app config traefik.gathering.org`
-  uncomment
-  ```
-  # BASIC_AUTH
-  COMPOSE_FILE="$COMPOSE_FILE:compose.basicauth.yml"
-  BASIC_AUTH=1
-  SECRET_USERSFILE_VERSION=v1
-  ```
-  - Generate userslist with httpasswd hashed password
-    `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
-    make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
-  - `abra app deploy -f traefik`
-1. `abra app new monitoring-ng`
-1. `abra app config monitoring.gathering.org`
-   for gathering only the main `compose.yml` is needed, nothing more.
-1. `abra app deploy monitoring.gathering.org`
-1. check that endpoints are up and basic-auth works
+2. [Configure Traefik to use BasicAuth](https://git.coopcloud.tech/coop-cloud/traefik#configuring-wildcard-ssl-using-dns)
+3. `abra app new monitoring-ng`
+4. `abra app config monitoring.gathering.org` (for gathering only the main `compose.yml` is needed, nothing more.)
+5. `abra app deploy monitoring.gathering.org`
+6. check that endpoints are up and basic-auth works
  - cadvisor.monitoring.gathering.org
  - node.monitoring.gathering.org

@ -56,16 +42,13 @@ In case you have no traefik running on the machine, you can expose the ports dir

 ## Setup Metrics Browser

+This builds upon [Setup Metrics Gathering](#setup-metrics-grathering) so make sure you did that first.

 1. Configure DNS
   - monitoring.example.org
-   - prometheus.monitoring.example.org
-   - loki.monitoring.example.org
 2. Setup monitoring stack
-   - `abra app new monitoring-ng`
-   - `abra app config monitoring.example.org`
-      Uncomment all the stuff
-   - `abra app secret insert monitoring.example.org basic_auth v1 <secret>`
+   - `abra app config monitoring.example.org` Uncomment prometheus, loki and grafana
+   - `abra app secret insert monitoring.example.org basic_auth v1 <password>`
      this needs the plaintext traefik basic-auth secret, not the hashed one!
   - `abra app secret ls monitoring.example.org`
   - `abra app deploy monitoring.example.org`
--- a/abra.sh
+++ b/abra.sh
@ -5,18 +5,18 @@ export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v2
 export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v2
 export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v2
 export GRAFANA_BACKUP_DASHBOARD_JSON_VERSION=v1
+export GRAFANA_ALERTS_JSON_VERSION=v3
 export GRAFANA_CUSTOM_INI_VERSION=v4
 export PROMTAIL_YML_VERSION=v3
 export LOKI_YML_VERSION=v2
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
-export GRAFANA_ALERTS_NODE_VERSION=v1c

 # creates a default prometheus scrape config for a given node
 add_node(){
  name=$1
-  add_domain "$name" "metrics.traefik.$name"
+  add_domain "$name" "$name:8082"
  add_domain "$name" "node.monitoring.$name"
  add_domain "$name" "cadvisor.monitoring.$name"
  cat "/prometheus/scrape_configs/$name.yml"
--- a/alerts/node.yml.tmpl
+++ b/alerts/node.yml.tmpl
@ -1,131 +0,0 @@
-apiVersion: 1
-
-# List of alert rule UIDs that should be deleted
-deleteRules:
-  {{ if ne (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-  - orgId: 1
-    uid: bds8bhxu97pxca
-  {{ end }}
-  {{ if ne (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-  - orgId: 1
-    uid: ads8cswmly96oa
-  {{ end }}
-
-groups:
- orgId: 1
-  name: node
-  folder: node
-  interval: 5m
-  rules:
-    {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
-    - uid: bds8bhxu97pxca
-      title: Node Disk Space
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: (node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"}) * 100
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - 10
-                    type: lt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 5m
-      annotations:
-        description: ""
-        runbook_url: ""
-        summary: Less than 10% disk space left on {{`{{ $labels.instance }}`}} ({{`{{ (index $values "A").Value }}`}}% left)
-      labels:
-        "": ""
-      isPaused: false
-    {{ end }}
-    {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
-    - uid: ads8cswmly96oa
-      title: Node Memory Usage
-      condition: C
-      data:
-        - refId: A
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: PBFA97CFB590B2093
-          model:
-            editorMode: code
-            expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100
-            instant: true
-            intervalMs: 1000
-            legendFormat: __auto
-            maxDataPoints: 43200
-            range: false
-            refId: A
-        - refId: C
-          relativeTimeRange:
-            from: 600
-            to: 0
-          datasourceUid: __expr__
-          model:
-            conditions:
-                - evaluator:
-                    params:
-                        - 85
-                    type: gt
-                  operator:
-                    type: and
-                  query:
-                    params:
-                        - C
-                  reducer:
-                    params: []
-                    type: last
-                  type: query
-            datasource:
-                type: __expr__
-                uid: __expr__
-            expression: A
-            intervalMs: 1000
-            maxDataPoints: 43200
-            refId: C
-            type: threshold
-      noDataState: NoData
-      execErrState: Error
-      for: 5m
-      annotations:
-        summary: Memory usage is above 85% on {{`{{ $labels.instance }}`}}  ({{`{{ printf "%.2f" (index $values "A").Value }}`}}%  usage)
-      isPaused: false
-    {{ end }}
--- a/compose.grafana-oidc.yml
+++ b/compose.grafana-oidc.yml
@ -1,17 +0,0 @@
-version: '3.8'
-
-services:
-  grafana:
-    secrets:
-      - grafana_oidc_client_secret
-    environment:
-      - OIDC_API_URL
-      - OIDC_AUTH_URL
-      - OIDC_CLIENT_ID
-      - OIDC_ENABLED
-      - OIDC_TOKEN_URL
-
-secrets:
-  grafana_oidc_client_secret:
-    external: true
-    name: ${STACK_NAME}_grafana_oidc_client_secret_${SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION}
--- a/compose.grafana-smtp.yml
+++ b/compose.grafana-smtp.yml
@ -1,18 +0,0 @@
-version: '3.8'
-
-services:
-  grafana:
-    secrets:
-      - grafana_smtp_password
-    environment:
-      - GF_SMTP_HOST
-      - GF_SMTP_USER
-      - GF_SMTP_PASSWORD__FILE=/run/secrets/grafana_smtp_password
-      - GF_SMTP_ENABLED
-      - GF_SMTP_FROM_ADDRESS
-      - GF_SMTP_SKIP_VERIFY
-
-secrets:
-  grafana_smtp_password:
-    external: true
-    name: ${STACK_NAME}_grafana_smtp_password_${SECRET_GRAFANA_SMTP_PASSWORD_VERSION}
--- a/compose.grafana.yml
+++ b/compose.grafana.yml
@ -7,6 +7,8 @@ services:
      - grafana-data:/var/lib/grafana:rw
    secrets:
      - grafana_admin_password
+      - grafana_oidc_client_secret
+      - grafana_smtp_password
    configs:
      - source: grafana_custom_ini
        target: /etc/grafana/grafana.ini
@ -22,18 +24,27 @@ services:
        target: /var/lib/grafana/dashboards/traefik.json
      - source: grafana_backup_dashboard_json
        target: /var/lib/grafana/dashboards/backup.json
-      - source: gf_alerts_node
-        target: /etc/grafana/provisioning/alerting/node.yml
+      - source: grafana_alerts_json
+        target: /var/lib/grafana/alerts/alerts.json
    networks:
      - proxy
      - internal
    environment:
      - GF_SERVER_ROOT_URL
      - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/grafana_admin_password
+      - GF_SMTP_HOST
+      - GF_SMTP_USER
+      - GF_SMTP_PASSWORD__FILE=/run/secrets/grafana_smtp_password
+      - GF_SMTP_ENABLED
+      - GF_SMTP_FROM_ADDRESS
+      - GF_SMTP_SKIP_VERIFY
      - GF_SECURITY_ALLOW_EMBEDDING
      - GF_INSTALL_PLUGINS
-      - ALERT_NODE_DISK_SPACE_ENABLED
-      - ALERT_NODE_MEMORY_USAGE_ENABLED
+      - OIDC_API_URL
+      - OIDC_AUTH_URL
+      - OIDC_CLIENT_ID
+      - OIDC_ENABLED
+      - OIDC_TOKEN_URL
    deploy:
      labels:
        - "traefik.enable=true"
@ -73,10 +84,10 @@ configs:
  grafana_backup_dashboard_json:
    name: ${STACK_NAME}_g_backup_dashboard_json_${GRAFANA_BACKUP_DASHBOARD_JSON_VERSION}
    file: grafana-backup-dashboard.json
-  gf_alerts_node:
+  grafana_alerts_json:
    template_driver: golang
-    name: ${STACK_NAME}_gf_alerts_node_${GRAFANA_ALERTS_NODE_VERSION}
-    file: alerts/node.yml.tmpl
+    name: ${STACK_NAME}_g_alerts_json_${GRAFANA_ALERTS_JSON_VERSION}
+    file: grafana-alerts.json.tmpl

 volumes:
  grafana-data:
@ -86,3 +97,9 @@ secrets:
  grafana_admin_password:
    external: true
    name: ${STACK_NAME}_grafana_admin_password_${SECRET_GRAFANA_ADMIN_PASSWORD_VERSION}
+  grafana_oidc_client_secret:
+    external: true
+    name: ${STACK_NAME}_grafana_oidc_client_secret_${SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION}
+  grafana_smtp_password:
+    external: true
+    name: ${STACK_NAME}_grafana_smtp_password_${SECRET_GRAFANA_SMTP_PASSWORD_VERSION}
--- a/compose.prometheus.yml
+++ b/compose.prometheus.yml
@ -30,6 +30,7 @@ services:
        - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
        - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
        - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"

 configs:
  prometheus_yml:
--- a/compose.yml
+++ b/compose.yml
@ -40,7 +40,7 @@ services:
        - "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
        - "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
        - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
-        - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT}"
+        - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.49.2
--- a/grafana-alerts.json.tmpl
+++ b/grafana-alerts.json.tmpl
@ -216,7 +216,7 @@
              "datasourceUid": "PBFA97CFB590B2093",
              "model": {
                "editorMode": "code",
-                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
+                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100",
                "instant": true,
                "intervalMs": 1000,
                "legendFormat": "__auto",
--- a/grafana-swarm-dashboard.json
+++ b/grafana-swarm-dashboard.json
@ -93,6 +93,7 @@
      },
      "hideTimeOverride": true,
      "id": 2,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -106,12 +107,10 @@
          "fields": "",
          "values": false
        },
-        "showPercentChange": false,
        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "textMode": "auto"
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -173,6 +172,7 @@
        "y": 0
      },
      "id": 1,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -186,12 +186,10 @@
          "fields": "",
          "values": false
        },
-        "showPercentChange": false,
        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "textMode": "auto"
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -253,6 +251,7 @@
      },
      "hideTimeOverride": true,
      "id": 4,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -266,12 +265,10 @@
          "fields": "",
          "values": false
        },
-        "showPercentChange": false,
        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "textMode": "auto"
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -338,10 +335,9 @@
        "y": 0
      },
      "id": 8,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
@ -352,10 +348,9 @@
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
-        "sizing": "auto",
        "text": {}
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -410,12 +405,13 @@
      },
      "lines": true,
      "linewidth": 1,
+      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
@ -511,6 +507,7 @@
      },
      "hideTimeOverride": true,
      "id": 3,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -524,12 +521,10 @@
          "fields": "",
          "values": false
        },
-        "showPercentChange": false,
        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "textMode": "auto"
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -590,6 +585,7 @@
      },
      "hideTimeOverride": true,
      "id": 9,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -603,12 +599,10 @@
          "fields": "",
          "values": false
        },
-        "showPercentChange": false,
        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "textMode": "auto"
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -677,10 +671,9 @@
      },
      "hideTimeOverride": true,
      "id": 11,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
@ -691,10 +684,9 @@
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
-        "sizing": "auto",
        "text": {}
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "targets": [
        {
          "datasource": {
@ -721,39 +713,7 @@
      "fieldConfig": {
        "defaults": {
          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "left",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
+            "mode": "thresholds"
          },
          "mappings": [
            {
@ -787,42 +747,33 @@
          },
          "unit": "percent"
        },
-        "overrides": [
-          {
-            "matcher": {
-              "id": "byType",
-              "options": "time"
-            },
-            "properties": [
-              {
-                "id": "custom.axisPlacement",
-                "value": "hidden"
-              }
-            ]
-          }
-        ]
+        "overrides": []
      },
      "gridPos": {
-        "h": 6,
-        "w": 6,
+        "h": 4,
+        "w": 2.6666666666666665,
        "x": 0,
        "y": 8
      },
      "id": 10,
+      "links": [],
      "maxDataPoints": 100,
+      "maxPerRow": 12,
      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
+        "textMode": "auto"
      },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "repeat": "node_id",
      "repeatDirection": "h",
      "targets": [
@ -831,20 +782,18 @@
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
-          "editorMode": "code",
          "exemplar": true,
-          "expr": "node_filesystem_free_bytes{fstype=\"ext4\"}  / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
+          "expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
-          "legendFormat": "{{mountpoint}}",
-          "range": true,
+          "legendFormat": "",
          "refId": "A",
          "step": 20
        }
      ],
      "title": "Available Disk Space $node_id",
-      "type": "timeseries"
+      "type": "stat"
    },
    {
      "aliasColors": {},
@ -862,7 +811,7 @@
        "h": 7,
        "w": 24,
        "x": 0,
-        "y": 14
+        "y": 12
      },
      "hiddenSeries": false,
      "id": 14,
@ -881,12 +830,13 @@
      },
      "lines": true,
      "linewidth": 1,
+      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
@ -950,7 +900,6 @@
            "mode": "palette-classic"
          },
          "custom": {
-            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
@ -964,7 +913,6 @@
              "tooltip": false,
              "viz": false
            },
-            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
@ -1013,11 +961,12 @@
      },
      "gridPos": {
        "h": 7,
-        "w": 6,
+        "w": 2.6666666666666665,
        "x": 0,
-        "y": 21
+        "y": 19
      },
      "id": 15,
+      "links": [],
      "maxPerRow": 12,
      "options": {
        "legend": {
@ -1125,7 +1074,7 @@
        "h": 7,
        "w": 24,
        "x": 0,
-        "y": 28
+        "y": 26
      },
      "hiddenSeries": false,
      "id": 16,
@ -1142,12 +1091,13 @@
      },
      "lines": true,
      "linewidth": 1,
+      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
@ -1228,7 +1178,7 @@
        "h": 7,
        "w": 12,
        "x": 0,
-        "y": 35
+        "y": 33
      },
      "hiddenSeries": false,
      "id": 18,
@ -1245,6 +1195,7 @@
      },
      "lines": true,
      "linewidth": 1,
+      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
@ -1330,7 +1281,7 @@
        "h": 7,
        "w": 12,
        "x": 12,
-        "y": 35
+        "y": 33
      },
      "hiddenSeries": false,
      "id": 19,
@ -1349,6 +1300,7 @@
      },
      "lines": true,
      "linewidth": 1,
+      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
@ -1424,7 +1376,7 @@
        "h": 7,
        "w": 18,
        "x": 0,
-        "y": 42
+        "y": 40
      },
      "hiddenSeries": false,
      "id": 12,
@ -1445,6 +1397,7 @@
      },
      "lines": true,
      "linewidth": 1,
+      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
@ -1546,9 +1499,10 @@
        "h": 7,
        "w": 6,
        "x": 18,
-        "y": 42
+        "y": 40
      },
      "id": 7,
+      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
@ -1646,9 +1600,10 @@
        "h": 7,
        "w": 24,
        "x": 0,
-        "y": 49
+        "y": 47
      },
      "id": 17,
+      "links": [],
      "options": {
        "legend": {
          "calcs": [],
@ -1703,7 +1658,7 @@
        "h": 9,
        "w": 24,
        "x": 0,
-        "y": 56
+        "y": 54
      },
      "id": 30,
      "options": {
@ -1733,7 +1688,8 @@
    }
  ],
  "refresh": "",
-  "schemaVersion": 39,
+  "schemaVersion": 38,
+  "style": "dark",
  "tags": [
    "swarmprom",
    "prometheus",
@ -1880,6 +1836,6 @@
  "timezone": "",
  "title": "Docker Swarm Nodes",
  "uid": "BPlb-Sgik",
-  "version": 7,
+  "version": 24,
  "weekStart": ""
-}
+}
--- a/release/next
+++ b/release/next
@ -1,9 +0,0 @@
-1. OIDC was moved into a seperate compose file. If you have oidc configured you need to add the following line to you .env file:
-
-COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
-
-2. SMTP was moved into a seperate compose file. If you have smtp configured you need to add the following line to you .env file:
-
-COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"
-
-3. The scape-config.example.yml file and add_node() command were updated to use a secure endpoint for the traefik metrics instead http. This requires an updated Traefik recipe that publishes the metrics on https.
--- a/scrape-config.example.yml
+++ b/scrape-config.example.yml
@ -1,4 +1,4 @@
 - targets:
-  - 'metrics.traefik.example.org'
+  - 'example.org:8082'
  - 'node.monitoring.example.org'
  - 'cadvisor.monitoring.example.org'