add all mountpoints to free disk space in Docker Swarm dashboard

2025-11-03 15:33:25 +01:00
4 changed files with 137 additions and 77 deletions
--- a/README.md
+++ b/README.md
@ -18,18 +18,32 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou

 <!-- endmetadata -->

-## Setup Metrics Gathering
+## Setup a Metrics Gathering

 Where gathering.org is the node you want to gather metrics from.

 1. Configure DNS
+  - monitoring.gathering.org
  - cadvisor.monitoring.gathering.org
  - node.monitoring.gathering.org
-2. [Configure Traefik to use BasicAuth](https://git.coopcloud.tech/coop-cloud/traefik#configuring-wildcard-ssl-using-dns)
-3. `abra app new monitoring-ng`
-4. `abra app config monitoring.gathering.org` (for gathering only the main `compose.yml` is needed, nothing more.)
-5. `abra app deploy monitoring.gathering.org`
-6. check that endpoints are up and basic-auth works
+1. Configure Traefik to use BasicAuth
+  * `abra app config traefik.gathering.org`
+  uncomment
+  ```
+  # BASIC_AUTH
+  COMPOSE_FILE="$COMPOSE_FILE:compose.basicauth.yml"
+  BASIC_AUTH=1
+  SECRET_USERSFILE_VERSION=v1
+  ```
+  - Generate userslist with httpasswd hashed password
+    `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
+    make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
+  - `abra app deploy -f traefik`
+1. `abra app new monitoring-ng`
+1. `abra app config monitoring.gathering.org`
+   for gathering only the main `compose.yml` is needed, nothing more.
+1. `abra app deploy monitoring.gathering.org`
+1. check that endpoints are up and basic-auth works
  - cadvisor.monitoring.gathering.org
  - node.monitoring.gathering.org

@ -42,13 +56,16 @@ In case you have no traefik running on the machine, you can expose the ports dir

 ## Setup Metrics Browser

-This builds upon [Setup Metrics Gathering](#setup-metrics-grathering) so make sure you did that first.

 1. Configure DNS
   - monitoring.example.org
+   - prometheus.monitoring.example.org
+   - loki.monitoring.example.org
 2. Setup monitoring stack
-   - `abra app config monitoring.example.org` Uncomment prometheus, loki and grafana
-   - `abra app secret insert monitoring.example.org basic_auth v1 <password>`
+   - `abra app new monitoring-ng`
+   - `abra app config monitoring.example.org`
+      Uncomment all the stuff
+   - `abra app secret insert monitoring.example.org basic_auth v1 <secret>`
      this needs the plaintext traefik basic-auth secret, not the hashed one!
   - `abra app secret ls monitoring.example.org`
   - `abra app deploy monitoring.example.org`
--- a/compose.prometheus.yml
+++ b/compose.prometheus.yml
@ -30,7 +30,6 @@ services:
        - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
        - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
        - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"

 configs:
  prometheus_yml:
--- a/grafana-alerts.json.tmpl
+++ b/grafana-alerts.json.tmpl
@ -216,7 +216,7 @@
              "datasourceUid": "PBFA97CFB590B2093",
              "model": {
                "editorMode": "code",
-                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100",
+                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
                "instant": true,
                "intervalMs": 1000,
                "legendFormat": "__auto",
--- a/grafana-swarm-dashboard.json
+++ b/grafana-swarm-dashboard.json
@ -93,7 +93,6 @@
      },
      "hideTimeOverride": true,
      "id": 2,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -107,10 +106,12 @@
          "fields": "",
          "values": false
        },
+        "showPercentChange": false,
        "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -172,7 +173,6 @@
        "y": 0
      },
      "id": 1,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -186,10 +186,12 @@
          "fields": "",
          "values": false
        },
+        "showPercentChange": false,
        "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -251,7 +253,6 @@
      },
      "hideTimeOverride": true,
      "id": 4,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -265,10 +266,12 @@
          "fields": "",
          "values": false
        },
+        "showPercentChange": false,
        "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -335,9 +338,10 @@
        "y": 0
      },
      "id": 8,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
@ -348,9 +352,10 @@
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
+        "sizing": "auto",
        "text": {}
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -405,13 +410,12 @@
      },
      "lines": true,
      "linewidth": 1,
-      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
@ -507,7 +511,6 @@
      },
      "hideTimeOverride": true,
      "id": 3,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -521,10 +524,12 @@
          "fields": "",
          "values": false
        },
+        "showPercentChange": false,
        "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -585,7 +590,6 @@
      },
      "hideTimeOverride": true,
      "id": 9,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "value",
@ -599,10 +603,12 @@
          "fields": "",
          "values": false
        },
+        "showPercentChange": false,
        "text": {},
-        "textMode": "auto"
+        "textMode": "auto",
+        "wideLayout": true
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -671,9 +677,10 @@
      },
      "hideTimeOverride": true,
      "id": 11,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
@ -684,9 +691,10 @@
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
+        "sizing": "auto",
        "text": {}
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "targets": [
        {
          "datasource": {
@ -713,7 +721,39 @@
      "fieldConfig": {
        "defaults": {
          "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "left",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
          },
          "mappings": [
            {
@ -747,33 +787,42 @@
          },
          "unit": "percent"
        },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byType",
+              "options": "time"
+            },
+            "properties": [
+              {
+                "id": "custom.axisPlacement",
+                "value": "hidden"
+              }
+            ]
+          }
+        ]
      },
      "gridPos": {
-        "h": 4,
-        "w": 2.6666666666666665,
+        "h": 6,
+        "w": 6,
        "x": 0,
        "y": 8
      },
      "id": 10,
-      "links": [],
      "maxDataPoints": 100,
-      "maxPerRow": 12,
      "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "last"
-          ],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
        },
-        "textMode": "auto"
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
      },
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "repeat": "node_id",
      "repeatDirection": "h",
      "targets": [
@ -782,18 +831,20 @@
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
+          "editorMode": "code",
          "exemplar": true,
-          "expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
+          "expr": "node_filesystem_free_bytes{fstype=\"ext4\"}  / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
          "format": "time_series",
          "interval": "",
          "intervalFactor": 2,
-          "legendFormat": "",
+          "legendFormat": "{{mountpoint}}",
+          "range": true,
          "refId": "A",
          "step": 20
        }
      ],
      "title": "Available Disk Space $node_id",
-      "type": "stat"
+      "type": "timeseries"
    },
    {
      "aliasColors": {},
@ -811,7 +862,7 @@
        "h": 7,
        "w": 24,
        "x": 0,
-        "y": 12
+        "y": 14
      },
      "hiddenSeries": false,
      "id": 14,
@ -830,13 +881,12 @@
      },
      "lines": true,
      "linewidth": 1,
-      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
@ -900,6 +950,7 @@
            "mode": "palette-classic"
          },
          "custom": {
+            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
@ -913,6 +964,7 @@
              "tooltip": false,
              "viz": false
            },
+            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
@ -961,12 +1013,11 @@
      },
      "gridPos": {
        "h": 7,
-        "w": 2.6666666666666665,
+        "w": 6,
        "x": 0,
-        "y": 19
+        "y": 21
      },
      "id": 15,
-      "links": [],
      "maxPerRow": 12,
      "options": {
        "legend": {
@ -1074,7 +1125,7 @@
        "h": 7,
        "w": 24,
        "x": 0,
-        "y": 26
+        "y": 28
      },
      "hiddenSeries": false,
      "id": 16,
@ -1091,13 +1142,12 @@
      },
      "lines": true,
      "linewidth": 1,
-      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
      },
      "percentage": false,
-      "pluginVersion": "10.0.2",
+      "pluginVersion": "10.4.14",
      "pointradius": 5,
      "points": false,
      "renderer": "flot",
@ -1178,7 +1228,7 @@
        "h": 7,
        "w": 12,
        "x": 0,
-        "y": 33
+        "y": 35
      },
      "hiddenSeries": false,
      "id": 18,
@ -1195,7 +1245,6 @@
      },
      "lines": true,
      "linewidth": 1,
-      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
@ -1281,7 +1330,7 @@
        "h": 7,
        "w": 12,
        "x": 12,
-        "y": 33
+        "y": 35
      },
      "hiddenSeries": false,
      "id": 19,
@ -1300,7 +1349,6 @@
      },
      "lines": true,
      "linewidth": 1,
-      "links": [],
      "nullPointMode": "null as zero",
      "options": {
        "alertThreshold": true
@ -1376,7 +1424,7 @@
        "h": 7,
        "w": 18,
        "x": 0,
-        "y": 40
+        "y": 42
      },
      "hiddenSeries": false,
      "id": 12,
@ -1397,7 +1445,6 @@
      },
      "lines": true,
      "linewidth": 1,
-      "links": [],
      "nullPointMode": "null",
      "options": {
        "alertThreshold": true
@ -1499,10 +1546,9 @@
        "h": 7,
        "w": 6,
        "x": 18,
-        "y": 40
+        "y": 42
      },
      "id": 7,
-      "links": [],
      "maxDataPoints": 100,
      "options": {
        "colorMode": "none",
@ -1600,10 +1646,9 @@
        "h": 7,
        "w": 24,
        "x": 0,
-        "y": 47
+        "y": 49
      },
      "id": 17,
-      "links": [],
      "options": {
        "legend": {
          "calcs": [],
@ -1658,7 +1703,7 @@
        "h": 9,
        "w": 24,
        "x": 0,
-        "y": 54
+        "y": 56
      },
      "id": 30,
      "options": {
@ -1688,8 +1733,7 @@
    }
  ],
  "refresh": "",
-  "schemaVersion": 38,
-  "style": "dark",
+  "schemaVersion": 39,
  "tags": [
    "swarmprom",
    "prometheus",
@ -1836,6 +1880,6 @@
  "timezone": "",
  "title": "Docker Swarm Nodes",
  "uid": "BPlb-Sgik",
-  "version": 24,
+  "version": 7,
  "weekStart": ""
-}
+}