docs: Update deploy docs and adds prometheus basic auth back

README.md

@@ -18,32 +18,18 @@ It's based heavily on the [monitoring-lite](https://git.coopcloud.tech/coop-clou
 
 <!-- endmetadata -->
 
-## Setup a Metrics Gathering
+## Setup Metrics Gathering
 
 Where gathering.org is the node you want to gather metrics from.
 
 1. Configure DNS
-  - monitoring.gathering.org
   - cadvisor.monitoring.gathering.org
   - node.monitoring.gathering.org
-1. Configure Traefik to use BasicAuth
+2. [Configure Traefik to use BasicAuth](https://git.coopcloud.tech/coop-cloud/traefik#configuring-wildcard-ssl-using-dns)
-  * `abra app config traefik.gathering.org`
+3. `abra app new monitoring-ng`
-  uncomment
+4. `abra app config monitoring.gathering.org` (for gathering only the main `compose.yml` is needed, nothing more.)
-  ```
+5. `abra app deploy monitoring.gathering.org`
-  # BASIC_AUTH
+6. check that endpoints are up and basic-auth works
-  COMPOSE_FILE="$COMPOSE_FILE:compose.basicauth.yml"
-  BASIC_AUTH=1
-  SECRET_USERSFILE_VERSION=v1
-  ```
-  - Generate userslist with httpasswd hashed password
-    `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
-    make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
-  - `abra app deploy -f traefik`
-1. `abra app new monitoring-ng`
-1. `abra app config monitoring.gathering.org`
-   for gathering only the main `compose.yml` is needed, nothing more.
-1. `abra app deploy monitoring.gathering.org`
-1. check that endpoints are up and basic-auth works
   - cadvisor.monitoring.gathering.org
   - node.monitoring.gathering.org
 
@@ -56,16 +42,13 @@ In case you have no traefik running on the machine, you can expose the ports dir
 
 ## Setup Metrics Browser
 
+This builds upon [Setup Metrics Gathering](#setup-metrics-grathering) so make sure you did that first.
 
 1. Configure DNS
    - monitoring.example.org
-   - prometheus.monitoring.example.org
-   - loki.monitoring.example.org
 2. Setup monitoring stack
-   - `abra app new monitoring-ng`
+   - `abra app config monitoring.example.org` Uncomment prometheus, loki and grafana
-   - `abra app config monitoring.example.org`
+   - `abra app secret insert monitoring.example.org basic_auth v1 <password>`
-      Uncomment all the stuff
-   - `abra app secret insert monitoring.example.org basic_auth v1 <secret>`
       this needs the plaintext traefik basic-auth secret, not the hashed one!
    - `abra app secret ls monitoring.example.org`
    - `abra app deploy monitoring.example.org`

compose.prometheus.yml

@@ -30,6 +30,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
 
 configs:
   prometheus_yml:

grafana-alerts.json.tmpl

@@ -216,7 +216,7 @@
               "datasourceUid": "PBFA97CFB590B2093",
               "model": {
                 "editorMode": "code",
-                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
+                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100",
                 "instant": true,
                 "intervalMs": 1000,
                 "legendFormat": "__auto",

grafana-swarm-dashboard.json

@@ -93,6 +93,7 @@
       },
       "hideTimeOverride": true,
       "id": 2,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -106,12 +107,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -173,6 +172,7 @@
         "y": 0
       },
       "id": 1,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -186,12 +186,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -253,6 +251,7 @@
       },
       "hideTimeOverride": true,
       "id": 4,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -266,12 +265,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -338,10 +335,9 @@
         "y": 0
       },
       "id": 8,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -352,10 +348,9 @@
         },
         "showThresholdLabels": false,
         "showThresholdMarkers": true,
-        "sizing": "auto",
         "text": {}
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -410,12 +405,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null",
       "options": {
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -511,6 +507,7 @@
       },
       "hideTimeOverride": true,
       "id": 3,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -524,12 +521,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -590,6 +585,7 @@
       },
       "hideTimeOverride": true,
       "id": 9,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "value",
@@ -603,12 +599,10 @@
           "fields": "",
           "values": false
         },
-        "showPercentChange": false,
         "text": {},
-        "textMode": "auto",
+        "textMode": "auto"
-        "wideLayout": true
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -677,10 +671,9 @@
       },
       "hideTimeOverride": true,
       "id": 11,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
-        "minVizHeight": 75,
-        "minVizWidth": 75,
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -691,10 +684,9 @@
         },
         "showThresholdLabels": false,
         "showThresholdMarkers": true,
-        "sizing": "auto",
         "text": {}
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -721,39 +713,7 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "left",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
           },
           "mappings": [
             {
@@ -787,42 +747,33 @@
           },
           "unit": "percent"
         },
-        "overrides": [
+        "overrides": []
-          {
-            "matcher": {
-              "id": "byType",
-              "options": "time"
-            },
-            "properties": [
-              {
-                "id": "custom.axisPlacement",
-                "value": "hidden"
-              }
-            ]
-          }
-        ]
       },
       "gridPos": {
-        "h": 6,
+        "h": 4,
-        "w": 6,
+        "w": 2.6666666666666665,
         "x": 0,
         "y": 8
       },
       "id": 10,
+      "links": [],
       "maxDataPoints": 100,
+      "maxPerRow": 12,
       "options": {
-        "legend": {
+        "colorMode": "value",
-          "calcs": [],
+        "graphMode": "area",
-          "displayMode": "list",
+        "justifyMode": "auto",
-          "placement": "bottom",
+        "orientation": "horizontal",
-          "showLegend": true
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
         },
-        "tooltip": {
+        "textMode": "auto"
-          "mode": "single",
-          "sort": "none"
-        }
       },
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "repeat": "node_id",
       "repeatDirection": "h",
       "targets": [
@@ -831,20 +782,18 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
-          "editorMode": "code",
           "exemplar": true,
-          "expr": "node_filesystem_free_bytes{fstype=\"ext4\"}  / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
+          "expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
-          "legendFormat": "{{mountpoint}}",
+          "legendFormat": "",
-          "range": true,
           "refId": "A",
           "step": 20
         }
       ],
       "title": "Available Disk Space $node_id",
-      "type": "timeseries"
+      "type": "stat"
     },
     {
       "aliasColors": {},
@@ -862,7 +811,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 14
+        "y": 12
       },
       "hiddenSeries": false,
       "id": 14,
@@ -881,12 +830,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -950,7 +900,6 @@
             "mode": "palette-classic"
           },
           "custom": {
-            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -964,7 +913,6 @@
               "tooltip": false,
               "viz": false
             },
-            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1013,11 +961,12 @@
       },
       "gridPos": {
         "h": 7,
-        "w": 6,
+        "w": 2.6666666666666665,
         "x": 0,
-        "y": 21
+        "y": 19
       },
       "id": 15,
+      "links": [],
       "maxPerRow": 12,
       "options": {
         "legend": {
@@ -1125,7 +1074,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 28
+        "y": 26
       },
       "hiddenSeries": false,
       "id": 16,
@@ -1142,12 +1091,13 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "10.4.14",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1228,7 +1178,7 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 35
+        "y": 33
       },
       "hiddenSeries": false,
       "id": 18,
@@ -1245,6 +1195,7 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
@@ -1330,7 +1281,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 35
+        "y": 33
       },
       "hiddenSeries": false,
       "id": 19,
@@ -1349,6 +1300,7 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null as zero",
       "options": {
         "alertThreshold": true
@@ -1424,7 +1376,7 @@
         "h": 7,
         "w": 18,
         "x": 0,
-        "y": 42
+        "y": 40
       },
       "hiddenSeries": false,
       "id": 12,
@@ -1445,6 +1397,7 @@
       },
       "lines": true,
       "linewidth": 1,
+      "links": [],
       "nullPointMode": "null",
       "options": {
         "alertThreshold": true
@@ -1546,9 +1499,10 @@
         "h": 7,
         "w": 6,
         "x": 18,
-        "y": 42
+        "y": 40
       },
       "id": 7,
+      "links": [],
       "maxDataPoints": 100,
       "options": {
         "colorMode": "none",
@@ -1646,9 +1600,10 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 49
+        "y": 47
       },
       "id": 17,
+      "links": [],
       "options": {
         "legend": {
           "calcs": [],
@@ -1703,7 +1658,7 @@
         "h": 9,
         "w": 24,
         "x": 0,
-        "y": 56
+        "y": 54
       },
       "id": 30,
       "options": {
@@ -1733,7 +1688,8 @@
     }
   ],
   "refresh": "",
-  "schemaVersion": 39,
+  "schemaVersion": 38,
+  "style": "dark",
   "tags": [
     "swarmprom",
     "prometheus",
@@ -1880,6 +1836,6 @@
   "timezone": "",
   "title": "Docker Swarm Nodes",
   "uid": "BPlb-Sgik",
-  "version": 7,
+  "version": 24,
   "weekStart": ""
-}
+}