add backup label

.env.sample

@@ -5,7 +5,7 @@ DOMAIN=monitoring.example.com
 TIMEOUT=120
 
 ## Enable this secret for Promtail / Prometheus
-# SECRET_BASIC_AUTH_ADMIN_PASSWORD_VERSION=v1
+# SECRET_BASIC_AUTH_VERSION=v1
 #
 # Promtail (Gathering Logs)
 # COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
@@ -36,16 +36,16 @@ TIMEOUT=120
 ## Grafana
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
-# GF_SERVER_ROOT_URL=https://${DOMAIN}
+# GF_SERVER_ROOT_URL=https://monitoring.example.com
 # SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
 #
 ## Single-Sign-On with OIDC
 # OIDC_ENABLED=1
 # SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION=v1
 # OIDC_CLIENT_ID=grafana
-# OIDC_AUTH_URL="https://sso.example.com/auth/realms/autonomic/protocol/openid-connect/auth"
-# OIDC_API_URL="https://sso.example.com/auth/realms/autonomic/protocol/openid-connect/userinfo"
-# OIDC_TOKEN_URL="https://sso.example.com/auth/realms/autonomic/protocol/openid-connect/token"
+# OIDC_AUTH_URL="https://authentik.example.com/application/o/authorize/"
+# OIDC_API_URL="https://authentik.example.com/application/o/userinfo/"
+# OIDC_TOKEN_URL="https://authentik.example.com/application/o/token/"
 #
 ## Additional grafana settings (unlikely to require editing)
 # GF_SECURITY_ALLOW_EMBEDDING=1
@@ -58,4 +58,4 @@ TIMEOUT=120
 # GF_SMTP_FROM_ADDRESS=grafana@example.com
 # GF_SMTP_SKIP_VERIFY=false
 # SECRET_GRAFANA_SMTP_PASSWORD_VERSION=v1
-#
+#

README.md

@@ -58,7 +58,7 @@ Where gathering.org is the node you want to gather metrics from.
    - `abra app new monitoring-ng`
    - `abra app config monitoring.example.org`
       Uncomment all the stuff
-   - `abra app secret insert monitoring.example.org basic_auth_admin_password v1 <secret>`
+   - `abra app secret insert monitoring.example.org basic_auth v1 <secret>`
       this needs the plaintext traefik basic-auth secret, not the hashed one!
    - `abra app secret ls monitoring.example.org`
    - `abra app deploy monitoring.example.org`
@@ -118,4 +118,4 @@ $ systemctl restart docker.service
 
 ---
 
-THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal
+THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal

abra.sh

@@ -1,13 +1,13 @@
-export NODE_EXPORTER_ENTRYPOINT_VERSION=v1
+export ENTRYPOINT_VERSION=v1
 export GRAFANA_DATASOURCES_YML_VERSION=v1
 export GRAFANA_DASHBOARDS_YML_VERSION=v1
-export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v1
-export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v1
-export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v1
-export GRAFANA_CUSTOM_INI_VERSION=v3
-export PROMTAIL_YML_VERSION=v2
+export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_CUSTOM_INI_VERSION=v4
+export PROMTAIL_YML_VERSION=v3
 export LOKI_YML_VERSION=v2
-export PROMETHEUS_YML_VERSION=v1
+export PROMETHEUS_YML_VERSION=v2
 
 # creates a default prometheus scrape config for a given node
 add_node(){

compose.grafana.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:10.0.2
+    image: grafana/grafana:10.4.2
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:

compose.loki.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   loki:
-    image: grafana/loki:2.8.2
+    image: grafana/loki:2.9.7
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - proxy

compose.prometheus.yml

@@ -2,9 +2,9 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v2.45.0
+    image: prom/prometheus:v2.51.2
     secrets:
-      - basic_auth_admin_password
+      - basic_auth
     volumes:
       - prometheus-data:/prometheus:rw
     configs:
@@ -38,4 +38,4 @@ configs:
     file: prometheus.yml.tmpl
 
 volumes:
-  prometheus-data:
+  prometheus-data:

compose.promtail.yml

@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   promtail:
-    image: grafana/promtail:2.8.2
+    image: grafana/promtail:2.9.7
     volumes:
       - /var/log:/var/log:ro
       - /var/run/docker.sock:/var/run/docker.sock
@@ -13,7 +13,7 @@ services:
     networks:
       - internal
     secrets:
-      - basic_auth_admin_password
+      - basic_auth
     environment:
       - DOMAIN
       - LOKI_PUSH_URL
@@ -25,6 +25,6 @@ configs:
     template_driver: golang
 
 secrets:
-  basic_auth_admin_password:
+  basic_auth:
     external: true
-    name: ${STACK_NAME}_basic_auth_admin_password_${SECRET_BASIC_AUTH_ADMIN_PASSWORD_VERSION}
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: prom/node-exporter:v1.6.0
+    image: prom/node-exporter:v1.7.0
     user: root
     environment:
       - NODE_ID={{.Node.ID}}
@@ -20,7 +20,7 @@ services:
       - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)"
       - "--no-collector.ipvs"
     configs:
-      - source: node_exporter_entrypoint_sh
+      - source: entrypoint
         target: /entrypoint.sh
     networks:
       - internal
@@ -30,6 +30,7 @@ services:
       restart_policy:
         condition: on-failure
       labels:
+        - "backupbot.backup=true"
         - "traefik.enable=true"
         - "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
         - "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
@@ -37,14 +38,14 @@ services:
         - "traefik.http.routers.${STACK_NAME}-node.tls=true"
         - "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
-        - "coop-cloud.${STACK_NAME}.version=0.3.0+v1.6.0"
+        - "coop-cloud.${STACK_NAME}.version=1.1.0+v1.7.0"
         - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
 
   cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.47.2
+    image: gcr.io/cadvisor/cadvisor:v0.49.1
     command:
       - "-logtostderr"
-      - "--enable_metrics=cpu,cpuLoad,disk,memory,network"
+      - "--enable_metrics=cpu,cpuLoad,disk,diskIO,process,memory,network"
       # all possible metrics: advtcp,app,cpu,cpuLoad,cpu_topology,cpuset,disk,diskIO,hugetlb,memory,memory_numa,network,oom_event,percpu,perf_event,process,referenced_memory,resctrl,sched,tcp,udp.
       - "--housekeeping_interval=120s"
       - "--docker_only=true"
@@ -76,8 +77,8 @@ services:
       start_period: 30s
 
 configs:
-  node_exporter_entrypoint_sh:
-    name: ${STACK_NAME}_node_exporter_entrypoint_${NODE_EXPORTER_ENTRYPOINT_VERSION}
+  entrypoint:
+    name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
     file: node-exporter-entrypoint.sh
 
 
@@ -85,4 +86,4 @@ configs:
 networks:
   proxy:
     external: true
-  internal:
+  internal:

grafana-traefik-dashboard.json

@@ -3,7 +3,10 @@
     "list": [
       {
         "builtIn": 1,
-        "datasource": "-- Grafana --",
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -18,17 +21,35 @@
       }
     ]
   },
-  "description": "Simple dashboard for Traefik 2",
+  "description": "Dashboards for Traefik Reverse Proxy",
   "editable": true,
   "fiscalYearStartMonth": 0,
   "gnetId": 11462,
   "graphTooltip": 0,
   "id": 3,
-  "iteration": 1684839198931,
-  "links": [],
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [
+        "menu"
+      ],
+      "targetBlank": false,
+      "title": "dashboards",
+      "tooltip": "",
+      "type": "dashboards",
+      "url": ""
+    }
+  ],
   "liveNow": false,
   "panels": [
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
@@ -87,7 +108,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -108,6 +129,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
@@ -167,7 +192,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -185,6 +210,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
@@ -243,7 +272,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -265,6 +294,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -324,7 +357,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -352,6 +385,10 @@
         "label": "Others",
         "threshold": 0
       },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fontSize": "80%",
       "format": "short",
@@ -397,6 +434,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "decimals": 0,
       "fieldConfig": {
         "defaults": {
@@ -433,7 +474,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -493,6 +534,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -530,7 +575,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -587,6 +632,10 @@
       }
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "mappings": [
@@ -652,7 +701,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -676,6 +725,10 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -712,7 +765,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -726,12 +779,14 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(delta(traefik_service_requests_total{instance=\"$instance\"}[$interval]))",
+          "expr": "sum(delta(traefik_service_requests_total{instance=\"${instance:raw}\"}[$interval]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
           "legendFormat": "Total requests",
+          "range": true,
           "refId": "A"
         }
       ],
@@ -769,6 +824,10 @@
       }
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": [],
@@ -812,7 +871,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -832,6 +891,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -839,6 +902,8 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
@@ -899,8 +964,11 @@
             "lastNotNull",
             "max"
           ],
-          "displayMode": "list",
-          "placement": "right"
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true
         },
         "tooltip": {
           "mode": "single",
@@ -914,12 +982,14 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "rate(traefik_service_request_duration_seconds_sum{ instance=\"$instance\" }[5m])",
+          "expr": "sum(rate(traefik_service_request_duration_seconds_sum{ instance=\"$instance\" }[5m])) by(service)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
           "legendFormat": "{{ service }}",
+          "range": true,
           "refId": "A"
         }
       ],
@@ -931,6 +1001,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -964,7 +1038,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1023,6 +1097,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "decimals": 0,
       "fieldConfig": {
         "defaults": {
@@ -1061,7 +1139,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1121,6 +1199,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -1158,7 +1240,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1217,6 +1299,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -1254,7 +1340,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1313,6 +1399,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "decimals": 0,
       "fieldConfig": {
         "defaults": {
@@ -1349,7 +1439,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1413,13 +1503,14 @@
     }
   ],
   "refresh": "",
-  "schemaVersion": 35,
+  "schemaVersion": 38,
   "style": "dark",
   "tags": [
     "traefik",
     "load-balancer",
     "docker",
-    "prometheus"
+    "prometheus",
+    "menu"
   ],
   "templating": {
     "list": [
@@ -1455,7 +1546,7 @@
       },
       {
         "current": {
-          "selected": false,
+          "selected": true,
           "text": [
             "All"
           ],
@@ -1492,7 +1583,7 @@
         "auto_count": 30,
         "auto_min": "10s",
         "current": {
-          "selected": true,
+          "selected": false,
           "text": "5m",
           "value": "5m"
         },
@@ -1562,17 +1653,12 @@
           }
         ],
         "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
-        "queryValue": "5m",
+        "queryValue": "",
         "refresh": 2,
         "skipUrlSync": false,
         "type": "interval"
       },
       {
-        "current": {
-          "selected": true,
-          "text": "demo.local-it.cloud:8082",
-          "value": "demo.local-it.cloud:8082"
-        },
         "datasource": {
           "type": "prometheus",
           "uid": "PBFA97CFB590B2093"
@@ -1586,12 +1672,12 @@
         "options": [],
         "query": {
           "query": "label_values(instance)",
-          "refId": "StandardVariableQuery"
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
-        "regex": ".*8082.*",
+        "regex": ".*8082",
         "skipUrlSync": false,
-        "sort": 0,
+        "sort": 1,
         "tagValuesQuery": "",
         "tagsQuery": "",
         "type": "query",
@@ -1600,7 +1686,7 @@
     ]
   },
   "time": {
-    "from": "now-15m",
+    "from": "now-2d",
     "to": "now"
   },
   "timepicker": {
@@ -1629,8 +1715,8 @@
     ]
   },
   "timezone": "",
-  "title": "Traefik 2",
+  "title": "Traefik Reverse Proxy",
   "uid": "3ipsWfViz",
-  "version": 5,
+  "version": 9,
   "weekStart": ""
-}
+}

prometheus.yml.tmpl

@@ -17,4 +17,4 @@ scrape_configs:
         - /prometheus/scrape_configs/*.yml
     basic_auth:
       username: admin
-      password: {{ secret "basic_auth_admin_password" }}
+      password: {{ secret "basic_auth" }}

promtail.yml.tmpl

@@ -9,7 +9,7 @@ clients:
   - url: {{ env "LOKI_PUSH_URL" }}
     basic_auth:
       username: admin
-      password: {{ secret "basic_auth_admin_password" }}
+      password: {{ secret "basic_auth" }}
     external_labels:
       hostname: {{ env "DOMAIN" }}
 
@@ -34,4 +34,4 @@ scrape_configs:
     - source_labels: ['__meta_docker_container_label_com_docker_stack_namespace']
       target_label: "stack_namespace"
     - source_labels: ['__meta_docker_container_label_com_docker_swarm_service_name']
-      target_label: "service_name"
+      target_label: "service_name"

release/1.0.0+v1.7.0

@@ -0,0 +1,4 @@
+Breakng change: secret `basic_auth_admin_password` was renamed to `basic_auth`. Insert the secret before upgrading. And change the env BASIC_AUTH_ADMIN_PASSWORD to BASIC_AUTH
+
+abra app secret insert monitoring.example.com basic_auth v1 $(abra app run monitoring.example.com promtail cat /var/run/secrets/basic_auth_admin_password)
+sed -i ~/.abra/servers/example.com/monitoring.example.com.env -e 's/BASIC_AUTH_ADMIN_PASSWORD/BASIC_AUTH/'