fix backup missing alert & bump Not successful alert to 60m

README.md

@@ -36,7 +36,7 @@ Where gathering.org is the node you want to gather metrics from.
   SECRET_USERSFILE_VERSION=v1
   ```
   - Generate userslist with httpasswd hashed password
-    `abra app secret insert traefik.gathering.org userslist v1 'admin:<hashed-secret>'`
+    `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
     make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
   - `abra app deploy -f traefik`
 1. `abra app new monitoring-ng`

compose.grafana.yml

@@ -48,6 +48,7 @@ services:
     deploy:
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
         - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"

compose.loki.yml

@@ -27,6 +27,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-loki.loadbalancer.server.port=3100"
         - "traefik.http.routers.${STACK_NAME}-loki.rule=Host(`loki.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-loki.entrypoints=web-secure"
@@ -48,4 +49,4 @@ volumes:
 # secrets:
 #   loki_aws_secret_access_key:
 #     external: true
-#     name: ${STACK_NAME}_loki_aws_secret_access_key_${SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION}
+#     name: ${STACK_NAME}_loki_aws_secret_access_key_${SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION}

compose.prometheus.yml

@@ -24,12 +24,12 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-prometheus.loadbalancer.server.port=9090"
         - "traefik.http.routers.${STACK_NAME}-prometheus.rule=Host(`prometheus.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
 
 configs:
   prometheus_yml:

compose.pushgateway.yml

@@ -17,6 +17,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-pushgateway.loadbalancer.server.port=9191"
         - "traefik.http.routers.${STACK_NAME}-pushgateway.rule=Host(`pushgateway.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-pushgateway.entrypoints=web-secure"

compose.yml

@@ -32,6 +32,7 @@ services:
       labels:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
         - "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure"
@@ -63,6 +64,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080"
         - "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure"

grafana-alerts.json.tmpl

@@ -68,50 +68,76 @@
           "condition": "B",
           "data": [
             {
-              "refId": "A",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "PBFA97CFB590B2093",
-              "model": {
-                "disableTextWrap": false,
-                "editorMode": "builder",
-                "expr": "rate(backup[24h])",
-                "fullMetaSearch": false,
-                "includeNullMetadata": true,
-                "instant": true,
-                "intervalMs": 1000,
-                "legendFormat": "__auto",
-                "maxDataPoints": 43200,
-                "range": false,
                 "refId": "A",
-                "useBackend": false
-              }
+                "relativeTimeRange": {
+                    "from": 28800,
+                    "to": 0
+                },
+                "datasourceUid": "PBFA97CFB590B2093",
+                "model": {
+                    "disableTextWrap": false,
+                    "editorMode": "code",
+                    "expr": "rate(backup[25h]) == 0",
+                    "fullMetaSearch": false,
+                    "includeNullMetadata": true,
+                    "instant": true,
+                    "intervalMs": 1000,
+                    "legendFormat": "__auto",
+                    "maxDataPoints": 43200,
+                    "range": false,
+                    "refId": "A",
+                    "useBackend": false
+                }
             },
             {
-              "refId": "B",
-              "relativeTimeRange": { "from": 600, "to": 0 },
-              "datasourceUid": "__expr__",
-              "model": {
-                "conditions": [
-                  {
-                    "evaluator": { "params": [0, 0], "type": "within_range" },
-                    "operator": { "type": "and" },
-                    "query": { "params": ["C"] },
-                    "reducer": { "params": [], "type": "last" },
-                    "type": "query"
-                  }
-                ],
-                "datasource": { "type": "__expr__", "uid": "__expr__" },
-                "expression": "A",
-                "intervalMs": 1000,
-                "maxDataPoints": 43200,
                 "refId": "B",
-                "type": "threshold"
-              }
+                "relativeTimeRange": {
+                    "from": 28800,
+                    "to": 0
+                },
+                "datasourceUid": "__expr__",
+                "model": {
+                    "conditions": [
+                        {
+                            "evaluator": {
+                                "params": [
+                                    -1,
+                                    0
+                                ],
+                                "type": "gt"
+                            },
+                            "operator": {
+                                "type": "and"
+                            },
+                            "query": {
+                                "params": []
+                            },
+                            "reducer": {
+                                "params": [],
+                                "type": "avg"
+                            },
+                            "type": "query"
+                        }
+                    ],
+                    "datasource": {
+                        "name": "Expression",
+                        "type": "__expr__",
+                        "uid": "__expr__"
+                    },
+                    "expression": "A",
+                    "hide": false,
+                    "intervalMs": 1000,
+                    "maxDataPoints": 43200,
+                    "refId": "B",
+                    "type": "threshold"
+                }
             }
           ],
           "noDataState": "NoData",
           "execErrState": "Error",
           "for": "5m",
+          "annotations": {},
+          "labels": {},
           "isPaused": false
         },
         {{ end }}
@@ -188,9 +214,9 @@
             ],
             "noDataState": "NoData",
             "execErrState": "Error",
-            "for": "20m",
+            "for": "60m",
             "annotations": {
-                "summary": "Backup did not finish within 20 minutes"
+                "summary": "Backup did not finish within 60 minutes"
             },
             "labels": {},
             "isPaused": false