fix: Removes duplicate basic auth from prometheus and a few other improvements

Reviewed-on: #2

.env.sample

@@ -3,28 +3,27 @@ LETS_ENCRYPT_ENV=production
 COMPOSE_FILE=compose.yml
 DOMAIN=monitoring.example.com
 TIMEOUT=120
+ENABLE_BACKUPS=true
 
-# Monitoring Client
-#
-## Node Exporter, Cadvisor (Gathering Metrics)
-# COMPOSE_FILE="$COMPOSE_FILE:compose.metrics.yml"
-#
 ## Enable this secret for Promtail / Prometheus
-# SECRET_BASIC_AUTH_ADMIN_PASSWORD_VERSION=v1
+# SECRET_BASIC_AUTH_VERSION=v1
 #
 # Promtail (Gathering Logs)
 # COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
 # LOKI_PUSH_URL=https://loki.monitoring.example.org/loki/api/v1/push
 
+## Expose node and cadvisor ports instead of traefik
+# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
+
 # Monitoring Server
 #
-## Prometheus, Alertmanager
+## Prometheus
 # COMPOSE_FILE="$COMPOSE_FILE:compose.prometheus.yml"
-# ALERTMANAGER_SMTP_FROM=noreply@autonomic.zone
-# ALERTMANAGER_SMTP_HOST=mail.gandi.net:587
-# ALERTMANAGER_SMTP_TO=kaboom@autonomic.zone
-# SECRET_ALERTMANAGER_SMTP_PASSWORD_VERSION=v1
 # PROMETHEUS_RETENTION_TIME=1y
+#
+## Prometheus Pushgateway
+# COMPOSE_FILE="$COMPOSE_FILE:compose.pushgateway.yml"
+#
 ## Loki
 # Loki Server
 #
@@ -45,16 +44,18 @@ TIMEOUT=120
 ## Grafana
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
-# GF_SERVER_ROOT_URL=https://${DOMAIN}
+# GF_SERVER_ROOT_URL=https://monitoring.example.com
 # SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
+## Seperate domain for Grafana
+#GRAFANA_DOMAIN=grafana.example.com
 #
 ## Single-Sign-On with OIDC
 # OIDC_ENABLED=1
 # SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION=v1
 # OIDC_CLIENT_ID=grafana
-# OIDC_AUTH_URL="https://sso.example.com/auth/realms/autonomic/protocol/openid-connect/auth"
-# OIDC_API_URL="https://sso.example.com/auth/realms/autonomic/protocol/openid-connect/userinfo"
-# OIDC_TOKEN_URL="https://sso.example.com/auth/realms/autonomic/protocol/openid-connect/token"
+# OIDC_AUTH_URL="https://authentik.example.com/application/o/authorize/"
+# OIDC_API_URL="https://authentik.example.com/application/o/userinfo/"
+# OIDC_TOKEN_URL="https://authentik.example.com/application/o/token/"
 #
 ## Additional grafana settings (unlikely to require editing)
 # GF_SECURITY_ALLOW_EMBEDDING=1
@@ -62,6 +63,23 @@ TIMEOUT=120
 #
 ## grafana SMTP configuration (optional)
 # GF_SMTP_HOST=changeme
-# GF_SMTP_ENABLED=1
+# GF_SMTP_USER=changme
+# GF_SMTP_ENABLED=true
 # GF_SMTP_FROM_ADDRESS=grafana@example.com
-# GF_SMTP_SKIP_VERIFY=1
+# GF_SMTP_SKIP_VERIFY=false
+# SECRET_GRAFANA_SMTP_PASSWORD_VERSION=v1
+#
+
+## Grafana Matrix Contact Point (optional)
+#COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
+#SECRET_MATRIX_ACCESS_TOKEN_VERSION=v1
+#GF_MATRIX_USER_ID="<user-id>"
+#GF_MATRIX_ROOM_ID="<room-id>"
+#GF_MATRIX_HOMESERVER_URL="<homeserver-url>"
+
+# ALerts
+#ALERT_BACKUP_FAILED_ENABLED=true
+#ALERT_BACKUP_MISSING_ENABLED=true
+#ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED=true
+#ALERT_NODE_DISK_SPACE_ENABLED=true
+#ALERT_NODE_MEMORY_USAGE_ENABLED=true

README.md

@@ -36,7 +36,7 @@ Where gathering.org is the node you want to gather metrics from.
   SECRET_USERSFILE_VERSION=v1
   ```
   - Generate userslist with httpasswd hashed password
-    `abra app secret insert traefik.gathering.org userslist v1 'admin:<hashed-secret>'`
+    `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
     make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
   - `abra app deploy -f traefik`
 1. `abra app new monitoring-ng`
@@ -47,6 +47,13 @@ Where gathering.org is the node you want to gather metrics from.
   - cadvisor.monitoring.gathering.org
   - node.monitoring.gathering.org
 
+### Expose node and cadvisor via ports instead of traefik
+
+In case you have no traefik running on the machine, you can expose the ports directly by uncommenting the following line:
+```
+# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
+```
+
 ## Setup Metrics Browser
 
 
@@ -54,15 +61,15 @@ Where gathering.org is the node you want to gather metrics from.
    - monitoring.example.org
    - prometheus.monitoring.example.org
    - loki.monitoring.example.org
-1. Setup monitoring stack
+2. Setup monitoring stack
    - `abra app new monitoring-ng`
    - `abra app config monitoring.example.org`
       Uncomment all the stuff
-   - `abra app secret insert monitoring.example.org basic_auth_admin_password v1 <secret>`
+   - `abra app secret insert monitoring.example.org basic_auth v1 <secret>`
       this needs the plaintext traefik basic-auth secret, not the hashed one!
    - `abra app secret ls monitoring.example.org`
    - `abra app deploy monitoring.example.org`
-1. add scrape config to prometheus
+3. Add scrape config to prometheus
   - `abra app cmd monitoring.example.org prometheus gathering.org`
   - or manually
     ```
@@ -85,7 +92,6 @@ Where gathering.org is the node you want to gather metrics from.
 | Cadvisor      | traefik basic-auth | cadvisor.monitoring.example.org   |
 | Node Exporter | traefik basic-auth | node.monitoring.example.org       |
 
-
 ### Logging from a docker host to loki server without anything else
 
 ```
@@ -101,8 +107,18 @@ $ echo '{
 $ systemctl restart docker.service
 ```
 
+## Setup Push Gateway
 
+1. Enable in the env fiöle by uncommenting the following lines:
+```
+## Prometheus Pushgateway
+# COMPOSE_FILE="$COMPOSE_FILE:compose.pushgateway.yml"
+```
+2. `abra app deploy monitoring.example.org`
 
+This will expose the pushgateway at `https://pushgateway.${DOMAIN}`.
+It is secured behind the same basic auth as the other services.
+After that you need to add the `pushgateway.${DOMAIN}` to the scare config.
 
 ## Post-setup guide
 
@@ -118,4 +134,35 @@ $ systemctl restart docker.service
 
 ---
 
-THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal
+THX to the previous work of @decentral1se @knooflok @3wc @cellarspoon @mirsal
+
+## Adding Matrix as Alert Contact point
+
+1. Enable the [matrix-alertmanager-receiver](https://github.com/metio/matrix-alertmanager-receiver/):
+```
+COMPOSE_FILE="$COMPOSE_FILE:compose.matrix-alertmanager-receiver.yml"
+```
+
+2. Insert the matrix access token secret:
+```
+abra app secret insert monitoring.marx.klasse-methode.it matrix_access_token v1
+```
+
+3. Set required configurations:
+```
+GF_MATRIX_USER_ID=
+GF_MATRIX_ROOM_ID=
+GF_MATRIX_HOME_SERVER_URL=
+```
+4. Configure Alertmanager webhook and set the url to `http://matrix-alertmanager-receiver:12345/alerts/<room-id>`
+
+## alerts
+
+It is possible to enable the following alerts, by setting the corresponding env variable to `true`:
+- backupbot failed: `ALERT_BACKUP_FAILED_ENABLED`
+- backupbot missing: `ALERT_BACKUP_MISSING_ENABLED`
+- backupbot not successfull: `ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED`
+- node disk space: `ALERT_NODE_DISK_SPACE_ENABLED`
+- node memory usage: `ALERT_NODE_MEMORY_USAGE_ENABLED`
+
+

abra.sh

@@ -1,14 +1,17 @@
-export NODE_EXPORTER_ENTRYPOINT_VERSION=v1
+export ENTRYPOINT_VERSION=v1
 export GRAFANA_DATASOURCES_YML_VERSION=v1
-export GRAFANA_DASHBOARDS_YML_VERSION=v1
-export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v1
-export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v1
-export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v1
-export GRAFANA_CUSTOM_INI_VERSION=v1
-export PROMTAIL_YML_VERSION=v1
-export LOKI_YML_VERSION=v1
-export PROMETHEUS_YML_VERSION=v1
-export ALERTMANAGER_CONFIG_VERSION=v1
+export GRAFANA_DASHBOARDS_YML_VERSION=v2
+export GRAFANA_SWARM_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_STACKS_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v2
+export GRAFANA_BACKUP_DASHBOARD_JSON_VERSION=v1
+export GRAFANA_ALERTS_JSON_VERSION=v3
+export GRAFANA_CUSTOM_INI_VERSION=v4
+export PROMTAIL_YML_VERSION=v3
+export LOKI_YML_VERSION=v2
+export PROMETHEUS_YML_VERSION=v2
+export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
+export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
 
 # creates a default prometheus scrape config for a given node
 add_node(){

alertmanager-matrix-config.yml.tmpl

@@ -0,0 +1,74 @@
+# configuration of the HTTP server
+http:
+  ## address: 127.0.0.1              # bind address for this service. Can be left unspecified to bind on all interfaces
+  port: 12345                     # port used by this service
+  alerts-path-prefix: /alerts     # URL path for the webhook receiver called by an Alertmanager. Defaults to /alerts
+  metrics-path: /metrics          # URL path to collect metrics. Defaults to /metrics
+  metrics-enabled: true           # Whether to enable metrics or not. Defaults to false
+  # basic-username: alertmanager    # Username for basic authentication. Defaults to alertmanager
+  # basic-password: secret          # If set, the alerts endpoint expects basic-auth credentials with the configured username and password
+
+# configuration for the Matrix connection
+matrix:
+  homeserver-url: "{{ env "GF_MATRIX_HOMESERVER_URL" }}"
+  user-id: "{{ env "GF_MATRIX_USER_ID" }}"
+  access-token: "{{ secret "matrix_access_token" }}"
+  room-mapping:
+    matrixroom: "{{ env "GF_MATRIX_ROOM_ID" }}"
+
+templating:
+  # mapping of ExternalURL values
+  external-url-mapping:
+    # key is the original value taken from the Alertmanager payload
+    # value is the mapped value which will be available as '.ExternalURL' in templates
+    "http://alertmanager:9093": https://alertmanager.example.com
+  # mapping of GeneratorURL values
+  generator-url-mapping:
+    # key is the original value taken from the Alertmanager payload
+    # value is the mapped value which will be available as '.GeneratorURL' in templates
+    "http://prometheus:8080": https://prometheus.example.com
+
+  # computation of arbitrary values based on matching alert annotations, labels, or status
+  # values will be evaluated top to bottom, last entry wins
+  computed-values:
+    - values: # always set 'color' to 'yellow'
+        color: yellow
+    - values: # set 'color' to 'orange' when alert label 'severity' is 'warning'
+        color: orange
+      when-matching-labels:
+        severity: warning
+    - values: # set 'color' to 'red' when alert label 'severity' is 'critical'
+        color: red
+      when-matching-labels:
+        severity: critical
+    - values: # set 'color' to 'green' when alert status is 'resolved'
+        color: green
+      when-matching-status: resolved
+
+  # template for alerts in status 'firing'
+  firing-template: '{{`
+    <p>
+      <strong><font color="{{ .ComputedValues.color }}">{{ .Alert.Status | ToUpper }}</font></strong>
+      {{ if .Alert.Labels.name }}
+        {{ .Alert.Labels.name }}
+      {{ else if .Alert.Labels.alertname }}
+        {{ .Alert.Labels.alertname }}
+      {{ end }}
+      >>
+      {{ if .Alert.Labels.severity }}
+        {{ .Alert.Labels.severity | ToUpper }}: 
+      {{ end }}
+      {{ if .Alert.Annotations.description }}
+        {{ .Alert.Annotations.description }}
+      {{ else if .Alert.Annotations.summary }}
+        {{ .Alert.Annotations.summary }}
+      {{ end }}
+      >>
+      {{ if .Alert.Annotations.runbook }}
+        <a href="{{ .Alert.Annotations.runbook }}">Runbook</a> | 
+      {{ end }}
+      {{ if .Alert.Annotations.dashboard }}
+        <a href="{{ .Alert.Annotations.dashboard }}">Dashboard</a> | 
+      {{ end }}
+      <a href="{{ .SilenceURL }}">Silence</a>
+    </p>`}}'

alertmanager.yml.tmpl

@@ -1,13 +0,0 @@
-global:
-  smtp_from: {{ env "ALERTMANAGER_SMTP_FROM" }}
-  smtp_smarthost: {{ env "ALERTMANAGER_SMTP_HOST" }}
-  smtp_auth_username: {{ env "ALERTMANAGER_SMTP_FROM" }}
-  smtp_auth_password: {{ secret "alertmanager_smtp_password" }}
-
-route:
-  receiver: "kaboom-mailer"
-
-receivers:
-  - name: "kaboom-mailer"
-    email_configs:
-      - to: {{ env "ALERTMANAGER_SMTP_TO" }}

compose.expose-ports.yml

@@ -0,0 +1,13 @@
+---
+version: "3.8"
+
+services:
+  app:
+    ports:
+      - "9100:9100"
+    deploy:
+
+  cadvisor:
+    ports:
+      - "9101:8080"
+    deploy:

compose.grafana.yml

@@ -2,12 +2,13 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:9.5.2
+    image: grafana/grafana:10.4.14
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:
       - grafana_admin_password
       - grafana_oidc_client_secret
+      - grafana_smtp_password
     configs:
       - source: grafana_custom_ini
         target: /etc/grafana/grafana.ini
@@ -21,18 +22,24 @@ services:
         target: /var/lib/grafana/dashboards/docker-swarm-stacks.json
       - source: grafana_traefik_dashboard_json
         target: /var/lib/grafana/dashboards/traefik.json
+      - source: grafana_backup_dashboard_json
+        target: /var/lib/grafana/dashboards/backup.json
+      - source: grafana_alerts_json
+        target: /var/lib/grafana/alerts/alerts.json
     networks:
       - proxy
       - internal
     environment:
-      - GF_SERVER_ROOT_URL=https://${GRAFANA_DOMAIN}
+      - GF_SERVER_ROOT_URL
       - GF_SECURITY_ADMIN_PASSWORD__FILE=/run/secrets/grafana_admin_password
       - GF_SMTP_HOST
+      - GF_SMTP_USER
+      - GF_SMTP_PASSWORD__FILE=/run/secrets/grafana_smtp_password
       - GF_SMTP_ENABLED
       - GF_SMTP_FROM_ADDRESS
       - GF_SMTP_SKIP_VERIFY
       - GF_SECURITY_ALLOW_EMBEDDING
-      - GF_INSTALL_PLUGINS=grafana-piechart-panel
+      - GF_INSTALL_PLUGINS
       - OIDC_API_URL
       - OIDC_AUTH_URL
       - OIDC_CLIENT_ID
@@ -41,8 +48,9 @@ services:
     deploy:
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
-        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
@@ -59,20 +67,27 @@ configs:
     name: ${STACK_NAME}_grafana_custom_ini_${GRAFANA_CUSTOM_INI_VERSION}
     file: grafana_custom.ini
   grafana_datasources_yml:
-    name: ${STACK_NAME}_grafana_datasources_yml_${GRAFANA_DATASOURCES_YML_VERSION}
+    name: ${STACK_NAME}_g_datasources_yml_${GRAFANA_DATASOURCES_YML_VERSION}
     file: grafana-datasources.yml
   grafana_dashboards_yml:
-    name: ${STACK_NAME}_grafana_dashboards_yml_${GRAFANA_DASHBOARDS_YML_VERSION}
+    name: ${STACK_NAME}_g_dashboards_yml_${GRAFANA_DASHBOARDS_YML_VERSION}
     file: grafana-dashboards.yml
   grafana_swarm_dashboard_json:
-    name: ${STACK_NAME}_grafana_swarm_dashboard_json_${GRAFANA_SWARM_DASHBOARD_JSON_VERSION}
+    name: ${STACK_NAME}_g_swarm_dashboard_json_${GRAFANA_SWARM_DASHBOARD_JSON_VERSION}
     file: grafana-swarm-dashboard.json
   grafana_stacks_dashboard_json:
-    name: ${STACK_NAME}_grafana_stacks_dashboard_json_${GRAFANA_STACKS_DASHBOARD_JSON_VERSION}
+    name: ${STACK_NAME}_g_stacks_dashboard_json_${GRAFANA_STACKS_DASHBOARD_JSON_VERSION}
     file: grafana-stacks-dashboard.json
   grafana_traefik_dashboard_json:
-    name: ${STACK_NAME}_grafana_traefik_dashboard_json_${GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION}
+    name: ${STACK_NAME}_g_traefik_dashboard_json_${GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION}
     file: grafana-traefik-dashboard.json
+  grafana_backup_dashboard_json:
+    name: ${STACK_NAME}_g_backup_dashboard_json_${GRAFANA_BACKUP_DASHBOARD_JSON_VERSION}
+    file: grafana-backup-dashboard.json
+  grafana_alerts_json:
+    template_driver: golang
+    name: ${STACK_NAME}_g_alerts_json_${GRAFANA_ALERTS_JSON_VERSION}
+    file: grafana-alerts.json.tmpl
 
 volumes:
   grafana-data:
@@ -85,3 +100,6 @@ secrets:
   grafana_oidc_client_secret:
     external: true
     name: ${STACK_NAME}_grafana_oidc_client_secret_${SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION}
+  grafana_smtp_password:
+    external: true
+    name: ${STACK_NAME}_grafana_smtp_password_${SECRET_GRAFANA_SMTP_PASSWORD_VERSION}

compose.loki.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   loki:
-    image: grafana/loki:2.8.2
+    image: grafana/loki:2.9.11
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - proxy
@@ -27,6 +27,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-loki.loadbalancer.server.port=3100"
         - "traefik.http.routers.${STACK_NAME}-loki.rule=Host(`loki.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-loki.entrypoints=web-secure"
@@ -48,4 +49,4 @@ volumes:
 # secrets:
 #   loki_aws_secret_access_key:
 #     external: true
-#     name: ${STACK_NAME}_loki_aws_secret_access_key_${SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION}
+#     name: ${STACK_NAME}_loki_aws_secret_access_key_${SECRET_LOKI_AWS_SECRET_ACCESS_KEY_VERSION}

compose.matrix-alertmanager-receiver.yml

@@ -0,0 +1,28 @@
+version: '3.8'
+
+services:
+  matrix-alertmanager-receiver:
+    image: metio/matrix-alertmanager-receiver:2025.2.9
+    secrets:
+      - matrix_access_token
+    configs:
+      - source: matrix-alertmanager-receiver-config
+        target: /etc/matrix-alertmanager-receiver/config.yml
+    networks:
+      - internal
+    environment:
+      - GF_MATRIX_USER_ID
+      - GF_MATRIX_ROOM_ID
+      - GF_MATRIX_HOMESERVER_URL
+    command: "--config-path=/etc/matrix-alertmanager-receiver/config.yml"
+
+configs:
+  matrix-alertmanager-receiver-config:
+    template_driver: golang
+    name: ${STACK_NAME}_mar_config_${MATRIX_ALERTMANAGER_CONFIG_VERSION}
+    file: alertmanager-matrix-config.yml.tmpl
+
+secrets:
+  matrix_access_token:
+    external: true
+    name: ${STACK_NAME}_matrix_access_token_${SECRET_MATRIX_ACCESS_TOKEN_VERSION}

compose.prometheus.yml

@@ -2,9 +2,9 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v2.44.0
+    image: prom/prometheus:v2.55.1
     secrets:
-      - basic_auth_admin_password
+      - basic_auth
     volumes:
       - prometheus-data:/prometheus:rw
     configs:
@@ -24,49 +24,18 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-prometheus.loadbalancer.server.port=9090"
         - "traefik.http.routers.${STACK_NAME}-prometheus.rule=Host(`prometheus.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
         - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
-
-
-  alertmanager:
-    image: prom/alertmanager:v0.25.0
-    volumes:
-      - alertmanager-data:/etc/alertmanager
-    command:
-      - "--config.file=/etc/alertmanager/config.yml"
-      - "--storage.path=/alertmanager"
-    networks:
-      - internal
-    secrets:
-      - alertmanager_smtp_password
-    configs:
-      - source: alertmanager_config
-        target: /etc/alertmanager/config.yml
-    environment:
-      - ALERTMANAGER_SMTP_FROM
-      - ALERTMANAGER_SMTP_HOST
-      - ALERTMANAGER_SMTP_TO
 
 configs:
   prometheus_yml:
     template_driver: golang
     name: ${STACK_NAME}_prometheus_yml_${PROMETHEUS_YML_VERSION}
     file: prometheus.yml.tmpl
-  alertmanager_config:
-    template_driver: golang
-    name: ${STACK_NAME}_alertmanager_config_${ALERTMANAGER_CONFIG_VERSION}
-    file: ./alertmanager.yml.tmpl
-
 
 volumes:
   prometheus-data:
-  alertmanager-data:
-
-secrets:
-  alertmanager_smtp_password:
-    external: true
-    name: ${STACK_NAME}_alertmanager_smtp_password_${SECRET_ALERTMANAGER_SMTP_PASSWORD_VERSION}

compose.promtail.yml

@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   promtail:
-    image: grafana/promtail:2.8.2
+    image: grafana/promtail:2.9.11
     volumes:
       - /var/log:/var/log:ro
       - /var/run/docker.sock:/var/run/docker.sock
@@ -13,8 +13,9 @@ services:
     networks:
       - internal
     secrets:
-      - basic_auth_admin_password
+      - basic_auth
     environment:
+      - DOMAIN
       - LOKI_PUSH_URL
 
 configs:
@@ -24,6 +25,6 @@ configs:
     template_driver: golang
 
 secrets:
-  basic_auth_admin_password:
+  basic_auth:
     external: true
-    name: ${STACK_NAME}_basic_auth_admin_password_${SECRET_BASIC_AUTH_ADMIN_PASSWORD_VERSION}
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.pushgateway.yml

@@ -0,0 +1,26 @@
+version: '3.8'
+
+services:
+  pushgateway:
+    image: prom/pushgateway:v1.10.0
+    command:
+      - '--web.listen-address=:9191'
+      - '--push.disable-consistency-check'
+      - '--persistence.interval=5m'
+    ports:
+      - 9191:9191
+    networks:
+      - internal
+      - proxy
+    deploy:
+      restart_policy:
+        condition: on-failure
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
+        - "traefik.http.services.${STACK_NAME}-pushgateway.loadbalancer.server.port=9191"
+        - "traefik.http.routers.${STACK_NAME}-pushgateway.rule=Host(`pushgateway.${DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-pushgateway.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-pushgateway.tls=true"
+        - "traefik.http.routers.${STACK_NAME}-pushgateway.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-pushgateway.middlewares=basicauth@file"

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: prom/node-exporter:v1.5.0
+    image: prom/node-exporter:v1.8.1
     user: root
     environment:
       - NODE_ID={{.Node.ID}}
@@ -20,7 +20,7 @@ services:
       - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)"
       - "--no-collector.ipvs"
     configs:
-      - source: node_exporter_entrypoint_sh
+      - source: entrypoint
         target: /entrypoint.sh
     networks:
       - internal
@@ -30,25 +30,26 @@ services:
       restart_policy:
         condition: on-failure
       labels:
+        - "backupbot.backup=${ENABLE_BACKUPS:-true}"
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
         - "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-node.tls=true"
         - "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
-        - "coop-cloud.${STACK_NAME}.version=0.1.0+v1.5.0"
+        - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
         - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
 
   cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.47.1
+    image: gcr.io/cadvisor/cadvisor:v0.49.2
     command:
       - "-logtostderr"
-      - "--enable_metrics=cpu,cpuLoad,disk,memory,network"
+      - "--enable_metrics=cpu,cpuLoad,disk,diskIO,process,memory,network"
       # all possible metrics: advtcp,app,cpu,cpuLoad,cpu_topology,cpuset,disk,diskIO,hugetlb,memory,memory_numa,network,oom_event,percpu,perf_event,process,referenced_memory,resctrl,sched,tcp,udp.
       - "--housekeeping_interval=120s"
       - "--docker_only=true"
-
     volumes:
       - /var/lib/docker/:/var/lib/docker:ro
       - /dev/disk/:/dev/disk:ro
@@ -63,6 +64,7 @@ services:
         condition: on-failure
       labels:
         - "traefik.enable=true"
+        - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080"
         - "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure"
@@ -77,8 +79,8 @@ services:
       start_period: 30s
 
 configs:
-  node_exporter_entrypoint_sh:
-    name: ${STACK_NAME}_node_exporter_entrypoint_${NODE_EXPORTER_ENTRYPOINT_VERSION}
+  entrypoint:
+    name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
     file: node-exporter-entrypoint.sh
 
 
@@ -86,4 +88,4 @@ configs:
 networks:
   proxy:
     external: true
-  internal:
+  internal:

grafana-alerts.json.tmpl

@@ -0,0 +1,315 @@
+{
+  "apiVersion": 1,
+  "groups": [
+    {
+      "orgId": 1,
+      "name": "backupbot",
+      "folder": "node",
+      "interval": "1m",
+      "rules": [
+        {{ if eq (env "ALERT_BACKUP_FAILED_ENABLED") "true" }}
+        {
+          "uid": "de8e5xxup7t34a",
+          "title": "Backup Failed",
+          "condition": "C",
+          "data": [
+            {
+              "refId": "A",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "PBFA97CFB590B2093",
+              "model": {
+                "disableTextWrap": false,
+                "editorMode": "builder",
+                "expr": "backup",
+                "fullMetaSearch": false,
+                "includeNullMetadata": true,
+                "instant": true,
+                "intervalMs": 1000,
+                "legendFormat": "__auto",
+                "maxDataPoints": 43200,
+                "range": false,
+                "refId": "A",
+                "useBackend": false
+              }
+            },
+            {
+              "refId": "C",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "__expr__",
+              "model": {
+                "conditions": [
+                  {
+                    "evaluator": { "params": [0], "type": "lt" },
+                    "operator": { "type": "and" },
+                    "query": { "params": ["C"] },
+                    "reducer": { "params": [], "type": "last" },
+                    "type": "query"
+                  }
+                ],
+                "datasource": { "type": "__expr__", "uid": "__expr__" },
+                "expression": "A",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "refId": "C",
+                "type": "threshold"
+              }
+            }
+          ],
+          "noDataState": "NoData",
+          "execErrState": "Error",
+          "for": "1m",
+          "isPaused": false
+        },
+        {{ end }}
+        {{ if eq (env "ALERT_BACKUP_MISSING_ENABLED") "true" }}
+        {
+          "uid": "ce8e65uddcwe8d",
+          "title": "Backup Missing",
+          "condition": "B",
+          "data": [
+            {
+              "refId": "A",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "PBFA97CFB590B2093",
+              "model": {
+                "disableTextWrap": false,
+                "editorMode": "builder",
+                "expr": "rate(backup[24h])",
+                "fullMetaSearch": false,
+                "includeNullMetadata": true,
+                "instant": true,
+                "intervalMs": 1000,
+                "legendFormat": "__auto",
+                "maxDataPoints": 43200,
+                "range": false,
+                "refId": "A",
+                "useBackend": false
+              }
+            },
+            {
+              "refId": "B",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "__expr__",
+              "model": {
+                "conditions": [
+                  {
+                    "evaluator": { "params": [0, 0], "type": "within_range" },
+                    "operator": { "type": "and" },
+                    "query": { "params": ["C"] },
+                    "reducer": { "params": [], "type": "last" },
+                    "type": "query"
+                  }
+                ],
+                "datasource": { "type": "__expr__", "uid": "__expr__" },
+                "expression": "A",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "refId": "B",
+                "type": "threshold"
+              }
+            }
+          ],
+          "noDataState": "NoData",
+          "execErrState": "Error",
+          "for": "5m",
+          "isPaused": false
+        },
+        {{ end }}
+        {{ if eq (env "ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED") "true" }}
+        {
+            "uid": "de8e6bc92a8lcc",
+            "title": "Backup Not Successfull",
+            "condition": "B",
+            "data": [
+                {
+                    "refId": "A",
+                    "relativeTimeRange": {
+                        "from": 60,
+                        "to": 0
+                    },
+                    "datasourceUid": "PBFA97CFB590B2093",
+                    "model": {
+                        "disableTextWrap": false,
+                        "editorMode": "builder",
+                        "expr": "backup",
+                        "fullMetaSearch": false,
+                        "includeNullMetadata": true,
+                        "instant": true,
+                        "intervalMs": 1000,
+                        "legendFormat": "__auto",
+                        "maxDataPoints": 43200,
+                        "range": false,
+                        "refId": "A",
+                        "useBackend": false
+                    }
+                },
+                {
+                    "refId": "B",
+                    "relativeTimeRange": {
+                        "from": 60,
+                        "to": 0
+                    },
+                    "datasourceUid": "__expr__",
+                    "model": {
+                        "conditions": [
+                            {
+                                "evaluator": {
+                                    "params": [
+                                        0
+                                    ],
+                                    "type": "gt"
+                                },
+                                "operator": {
+                                    "type": "and"
+                                },
+                                "query": {
+                                    "params": [
+                                        "C"
+                                    ]
+                                },
+                                "reducer": {
+                                    "params": [],
+                                    "type": "last"
+                                },
+                                "type": "query"
+                            }
+                        ],
+                        "datasource": {
+                            "type": "__expr__",
+                            "uid": "__expr__"
+                        },
+                        "expression": "A",
+                        "intervalMs": 1000,
+                        "maxDataPoints": 43200,
+                        "refId": "B",
+                        "type": "threshold"
+                    }
+                }
+            ],
+            "noDataState": "NoData",
+            "execErrState": "Error",
+            "for": "20m",
+            "annotations": {
+                "summary": "Backup did not finish within 20 minutes"
+            },
+            "labels": {},
+            "isPaused": false
+        }
+        {{ end }}
+      ]
+    },
+    {
+      "orgId": 1,
+      "name": "node",
+      "folder": "node",
+      "interval": "5m",
+      "rules": [
+        {{ if eq (env "ALERT_NODE_DISK_SPACE_ENABLED") "true" }}
+        {
+          "uid": "bds8bhxu97pxca",
+          "title": "Node Disk Space",
+          "condition": "C",
+          "data": [
+            {
+              "refId": "A",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "PBFA97CFB590B2093",
+              "model": {
+                "editorMode": "code",
+                "expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100",
+                "instant": true,
+                "intervalMs": 1000,
+                "legendFormat": "__auto",
+                "maxDataPoints": 43200,
+                "range": false,
+                "refId": "A"
+              }
+            },
+            {
+              "refId": "C",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "__expr__",
+              "model": {
+                "conditions": [
+                  {
+                    "evaluator": { "params": [10], "type": "lt" },
+                    "operator": { "type": "and" },
+                    "query": { "params": ["C"] },
+                    "reducer": { "params": [], "type": "last" },
+                    "type": "query"
+                  }
+                ],
+                "datasource": { "type": "__expr__", "uid": "__expr__" },
+                "expression": "A",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "refId": "C",
+                "type": "threshold"
+              }
+            }
+          ],
+          "noDataState": "NoData",
+          "execErrState": "Error",
+          "for": "5m",
+          "annotations": {},
+          "labels": {},
+          "isPaused": false
+        },
+        {{ end }}
+        {{ if eq (env "ALERT_NODE_MEMORY_USAGE_ENABLED") "true" }}
+        {
+          "uid": "ads8cswmly96oa",
+          "title": "Node Memory Usage",
+          "condition": "C",
+          "data": [
+            {
+              "refId": "A",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "PBFA97CFB590B2093",
+              "model": {
+                "editorMode": "code",
+                "expr": "(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
+                "instant": true,
+                "intervalMs": 1000,
+                "legendFormat": "__auto",
+                "maxDataPoints": 43200,
+                "range": false,
+                "refId": "A"
+              }
+            },
+            {
+              "refId": "C",
+              "relativeTimeRange": { "from": 600, "to": 0 },
+              "datasourceUid": "__expr__",
+              "model": {
+                "conditions": [
+                  {
+                    "evaluator": { "params": [90], "type": "gt" },
+                    "operator": { "type": "and" },
+                    "query": { "params": ["C"] },
+                    "reducer": { "params": [], "type": "last" },
+                    "type": "query"
+                  }
+                ],
+                "datasource": { "type": "__expr__", "uid": "__expr__" },
+                "expression": "A",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "refId": "C",
+                "type": "threshold"
+              }
+            }
+          ],
+          "noDataState": "NoData",
+          "execErrState": "Error",
+          "for": "5m",
+          "annotations": {},
+          "labels": {},
+          "isPaused": false
+        }
+        {{ end }}
+      ]
+    }
+  ]
+}
+

grafana-backup-dashboard.json

@@ -0,0 +1,228 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 6,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 2,
+            "axisSoftMin": -2,
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "dark-green",
+                  "index": 0
+                },
+                "1": {
+                  "color": "dark-yellow",
+                  "index": 1,
+                  "text": "Running"
+                },
+                "-1": {
+                  "index": 2,
+                  "text": "Fail"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "max": 1,
+          "min": -1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "string"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "backup",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Backup Status",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 24,
+        "x": 0,
+        "y": 7
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "builder",
+          "expr": "{service_name=\"$ServiceName\"} |= ``",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Backupbot Logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "auto",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "backup_marx_klasse-methode_it_app",
+          "value": "backup_marx_klasse-methode_it_app"
+        },
+        "datasource": {
+          "type": "loki",
+          "uid": "P8E80F9AEF21F6940"
+        },
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Backupbot Service",
+        "multi": false,
+        "name": "ServiceName",
+        "options": [],
+        "query": {
+          "label": "service_name",
+          "refId": "LokiVariableQueryEditor-VariableQuery",
+          "stream": "",
+          "type": 1
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "backupbot-two",
+  "uid": "be8e2xeofw4xsa",
+  "version": 3,
+  "weekStart": ""
+}

grafana-dashboards.yml

@@ -11,3 +11,13 @@ providers:
     options:
       path: /var/lib/grafana/dashboards
       foldersFromFilesStructure: true
+  - name: 'default-alert-provider'
+    orgId: 1
+    folder: 'default-alerts'
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/alerts
+      foldersFromFilesStructure: true

grafana-traefik-dashboard.json

@@ -3,7 +3,10 @@
     "list": [
       {
         "builtIn": 1,
-        "datasource": "-- Grafana --",
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -18,17 +21,35 @@
       }
     ]
   },
-  "description": "Simple dashboard for Traefik 2",
+  "description": "Dashboards for Traefik Reverse Proxy",
   "editable": true,
   "fiscalYearStartMonth": 0,
   "gnetId": 11462,
   "graphTooltip": 0,
   "id": 3,
-  "iteration": 1684839198931,
-  "links": [],
+  "links": [
+    {
+      "asDropdown": false,
+      "icon": "external link",
+      "includeVars": false,
+      "keepTime": false,
+      "tags": [
+        "menu"
+      ],
+      "targetBlank": false,
+      "title": "dashboards",
+      "tooltip": "",
+      "type": "dashboards",
+      "url": ""
+    }
+  ],
   "liveNow": false,
   "panels": [
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
@@ -87,7 +108,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -108,6 +129,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
@@ -167,7 +192,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -185,6 +210,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
@@ -243,7 +272,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -265,6 +294,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -324,7 +357,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -352,6 +385,10 @@
         "label": "Others",
         "threshold": 0
       },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fontSize": "80%",
       "format": "short",
@@ -397,6 +434,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "decimals": 0,
       "fieldConfig": {
         "defaults": {
@@ -433,7 +474,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -493,6 +534,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -530,7 +575,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -587,6 +632,10 @@
       }
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "mappings": [
@@ -652,7 +701,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -676,6 +725,10 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -712,7 +765,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -726,12 +779,14 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(delta(traefik_service_requests_total{instance=\"$instance\"}[$interval]))",
+          "expr": "sum(delta(traefik_service_requests_total{instance=\"${instance:raw}\"}[$interval]))",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
           "legendFormat": "Total requests",
+          "range": true,
           "refId": "A"
         }
       ],
@@ -769,6 +824,10 @@
       }
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": [],
@@ -812,7 +871,7 @@
         },
         "textMode": "auto"
       },
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "datasource": {
@@ -832,6 +891,10 @@
       "type": "stat"
     },
     {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -839,6 +902,8 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
@@ -899,8 +964,11 @@
             "lastNotNull",
             "max"
           ],
-          "displayMode": "list",
-          "placement": "right"
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true
         },
         "tooltip": {
           "mode": "single",
@@ -914,12 +982,14 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "rate(traefik_service_request_duration_seconds_sum{ instance=\"$instance\" }[5m])",
+          "expr": "sum(rate(traefik_service_request_duration_seconds_sum{ instance=\"$instance\" }[5m])) by(service)",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 2,
           "legendFormat": "{{ service }}",
+          "range": true,
           "refId": "A"
         }
       ],
@@ -931,6 +1001,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -964,7 +1038,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1023,6 +1097,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "decimals": 0,
       "fieldConfig": {
         "defaults": {
@@ -1061,7 +1139,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1121,6 +1199,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -1158,7 +1240,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1217,6 +1299,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "fieldConfig": {
         "defaults": {
           "links": []
@@ -1254,7 +1340,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1313,6 +1399,10 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
       "decimals": 0,
       "fieldConfig": {
         "defaults": {
@@ -1349,7 +1439,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "8.4.4",
+      "pluginVersion": "10.0.2",
       "pointradius": 5,
       "points": false,
       "renderer": "flot",
@@ -1413,13 +1503,14 @@
     }
   ],
   "refresh": "",
-  "schemaVersion": 35,
+  "schemaVersion": 38,
   "style": "dark",
   "tags": [
     "traefik",
     "load-balancer",
     "docker",
-    "prometheus"
+    "prometheus",
+    "menu"
   ],
   "templating": {
     "list": [
@@ -1455,7 +1546,7 @@
       },
       {
         "current": {
-          "selected": false,
+          "selected": true,
           "text": [
             "All"
           ],
@@ -1492,7 +1583,7 @@
         "auto_count": 30,
         "auto_min": "10s",
         "current": {
-          "selected": true,
+          "selected": false,
           "text": "5m",
           "value": "5m"
         },
@@ -1562,17 +1653,12 @@
           }
         ],
         "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
-        "queryValue": "5m",
+        "queryValue": "",
         "refresh": 2,
         "skipUrlSync": false,
         "type": "interval"
       },
       {
-        "current": {
-          "selected": true,
-          "text": "demo.local-it.cloud:8082",
-          "value": "demo.local-it.cloud:8082"
-        },
         "datasource": {
           "type": "prometheus",
           "uid": "PBFA97CFB590B2093"
@@ -1586,12 +1672,12 @@
         "options": [],
         "query": {
           "query": "label_values(instance)",
-          "refId": "StandardVariableQuery"
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
-        "regex": ".*8082.*",
+        "regex": ".*8082",
         "skipUrlSync": false,
-        "sort": 0,
+        "sort": 1,
         "tagValuesQuery": "",
         "tagsQuery": "",
         "type": "query",
@@ -1600,7 +1686,7 @@
     ]
   },
   "time": {
-    "from": "now-15m",
+    "from": "now-2d",
     "to": "now"
   },
   "timepicker": {
@@ -1629,8 +1715,8 @@
     ]
   },
   "timezone": "",
-  "title": "Traefik 2",
+  "title": "Traefik Reverse Proxy",
   "uid": "3ipsWfViz",
-  "version": 5,
+  "version": 9,
   "weekStart": ""
-}
+}

grafana_custom.ini

@@ -10,6 +10,7 @@ auto_assign_org_role = Admin
 {{ if eq (env "OIDC_ENABLED") "1" }}
 [auth]
 disable_login_form = true
+oauth_allow_insecure_email_lookup=true # https://github.com/grafana/grafana/issues/70203
 
 [auth.generic_oauth]
 enabled = true
@@ -18,6 +19,7 @@ name = oauth
 icon = signin
 tls_skip_verify_insecure = false
 allow_sign_up = true
+auto_login = true
 client_id = {{ env "OIDC_CLIENT_ID" }}
 client_secret = {{ secret "grafana_oidc_client_secret" }}
 auth_url = {{ env "OIDC_AUTH_URL" }}
@@ -28,6 +30,9 @@ api_url = {{ env "OIDC_API_URL" }}
 enabled = false
 {{ end }}
 
-
 [plugins]
-enable_alpha = true
+enable_alpha = true
+
+[database]
+type = sqlite3
+wal = true

loki.yml.tmpl

@@ -114,6 +114,14 @@ limits_config:
   reject_old_samples: true
   reject_old_samples_max_age: 168h
   retention_period: {{ env "LOKI_RETENTION_PERIOD" }}
+  split_queries_by_interval: 24h
+  max_query_parallelism: 100
+
+query_scheduler:
+  max_outstanding_requests_per_tenant: 4096
+
+frontend:
+  max_outstanding_per_tenant: 4096
 
 chunk_store_config:
   max_look_back_period: 0s

prometheus.yml.tmpl

@@ -17,4 +17,4 @@ scrape_configs:
         - /prometheus/scrape_configs/*.yml
     basic_auth:
       username: admin
-      password: {{ secret "basic_auth_admin_password" }}
+      password: {{ secret "basic_auth" }}

promtail.yml.tmpl

@@ -9,7 +9,9 @@ clients:
   - url: {{ env "LOKI_PUSH_URL" }}
     basic_auth:
       username: admin
-      password: {{ secret "basic_auth_admin_password" }}
+      password: {{ secret "basic_auth" }}
+    external_labels:
+      hostname: {{ env "DOMAIN" }}
 
 scrape_configs:
 - job_name: system
@@ -32,4 +34,4 @@ scrape_configs:
     - source_labels: ['__meta_docker_container_label_com_docker_stack_namespace']
       target_label: "stack_namespace"
     - source_labels: ['__meta_docker_container_label_com_docker_swarm_service_name']
-      target_label: "service_name"
+      target_label: "service_name"

release/1.0.0+v1.7.0

@@ -0,0 +1,4 @@
+Breakng change: secret `basic_auth_admin_password` was renamed to `basic_auth`. Insert the secret before upgrading. And change the env BASIC_AUTH_ADMIN_PASSWORD to BASIC_AUTH
+
+abra app secret insert monitoring.example.com basic_auth v1 $(abra app run monitoring.example.com promtail cat /var/run/secrets/basic_auth_admin_password)
+sed -i ~/.abra/servers/example.com/monitoring.example.com.env -e 's/BASIC_AUTH_ADMIN_PASSWORD/BASIC_AUTH/'

release/1.4.0+v1.8.1

@@ -0,0 +1 @@
+Adds an optional GRAFANA_DOMAIN

release/1.5.0+v1.8.1

@@ -0,0 +1 @@
+Adds an optional matrix contact point for grafana

release/1.6.0+v1.8.1

@@ -0,0 +1 @@
+Adds option to expose ports for node and cadvisor service