add file

.env.sample

@@ -44,10 +44,10 @@ ENABLE_BACKUPS=true
 ## Grafana
 #
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana.yml"
-# GF_SERVER_ROOT_URL=https://monitoring.example.com
+## GRAFANA_DOMAIN needs to be set. change it for a different domain
+#GRAFANA_DOMAIN=$DOMAIN
+# GF_SERVER_ROOT_URL=https://${GRAFANA_DOMAIN}
 # SECRET_GRAFANA_ADMIN_PASSWORD_VERSION=v1
-## Seperate domain for Grafana
-#GRAFANA_DOMAIN=grafana.example.com
 #
 ## Single-Sign-On with OIDC
 # COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
@@ -85,3 +85,5 @@ ENABLE_BACKUPS=true
 #ALERT_BACKUP_NOT_SUCCESSFULL_ENABLED=true
 #ALERT_NODE_DISK_SPACE_ENABLED=true
 #ALERT_NODE_MEMORY_USAGE_ENABLED=true
+#ALERT_RESTIC_CHECK_FAILED_ENABLED=true
+#ALERT_RESTIC_OUTDATED_BACKUP_ENABLED=true

abra.sh

@@ -7,16 +7,17 @@ export GRAFANA_TRAEFIK_DASHBOARD_JSON_VERSION=v2
 export GRAFANA_BACKUP_DASHBOARD_JSON_VERSION=v1
 export GRAFANA_CUSTOM_INI_VERSION=v4
 export PROMTAIL_YML_VERSION=v3
-export LOKI_YML_VERSION=v3
+export LOKI_YML_VERSION=v2
 export PROMETHEUS_YML_VERSION=v2
 export MATRIX_ALERTMANAGER_CONFIG_VERSION=e
 export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=a
 export GRAFANA_ALERTS_NODE_VERSION=v1c
+export GRAFANA_ALERTS_RESTIC_VERSION=v2
 
 # creates a default prometheus scrape config for a given node
 add_node(){
   name=$1
-  add_domain "$name" "metrics.traefik.$name"
+  add_domain "$name" "$name:8082"
   add_domain "$name" "node.monitoring.$name"
   add_domain "$name" "cadvisor.monitoring.$name"
   cat "/prometheus/scrape_configs/$name.yml"

alerts/restic.yml.tmpl

@@ -0,0 +1,191 @@
+apiVersion: 1
+
+deleteRules:
+  {{ if ne (env "ALERT_RESTIC_CHECK_FAILED_ENABLED") "true" }}
+  - orgId: 1
+    uid: ffglj6egxy8e8c
+  {{ end }}
+  {{ if ne (env "ALERT_RESTIC_OUTDATED_BACKUP_ENABLED") "true" }}
+  - orgId: 1
+    uid: ffgljntkp9ce8b
+  {{ end }}
+
+groups:
+    - orgId: 1
+      name: restic
+      folder: restic
+      interval: 5m
+      rules:
+      {{ if eq (env "ALERT_RESTIC_CHECK_FAILED_ENABLED") "true" }}
+        - uid: ffglj6egxy8e8c
+          title: Restic Check Failed
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: prometheus
+              model:
+                disableTextWrap: false
+                editorMode: builder
+                expr: restic_check_success
+                fullMetaSearch: false
+                includeNullMetadata: true
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: A
+                useBackend: false
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params: []
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - B
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 1
+                            - 0
+                        type: lt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          noDataState: Alerting
+          execErrState: Error
+          for: 5m
+          annotations: {}
+          labels: {}
+          isPaused: false
+        {{ end }}
+        {{ if eq (env "ALERT_RESTIC_OUTDATED_BACKUP_ENABLED") "true" }}
+        - uid: ffgljntkp9ce8b
+          title: Restic Outdated Backup
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: prometheus
+              model:
+                disableTextWrap: false
+                editorMode: builder
+                expr: time() - max by(instance) (restic_backup_timestamp)
+                fullMetaSearch: false
+                includeNullMetadata: true
+                instant: true
+                intervalMs: 1000
+                legendFormat: __auto
+                maxDataPoints: 43200
+                range: false
+                refId: A
+                useBackend: false
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params: []
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - B
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: A
+                intervalMs: 1000
+                maxDataPoints: 43200
+                reducer: last
+                refId: B
+                type: reduce
+            - refId: C
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 93600
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params:
+                            - C
+                      reducer:
+                        params: []
+                        type: last
+                      type: query
+                datasource:
+                    type: __expr__
+                    uid: __expr__
+                expression: B
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: threshold
+          noDataState: NoData
+          execErrState: Error
+          for: 5m
+          annotations: {}
+          labels: {}
+          isPaused: false
+        {{ end }}

compose.grafana.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   grafana:
-    image: grafana/grafana:12.4.0
+    image: grafana/grafana:10.4.14
     volumes:
       - grafana-data:/var/lib/grafana:rw
     secrets:
@@ -24,6 +24,8 @@ services:
         target: /var/lib/grafana/dashboards/backup.json
       - source: gf_alerts_node
         target: /etc/grafana/provisioning/alerting/node.yml
+      - source: gf_alerts_restic
+        target: /etc/grafana/provisioning/alerting/restic.yml
     networks:
       - proxy
       - internal
@@ -34,12 +36,15 @@ services:
       - GF_INSTALL_PLUGINS
       - ALERT_NODE_DISK_SPACE_ENABLED
       - ALERT_NODE_MEMORY_USAGE_ENABLED
+      - ALERT_RESTIC_CHECK_FAILED_ENABLED
+      - ALERT_RESTIC_OUTDATED_BACKUP_ENABLED
+      - DOMAIN
     deploy:
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
-        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls=true"
         - "traefik.http.routers.${STACK_NAME}-grafana.tls.certresolver=${LETS_ENCRYPT_ENV}"
@@ -77,6 +82,10 @@ configs:
     template_driver: golang
     name: ${STACK_NAME}_gf_alerts_node_${GRAFANA_ALERTS_NODE_VERSION}
     file: alerts/node.yml.tmpl
+  gf_alerts_restic:
+    template_driver: golang
+    name: ${STACK_NAME}_gf_alerts_restiv_${GRAFANA_ALERTS_RESTIC_VERSION}
+    file: alerts/restic.yml.tmpl
 
 volumes:
   grafana-data:

compose.loki.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   loki:
-    image: grafana/loki:3.6.7
+    image: grafana/loki:2.9.11
     command: -config.file=/etc/loki/local-config.yaml
     networks:
       - proxy

compose.matrix-alertmanager-receiver.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   matrix-alertmanager-receiver:
-    image: metio/matrix-alertmanager-receiver:2026.2.25
+    image: metio/matrix-alertmanager-receiver:2025.2.9
     secrets:
       - matrix_access_token
     configs:

compose.prometheus.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   prometheus:
-    image: prom/prometheus:v3.10.0
+    image: prom/prometheus:v2.55.1
     secrets:
       - basic_auth
     volumes:
@@ -39,3 +39,8 @@ configs:
 
 volumes:
   prometheus-data:
+
+secrets:
+  basic_auth:
+    external: true
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.promtail.yml

@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   promtail:
-    image: grafana/promtail:3.6.7
+    image: grafana/promtail:2.9.11
     volumes:
       - /var/log:/var/log:ro
       - /var/run/docker.sock:/var/run/docker.sock
@@ -27,4 +27,4 @@ configs:
 secrets:
   basic_auth:
     external: true
-    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}
+    name: ${STACK_NAME}_basic_auth_${SECRET_BASIC_AUTH_VERSION}

compose.pushgateway.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   pushgateway:
-    image: prom/pushgateway:v1.11.2
+    image: prom/pushgateway:v1.10.0
     command:
       - '--web.listen-address=:9191'
       - '--push.disable-consistency-check'

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: prom/node-exporter:v1.10.2
+    image: prom/node-exporter:v1.8.1
     user: root
     environment:
       - NODE_ID={{.Node.ID}}
@@ -43,7 +43,7 @@ services:
         - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT}"
 
   cadvisor:
-    image: gcr.io/cadvisor/cadvisor:v0.55.1
+    image: gcr.io/cadvisor/cadvisor:v0.49.2
     command:
       - "-logtostderr"
       - "--enable_metrics=cpu,cpuLoad,disk,diskIO,process,memory,network"

grafana-dashboards.yml

@@ -11,13 +11,3 @@ providers:
     options:
       path: /var/lib/grafana/dashboards
       foldersFromFilesStructure: true
-  - name: 'default-alert-provider'
-    orgId: 1
-    folder: 'default-alerts'
-    type: file
-    disableDeletion: false
-    updateIntervalSeconds: 10
-    allowUiUpdates: true
-    options:
-      path: /var/lib/grafana/alerts
-      foldersFromFilesStructure: true

grafana-datasources.yml

@@ -3,6 +3,7 @@ apiVersion: 1
 datasources:
   - name: Prometheus
     type: prometheus
+    uid: prometheus
     access: proxy
     orgId: 1
     url: http://prometheus:9090
@@ -10,6 +11,7 @@ datasources:
     editable: false
   - name: Loki
     type: loki
+    uid: loki
     access: proxy
     orgId: 1
     url: http://loki:3100

loki.yml.tmpl

@@ -34,6 +34,7 @@ ingester:
   max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
   chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
   chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
   wal:
     dir: "/tmp/wal"
 
@@ -52,7 +53,7 @@ schema_config:
     - from: 2020-10-24
       store: boltdb-shipper
       object_store: filesystem
-      schema: v13
+      schema: v11
       index:
         prefix: index_
         period: 24h
@@ -62,6 +63,7 @@ storage_config:
     active_index_directory: /loki/boltdb-shipper-active
     cache_location: /loki/boltdb-shipper-cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
+    shared_store: filesystem
   filesystem:
     directory: /loki/chunks
 {{ end }}
@@ -70,6 +72,7 @@ schema_config:
   configs:
   - from: 2020-11-25
     store: boltdb-shipper
+    object_store: aws
     schema: v11
     index:
       prefix: index_
@@ -100,24 +103,19 @@ storage_config:
 
 compactor:
   working_directory: /loki/boltdb-shipper-compactor
+  shared_store: filesystem
   compaction_interval: 10m
   retention_enabled: true
   retention_delete_delay: 2h
   retention_delete_worker_count: 150
-{{ if eq (env "LOKI_STORAGE_FILESYSTEM") "1" }}
-  delete_request_store: filesystem
-{{ end }}
-{{ if eq (env "LOKI_STORAGE_S3") "1" }}
-  delete_request_store: aws
-{{ end }}
 
 limits_config:
+  enforce_metric_name: false
   reject_old_samples: true
   reject_old_samples_max_age: 168h
   retention_period: {{ env "LOKI_RETENTION_PERIOD" }}
   split_queries_by_interval: 24h
   max_query_parallelism: 100
-  allow_structured_metadata: false
 
 query_scheduler:
   max_outstanding_requests_per_tenant: 4096
@@ -125,6 +123,9 @@ query_scheduler:
 frontend:
   max_outstanding_per_tenant: 4096
 
+chunk_store_config:
+  max_look_back_period: 0s
+
 table_manager:
   retention_deletes_enabled: false
   retention_period: 0s

release/next

@@ -5,5 +5,3 @@ COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-oidc.yml"
 2. SMTP was moved into a seperate compose file. If you have smtp configured you need to add the following line to you .env file:
 
 COMPOSE_FILE="$COMPOSE_FILE:compose.grafana-smtp.yml"
-
-3. The scape-config.example.yml file and add_node() command were updated to use a secure endpoint for the traefik metrics instead http. This requires an updated Traefik recipe that publishes the metrics on https.

scrape-config.example.yml

@@ -1,4 +1,4 @@
 - targets:
-  - 'metrics.traefik.example.org'
+  - 'example.org:8082'
   - 'node.monitoring.example.org'
   - 'cadvisor.monitoring.example.org'