4 Commits

11 changed files with 144 additions and 72 deletions

View File

@ -12,6 +12,9 @@ ENABLE_BACKUPS=true
# COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml" # COMPOSE_FILE="$COMPOSE_FILE:compose.promtail.yml"
# LOKI_PUSH_URL=https://loki.monitoring.example.org/loki/api/v1/push # LOKI_PUSH_URL=https://loki.monitoring.example.org/loki/api/v1/push
## Expose node and cadvisor ports instead of traefik
# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
# Monitoring Server # Monitoring Server
# #
## Prometheus ## Prometheus

View File

@ -36,7 +36,7 @@ Where gathering.org is the node you want to gather metrics from.
SECRET_USERSFILE_VERSION=v1 SECRET_USERSFILE_VERSION=v1
``` ```
- Generate userslist with httpasswd hashed password - Generate userslist with httpasswd hashed password
`abra app secret insert traefik.gathering.org userslist v1 'admin:<hashed-secret>'` `abra app secret insert traefik.gathering.org usersfile v1 'admin:<hashed-secret>'`
make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff... make sure there is no whitespace in between `admin:<hashed-secret>`, it seems to break stuff...
- `abra app deploy -f traefik` - `abra app deploy -f traefik`
1. `abra app new monitoring-ng` 1. `abra app new monitoring-ng`
@ -47,6 +47,13 @@ Where gathering.org is the node you want to gather metrics from.
- cadvisor.monitoring.gathering.org - cadvisor.monitoring.gathering.org
- node.monitoring.gathering.org - node.monitoring.gathering.org
### Expose node and cadvisor via ports instead of traefik
In case you have no traefik running on the machine, you can expose the ports directly by uncommenting the following line:
```
# COMPOSE_FILE="$COMPOSE_FILE:compose.expose-ports.yml"
```
## Setup Metrics Browser ## Setup Metrics Browser
@ -85,7 +92,6 @@ Where gathering.org is the node you want to gather metrics from.
| Cadvisor | traefik basic-auth | cadvisor.monitoring.example.org | | Cadvisor | traefik basic-auth | cadvisor.monitoring.example.org |
| Node Exporter | traefik basic-auth | node.monitoring.example.org | | Node Exporter | traefik basic-auth | node.monitoring.example.org |
### Logging from a docker host to loki server without anything else ### Logging from a docker host to loki server without anything else
``` ```

13
compose.expose-ports.yml Normal file
View File

@ -0,0 +1,13 @@
---
version: "3.8"
services:
app:
ports:
- "9100:9100"
deploy:
cadvisor:
ports:
- "9101:8080"
deploy:

View File

@ -48,6 +48,7 @@ services:
deploy: deploy:
labels: labels:
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000" - "traefik.http.services.${STACK_NAME}-grafana.loadbalancer.server.port=3000"
- "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}-grafana.rule=Host(`${GRAFANA_DOMAIN:-$DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}-grafana.entrypoints=web-secure"

View File

@ -27,6 +27,7 @@ services:
condition: on-failure condition: on-failure
labels: labels:
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-loki.loadbalancer.server.port=3100" - "traefik.http.services.${STACK_NAME}-loki.loadbalancer.server.port=3100"
- "traefik.http.routers.${STACK_NAME}-loki.rule=Host(`loki.${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}-loki.rule=Host(`loki.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-loki.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}-loki.entrypoints=web-secure"

View File

@ -24,12 +24,12 @@ services:
condition: on-failure condition: on-failure
labels: labels:
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-prometheus.loadbalancer.server.port=9090" - "traefik.http.services.${STACK_NAME}-prometheus.loadbalancer.server.port=9090"
- "traefik.http.routers.${STACK_NAME}-prometheus.rule=Host(`prometheus.${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}-prometheus.rule=Host(`prometheus.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}-prometheus.entrypoints=web-secure"
- "traefik.http.routers.${STACK_NAME}-prometheus.tls=true" - "traefik.http.routers.${STACK_NAME}-prometheus.tls=true"
- "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}" - "traefik.http.routers.${STACK_NAME}-prometheus.tls.certresolver=${LETS_ENCRYPT_ENV}"
- "traefik.http.routers.${STACK_NAME}-prometheus.middlewares=basicauth@file"
configs: configs:
prometheus_yml: prometheus_yml:

View File

@ -17,6 +17,7 @@ services:
condition: on-failure condition: on-failure
labels: labels:
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-pushgateway.loadbalancer.server.port=9191" - "traefik.http.services.${STACK_NAME}-pushgateway.loadbalancer.server.port=9191"
- "traefik.http.routers.${STACK_NAME}-pushgateway.rule=Host(`pushgateway.${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}-pushgateway.rule=Host(`pushgateway.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-pushgateway.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}-pushgateway.entrypoints=web-secure"

View File

@ -32,13 +32,14 @@ services:
labels: labels:
- "backupbot.backup=${ENABLE_BACKUPS:-true}" - "backupbot.backup=${ENABLE_BACKUPS:-true}"
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100" - "traefik.http.services.${STACK_NAME}-node.loadbalancer.server.port=9100"
- "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}-node.rule=Host(`node.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}-node.entrypoints=web-secure"
- "traefik.http.routers.${STACK_NAME}-node.tls=true" - "traefik.http.routers.${STACK_NAME}-node.tls=true"
- "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}" - "traefik.http.routers.${STACK_NAME}-node.tls.certresolver=${LETS_ENCRYPT_ENV}"
- "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file" - "traefik.http.routers.${STACK_NAME}-node.middlewares=basicauth@file"
- "coop-cloud.${STACK_NAME}.version=1.5.0+v1.8.1" - "coop-cloud.${STACK_NAME}.version=1.6.0+v1.8.1"
- "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}" - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
cadvisor: cadvisor:
@ -63,6 +64,7 @@ services:
condition: on-failure condition: on-failure
labels: labels:
- "traefik.enable=true" - "traefik.enable=true"
- "traefik.docker.network=proxy"
- "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080" - "traefik.http.services.${STACK_NAME}-cadvisor.loadbalancer.server.port=8080"
- "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}-cadvisor.rule=Host(`cadvisor.${DOMAIN}`)"
- "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}-cadvisor.entrypoints=web-secure"

View File

@ -216,7 +216,7 @@
"datasourceUid": "PBFA97CFB590B2093", "datasourceUid": "PBFA97CFB590B2093",
"model": { "model": {
"editorMode": "code", "editorMode": "code",
"expr": "(node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * 100", "expr": "(node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"}) * 100",
"instant": true, "instant": true,
"intervalMs": 1000, "intervalMs": 1000,
"legendFormat": "__auto", "legendFormat": "__auto",

View File

@ -93,7 +93,6 @@
}, },
"hideTimeOverride": true, "hideTimeOverride": true,
"id": 2, "id": 2,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"colorMode": "value", "colorMode": "value",
@ -107,10 +106,12 @@
"fields": "", "fields": "",
"values": false "values": false
}, },
"showPercentChange": false,
"text": {}, "text": {},
"textMode": "auto" "textMode": "auto",
"wideLayout": true
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -172,7 +173,6 @@
"y": 0 "y": 0
}, },
"id": 1, "id": 1,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"colorMode": "value", "colorMode": "value",
@ -186,10 +186,12 @@
"fields": "", "fields": "",
"values": false "values": false
}, },
"showPercentChange": false,
"text": {}, "text": {},
"textMode": "auto" "textMode": "auto",
"wideLayout": true
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -251,7 +253,6 @@
}, },
"hideTimeOverride": true, "hideTimeOverride": true,
"id": 4, "id": 4,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"colorMode": "value", "colorMode": "value",
@ -265,10 +266,12 @@
"fields": "", "fields": "",
"values": false "values": false
}, },
"showPercentChange": false,
"text": {}, "text": {},
"textMode": "auto" "textMode": "auto",
"wideLayout": true
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -335,9 +338,10 @@
"y": 0 "y": 0
}, },
"id": 8, "id": 8,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "horizontal", "orientation": "horizontal",
"reduceOptions": { "reduceOptions": {
"calcs": [ "calcs": [
@ -348,9 +352,10 @@
}, },
"showThresholdLabels": false, "showThresholdLabels": false,
"showThresholdMarkers": true, "showThresholdMarkers": true,
"sizing": "auto",
"text": {} "text": {}
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -405,13 +410,12 @@
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
"links": [],
"nullPointMode": "null", "nullPointMode": "null",
"options": { "options": {
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"pointradius": 5, "pointradius": 5,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -507,7 +511,6 @@
}, },
"hideTimeOverride": true, "hideTimeOverride": true,
"id": 3, "id": 3,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"colorMode": "value", "colorMode": "value",
@ -521,10 +524,12 @@
"fields": "", "fields": "",
"values": false "values": false
}, },
"showPercentChange": false,
"text": {}, "text": {},
"textMode": "auto" "textMode": "auto",
"wideLayout": true
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -585,7 +590,6 @@
}, },
"hideTimeOverride": true, "hideTimeOverride": true,
"id": 9, "id": 9,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"colorMode": "value", "colorMode": "value",
@ -599,10 +603,12 @@
"fields": "", "fields": "",
"values": false "values": false
}, },
"showPercentChange": false,
"text": {}, "text": {},
"textMode": "auto" "textMode": "auto",
"wideLayout": true
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -671,9 +677,10 @@
}, },
"hideTimeOverride": true, "hideTimeOverride": true,
"id": 11, "id": 11,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "horizontal", "orientation": "horizontal",
"reduceOptions": { "reduceOptions": {
"calcs": [ "calcs": [
@ -684,9 +691,10 @@
}, },
"showThresholdLabels": false, "showThresholdLabels": false,
"showThresholdMarkers": true, "showThresholdMarkers": true,
"sizing": "auto",
"text": {} "text": {}
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -713,7 +721,39 @@
"fieldConfig": { "fieldConfig": {
"defaults": { "defaults": {
"color": { "color": {
"mode": "thresholds" "mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "left",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
}, },
"mappings": [ "mappings": [
{ {
@ -747,33 +787,42 @@
}, },
"unit": "percent" "unit": "percent"
}, },
"overrides": [] "overrides": [
{
"matcher": {
"id": "byType",
"options": "time"
},
"properties": [
{
"id": "custom.axisPlacement",
"value": "hidden"
}
]
}
]
}, },
"gridPos": { "gridPos": {
"h": 4, "h": 6,
"w": 2.6666666666666665, "w": 6,
"x": 0, "x": 0,
"y": 8 "y": 8
}, },
"id": 10, "id": 10,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"maxPerRow": 12,
"options": { "options": {
"colorMode": "value", "legend": {
"graphMode": "area", "calcs": [],
"justifyMode": "auto", "displayMode": "list",
"orientation": "horizontal", "placement": "bottom",
"reduceOptions": { "showLegend": true
"calcs": [
"last"
],
"fields": "",
"values": false
}, },
"textMode": "auto" "tooltip": {
"mode": "single",
"sort": "none"
}
}, },
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"repeat": "node_id", "repeat": "node_id",
"repeatDirection": "h", "repeatDirection": "h",
"targets": [ "targets": [
@ -782,18 +831,20 @@
"type": "prometheus", "type": "prometheus",
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"editorMode": "code",
"exemplar": true, "exemplar": true,
"expr": "sum((node_filesystem_free_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} / node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"}) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})", "expr": "node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
"format": "time_series", "format": "time_series",
"interval": "", "interval": "",
"intervalFactor": 2, "intervalFactor": 2,
"legendFormat": "", "legendFormat": "{{mountpoint}}",
"range": true,
"refId": "A", "refId": "A",
"step": 20 "step": 20
} }
], ],
"title": "Available Disk Space $node_id", "title": "Available Disk Space $node_id",
"type": "stat" "type": "timeseries"
}, },
{ {
"aliasColors": {}, "aliasColors": {},
@ -811,7 +862,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 12 "y": 14
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 14, "id": 14,
@ -830,13 +881,12 @@
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
"links": [],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"options": { "options": {
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"pointradius": 5, "pointradius": 5,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -900,6 +950,7 @@
"mode": "palette-classic" "mode": "palette-classic"
}, },
"custom": { "custom": {
"axisBorderShow": false,
"axisCenteredZero": false, "axisCenteredZero": false,
"axisColorMode": "text", "axisColorMode": "text",
"axisLabel": "", "axisLabel": "",
@ -913,6 +964,7 @@
"tooltip": false, "tooltip": false,
"viz": false "viz": false
}, },
"insertNulls": false,
"lineInterpolation": "linear", "lineInterpolation": "linear",
"lineWidth": 1, "lineWidth": 1,
"pointSize": 5, "pointSize": 5,
@ -961,12 +1013,11 @@
}, },
"gridPos": { "gridPos": {
"h": 7, "h": 7,
"w": 2.6666666666666665, "w": 6,
"x": 0, "x": 0,
"y": 19 "y": 21
}, },
"id": 15, "id": 15,
"links": [],
"maxPerRow": 12, "maxPerRow": 12,
"options": { "options": {
"legend": { "legend": {
@ -1074,7 +1125,7 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 26 "y": 28
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 16, "id": 16,
@ -1091,13 +1142,12 @@
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
"links": [],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"options": { "options": {
"alertThreshold": true "alertThreshold": true
}, },
"percentage": false, "percentage": false,
"pluginVersion": "10.0.2", "pluginVersion": "10.4.14",
"pointradius": 5, "pointradius": 5,
"points": false, "points": false,
"renderer": "flot", "renderer": "flot",
@ -1178,7 +1228,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 0, "x": 0,
"y": 33 "y": 35
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 18, "id": 18,
@ -1195,7 +1245,6 @@
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
"links": [],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"options": { "options": {
"alertThreshold": true "alertThreshold": true
@ -1281,7 +1330,7 @@
"h": 7, "h": 7,
"w": 12, "w": 12,
"x": 12, "x": 12,
"y": 33 "y": 35
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 19, "id": 19,
@ -1300,7 +1349,6 @@
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
"links": [],
"nullPointMode": "null as zero", "nullPointMode": "null as zero",
"options": { "options": {
"alertThreshold": true "alertThreshold": true
@ -1376,7 +1424,7 @@
"h": 7, "h": 7,
"w": 18, "w": 18,
"x": 0, "x": 0,
"y": 40 "y": 42
}, },
"hiddenSeries": false, "hiddenSeries": false,
"id": 12, "id": 12,
@ -1397,7 +1445,6 @@
}, },
"lines": true, "lines": true,
"linewidth": 1, "linewidth": 1,
"links": [],
"nullPointMode": "null", "nullPointMode": "null",
"options": { "options": {
"alertThreshold": true "alertThreshold": true
@ -1499,10 +1546,9 @@
"h": 7, "h": 7,
"w": 6, "w": 6,
"x": 18, "x": 18,
"y": 40 "y": 42
}, },
"id": 7, "id": 7,
"links": [],
"maxDataPoints": 100, "maxDataPoints": 100,
"options": { "options": {
"colorMode": "none", "colorMode": "none",
@ -1600,10 +1646,9 @@
"h": 7, "h": 7,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 47 "y": 49
}, },
"id": 17, "id": 17,
"links": [],
"options": { "options": {
"legend": { "legend": {
"calcs": [], "calcs": [],
@ -1658,7 +1703,7 @@
"h": 9, "h": 9,
"w": 24, "w": 24,
"x": 0, "x": 0,
"y": 54 "y": 56
}, },
"id": 30, "id": 30,
"options": { "options": {
@ -1688,8 +1733,7 @@
} }
], ],
"refresh": "", "refresh": "",
"schemaVersion": 38, "schemaVersion": 39,
"style": "dark",
"tags": [ "tags": [
"swarmprom", "swarmprom",
"prometheus", "prometheus",
@ -1836,6 +1880,6 @@
"timezone": "", "timezone": "",
"title": "Docker Swarm Nodes", "title": "Docker Swarm Nodes",
"uid": "BPlb-Sgik", "uid": "BPlb-Sgik",
"version": 24, "version": 7,
"weekStart": "" "weekStart": ""
} }

1
release/1.6.0+v1.8.1 Normal file
View File

@ -0,0 +1 @@
Adds option to expose ports for node and cadvisor service