eCommons/monitoring-ng
0
0
Fork 0
forked from coop-cloud/monitoring-ng
Code Pull Requests Activity

fix: make swarm/stacks dashboards work with old and new metric data

Browse Source 
Replace node_meta-based joins with label_replace() of the instance
label, so old (pull-model) and new (Alloy push) series for the same
host render as one continuous series. Each rewritten query has a TODO
noting the simpler form to use once old data ages out of retention.
This commit is contained in:
dannygroenewegen
2026-08-07 16:57:39 +02:00
parent 0f989c4567
commit 7ab6cc74be
3 changed files with 54 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
alerts/node.yml.tmpl
+1 -1
View File
@@ -29,7 +29,7 @@ groups:
datasourceUid: PBFA97CFB590B2093
model:
editorMode: code
expr: (node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"}) * 100
expr: (node_filesystem_free_bytes{fstype=~"ext4|xfs"} / node_filesystem_size_bytes{fstype=~"ext4|xfs"}) * 100
instant: true
intervalMs: 1000
legendFormat: __auto
grafana-stacks-dashboard.json
+15 -15
View File
@@ -110,7 +110,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "(time() - min(container_start_time_seconds{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}))",
"expr": "(time() - min(container_start_time_seconds{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"})) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: (time() - min(container_start_time_seconds{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}))",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
@@ -215,7 +215,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sort(\n sum(\n rate(\n container_cpu_usage_seconds_total{\n container_label_com_docker_stack_namespace=~\"$stack\", \n instance=~\"cadvisor.monitoring.$instance.*\"}[$interval]\n )\n ) \n by (container_label_com_docker_swarm_service_name)\n)",
"expr": "sort(\n sum(\n rate(\n container_cpu_usage_seconds_total{\n container_label_com_docker_stack_namespace=~\"$stack\", \n instance=~\"((node|cadvisor).)?($instance)\"}[$interval]\n )\n ) \n by (container_label_com_docker_swarm_service_name)\n) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sort(sum(rate(container_cpu_usage_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ container_label_com_docker_swarm_service_name }}",
@@ -283,7 +283,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "count(rate(container_last_seen{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval]))",
"expr": "count(rate(container_last_seen{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: count(rate(container_last_seen{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval]))",
"format": "time_series",
"intervalFactor": 2,
"range": true,
@@ -386,7 +386,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(\n container_memory_rss{\n container_label_com_docker_stack_namespace=~\"$stack\", \n instance=~\"cadvisor.monitoring.$instance.*\"}\n ) \nby (container_label_com_docker_swarm_service_name)",
"expr": "sum(\n container_memory_rss{\n container_label_com_docker_stack_namespace=~\"$stack\", \n instance=~\"((node|cadvisor).)?($instance)\"}\n ) \nby (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(container_memory_rss{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -493,7 +493,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_network_receive_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_network_receive_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_network_receive_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "RX: {{ container_label_com_docker_swarm_service_name }}",
@@ -507,7 +507,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_network_transmit_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name) * -1",
"expr": "sum(rate(container_network_transmit_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) * -1 #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_network_transmit_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name) * -1",
"hide": false,
"legendFormat": "TX: {{container_label_com_docker_swarm_service_name}}",
"range": true,
@@ -634,7 +634,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(container_fs_usage_bytes{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(container_fs_usage_bytes{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(container_fs_usage_bytes{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -742,7 +742,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_fs_usage_bytes{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_fs_usage_bytes{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_fs_usage_bytes{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -865,7 +865,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_fs_io_time_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_fs_io_time_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_fs_io_time_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -973,7 +973,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "container_fs_io_current{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}",
"expr": "container_fs_io_current{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"} #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: container_fs_io_current{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
@@ -1081,7 +1081,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_fs_reads_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_fs_reads_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_fs_reads_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ container_label_com_docker_swarm_service_name }}",
@@ -1188,7 +1188,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_fs_read_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_fs_read_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_fs_read_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ container_label_com_docker_swarm_service_name }}",
@@ -1295,7 +1295,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_fs_writes_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_fs_writes_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_fs_writes_bytes_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ container_label_com_docker_swarm_service_name }}",
@@ -1402,7 +1402,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum(rate(container_fs_write_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"cadvisor.monitoring.$instance.*\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_fs_write_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"((node|cadvisor).)?($instance)\"}[$interval])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_fs_write_seconds_total{container_label_com_docker_stack_namespace=~\"$stack\", instance=~\"$instance\"}[$interval])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{ container_label_com_docker_swarm_service_name }}",
@@ -1579,7 +1579,7 @@
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "/.*cadvisor.monitoring.(?<instance>.*):80/",
"regex": "/^(?:(?:node|cadvisor).)?(.*)$/",
"skipUrlSync": false,
"sort": 0,
"type": "query"
grafana-swarm-dashboard.json
+38 -38
View File
@@ -118,7 +118,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "topk(1, sum((node_time_seconds - node_boot_time_seconds) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name))",
"expr": "topk(1, sum(label_replace(node_time_seconds{instance=~\"((node|cadvisor).)?($node_id)\"} - node_boot_time_seconds{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance)) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: topk(1, sum(node_time_seconds{instance=~\"$node_id\"} - node_boot_time_seconds{instance=~\"$node_id\"}) by (instance))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@@ -198,7 +198,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "count(label_replace(node_uname_info{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: count(node_uname_info{instance=~\"$node_id\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@@ -278,7 +278,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "count(node_cpu_seconds_total{mode=\"idle\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "count(label_replace(node_cpu_seconds_total{mode=\"idle\", instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: count(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node_id\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@@ -363,7 +363,7 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) / count(node_meta * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "sum(label_replace((node_memory_MemAvailable_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} / node_memory_MemTotal_bytes{instance=~\"((node|cadvisor).)?($node_id)\"}) * 100, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) / count(label_replace(node_uname_info{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum((node_memory_MemAvailable_bytes{instance=~\"$node_id\"} / node_memory_MemTotal_bytes{instance=~\"$node_id\"}) * 100) / count(node_uname_info{instance=~\"$node_id\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@@ -429,11 +429,11 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "node_load5 * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}",
"expr": "label_replace(node_load5{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\") #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: node_load5{instance=~\"$node_id\"}",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "load5 {{node_name}}",
"legendFormat": "load5 {{instance}}",
"refId": "A",
"step": 2
}
@@ -536,7 +536,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(node_memory_MemTotal_bytes * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "sum(label_replace(node_memory_MemTotal_bytes{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(node_memory_MemTotal_bytes{instance=~\"$node_id\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@@ -616,7 +616,7 @@
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "sum(node_filesystem_size_bytes{fstype=\"ext4\",mountpoint=~\"(/$)|(/media.*)\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "sum(label_replace(node_filesystem_size_bytes{fstype=~\"ext4|xfs\",mountpoint=~\"(/$)|(/media.*)\", instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(node_filesystem_size_bytes{fstype=~\"ext4|xfs\",mountpoint=~\"(/$)|(/media.*)\", instance=~\"$node_id\"})",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -701,7 +701,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\"}[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) * 100 / count(node_cpu_seconds_total{mode=\"user\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) ",
"expr": "sum(label_replace(irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) * 100 / count(label_replace(node_cpu_seconds_total{mode=\"user\", instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node_id\"}[$interval])) * 100 / count(node_cpu_seconds_total{mode=\"user\", instance=~\"$node_id\"})",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "",
@@ -833,7 +833,7 @@
},
"editorMode": "code",
"exemplar": true,
"expr": "node_filesystem_free_bytes{fstype=\"ext4\"} / node_filesystem_size_bytes{fstype=\"ext4\"} * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100",
"expr": "label_replace(node_filesystem_free_bytes{fstype=~\"ext4|xfs\", instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\") / label_replace(node_filesystem_size_bytes{fstype=~\"ext4|xfs\", instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\") * 100 #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: node_filesystem_free_bytes{fstype=~\"ext4|xfs\", instance=~\"$node_id\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs\", instance=~\"$node_id\"} * 100",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -900,10 +900,10 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) by (node_name))",
"expr": "100 - (avg(label_replace(irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\") * 100) by (instance)) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: 100 - (avg(irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$node_id\"}[$interval]) * 100) by (instance))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{node_name}}",
"legendFormat": "{{instance}}",
"refId": "A",
"step": 2
}
@@ -1045,12 +1045,12 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum((node_memory_MemFree_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(node_memory_MemFree_bytes{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(node_memory_MemFree_bytes{instance=~\"$node_id\"}) by (instance)",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "Free {{node_name}}",
"legendFormat": "Free {{instance}}",
"range": true,
"refId": "free",
"step": 2
@@ -1061,12 +1061,12 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum((node_memory_Cached_bytes + node_memory_Buffers_bytes + node_memory_Slab_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(node_memory_Cached_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} + node_memory_Buffers_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} + node_memory_Slab_bytes{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(node_memory_Cached_bytes{instance=~\"$node_id\"} + node_memory_Buffers_bytes{instance=~\"$node_id\"} + node_memory_Slab_bytes{instance=~\"$node_id\"}) by (instance)",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "cache,buffer,slab {{node_name}}",
"legendFormat": "cache,buffer,slab {{instance}}",
"range": true,
"refId": "cache,buffer,slab",
"step": 2
@@ -1077,12 +1077,12 @@
"uid": "PBFA97CFB590B2093"
},
"editorMode": "code",
"expr": "sum((node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Cached_bytes - node_memory_Buffers_bytes - node_memory_Slab_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(node_memory_MemTotal_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} - node_memory_MemFree_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} - node_memory_Cached_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} - node_memory_Buffers_bytes{instance=~\"((node|cadvisor).)?($node_id)\"} - node_memory_Slab_bytes{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(node_memory_MemTotal_bytes{instance=~\"$node_id\"} - node_memory_MemFree_bytes{instance=~\"$node_id\"} - node_memory_Cached_bytes{instance=~\"$node_id\"} - node_memory_Buffers_bytes{instance=~\"$node_id\"} - node_memory_Slab_bytes{instance=~\"$node_id\"}) by (instance)",
"format": "time_series",
"hide": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "Used {{node_name}}",
"legendFormat": "Used {{instance}}",
"range": true,
"refId": "Used",
"step": 2
@@ -1094,13 +1094,13 @@
},
"editorMode": "code",
"exemplar": false,
"expr": "sum((node_memory_MemTotal_bytes) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(node_memory_MemTotal_bytes{instance=~\"((node|cadvisor).)?($node_id)\"}, \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(node_memory_MemTotal_bytes{instance=~\"$node_id\"}) by (instance)",
"format": "time_series",
"hide": false,
"instant": false,
"interval": "",
"intervalFactor": 2,
"legendFormat": "Total {{node_name}}",
"legendFormat": "Total {{instance}}",
"range": true,
"refId": "total",
"step": 2
@@ -1161,11 +1161,11 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(irate(node_disk_read_bytes_total[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(irate(node_disk_read_bytes_total{instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(irate(node_disk_read_bytes_total{instance=~\"$node_id\"}[$interval])) by (instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "Read {{node_name}}",
"legendFormat": "Read {{instance}}",
"refId": "A",
"step": 2
},
@@ -1174,10 +1174,10 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(irate(node_disk_written_bytes_total[$interval]) * on(instance) group_left(node_name) node_meta{node_id=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(irate(node_disk_written_bytes_total{instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(irate(node_disk_written_bytes_total{instance=~\"$node_id\"}[$interval])) by (instance)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Written {{node_name}}",
"legendFormat": "Written {{instance}}",
"refId": "B",
"step": 2
}
@@ -1264,10 +1264,10 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(irate(node_disk_reads_completed_total[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(irate(node_disk_reads_completed_total{instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(irate(node_disk_reads_completed_total{instance=~\"$node_id\"}[$interval])) by (instance)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Reads {{node_name}}",
"legendFormat": "Reads {{instance}}",
"refId": "A",
"step": 2
},
@@ -1276,10 +1276,10 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(irate(node_disk_writes_completed_total[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"}) by (node_name)",
"expr": "sum(label_replace(irate(node_disk_writes_completed_total{instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(irate(node_disk_writes_completed_total{instance=~\"$node_id\"}[$interval])) by (instance)",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "Writes {{node_name}}",
"legendFormat": "Writes {{instance}}",
"refId": "B",
"step": 2
}
@@ -1368,10 +1368,10 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "(avg(irate(node_cpu_seconds_total{mode=\"iowait\"}[$interval]) * on(instance) group_left(node_name) node_meta{node_name=~\"$node_id\"} * 100) by (node_name))",
"expr": "(avg(label_replace(irate(node_cpu_seconds_total{mode=\"iowait\", instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\") * 100) by (instance)) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: (avg(irate(node_cpu_seconds_total{mode=\"iowait\", instance=~\"$node_id\"}[$interval]) * 100) by (instance))",
"format": "time_series",
"intervalFactor": 2,
"legendFormat": "{{node_name}}",
"legendFormat": "{{instance}}",
"refId": "A",
"step": 2
}
@@ -1464,7 +1464,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(rate(container_last_seen[5m]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"${node_id}\"}) by (container_label_com_docker_swarm_service_name)",
"expr": "sum(rate(container_last_seen{instance=~\"((node|cadvisor).)?($node_id)\"}[5m])) by (container_label_com_docker_swarm_service_name) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_last_seen{instance=~\"$node_id\"}[5m])) by (container_label_com_docker_swarm_service_name)",
"format": "time_series",
"hide": false,
"intervalFactor": 10,
@@ -1573,7 +1573,7 @@
"uid": "PBFA97CFB590B2093"
},
"exemplar": true,
"expr": "count(rate(container_last_seen[5m]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"$node_id\"})",
"expr": "count(rate(container_last_seen{instance=~\"((node|cadvisor).)?($node_id)\"}[5m])) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: count(rate(container_last_seen{instance=~\"$node_id\"}[5m]))",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
@@ -1668,11 +1668,11 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "sum(rate(container_network_receive_bytes_total[$interval]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"${node_id}\"}) by (node_name)",
"expr": "sum(label_replace(rate(container_network_receive_bytes_total{instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: sum(rate(container_network_receive_bytes_total{instance=~\"$node_id\"}[$interval])) by (instance)",
"format": "time_series",
"interval": "",
"intervalFactor": 2,
"legendFormat": "IN {{node_name}}",
"legendFormat": "IN {{instance}}",
"refId": "A",
"step": 2
},
@@ -1681,11 +1681,11 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "- sum(rate(container_network_transmit_bytes_total[$interval]) * on(container_label_com_docker_swarm_node_id) group_left(node_name) node_meta{node_name=~\"${node_id}\"}) by (node_name)",
"expr": "- sum(label_replace(rate(container_network_transmit_bytes_total{instance=~\"((node|cadvisor).)?($node_id)\"}[$interval]), \"instance\", \"$1\", \"instance\", \"^(?:node|cadvisor).(.*)$\")) by (instance) #TODO once old cadvisor/node_exporter data has aged out of retention, replace with: - sum(rate(container_network_transmit_bytes_total{instance=~\"$node_id\"}[$interval])) by (instance)",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
"legendFormat": "OUT {{node_name}}",
"legendFormat": "OUT {{instance}}",
"metric": "",
"refId": "B",
"step": 2
@@ -1762,11 +1762,11 @@
"name": "node_id",
"options": [],
"query": {
"query": "node_meta",
"query": "label_values(node_uname_info, instance)",
"refId": "Prometheus-node_id-Variable-Query"
},
"refresh": 1,
"regex": "/node_name=\"([^\"]+)\"/",
"regex": "/^(?:node.)?(.*)$/",
"skipUrlSync": false,
"sort": 0,
"type": "query"