forked from coop-cloud/monitoring-ng
Following the discussion from the matrix channel, I made the disk usage metrics optional, as it currently causes ~35% of cpu usage on our system.
[Old cadvisor-config](02b01e5c23/compose.yml) from this repo had a `- "--housekeeping_interval=120s"` option enabled which seems currently not configurable for alloy - maybe this allows us reenabling disk metrics at a later point again.
dropped from 35% to 5% when removing `disk` from
` enabled_metrics = ["cpu", "cpuLoad", "diskIO", "memory", "network"]` for cadvisor.
go profiling:
```
File: alloy
Build ID: de7ba8978f6753c25c13f6886ab3b896d3b05d3f
Type: cpu
Time: Sep 7, 2026 at 12:53pm (CEST)
Duration: 120s, Total samples = 37.23s (31.02%)
Showing nodes accounting for 32.20s, 86.49% of 37.23s total
Dropped 790 nodes (cum <= 0.19s)
flat flat% sum% cum cum%
28.72s 77.14% 77.14% 28.72s 77.14% internal/runtime/syscall/linux.Syscall6
0.47s 1.26% 78.40% 0.48s 1.29% internal/filepathlite.(*lazybuf).append (inline)
0.30s 0.81% 79.21% 0.94s 2.52% internal/filepathlite.Clean
0.26s 0.7% 79.91% 0.26s 0.7% runtime.nextFreeFast (inline)
0.23s 0.62% 80.53% 0.23s 0.62% runtime.futex
0.21s 0.56% 81.09% 0.21s 0.56% runtime.memclrNoHeapPointers
0.21s 0.56% 81.65% 0.21s 0.56% runtime.memmove
0.19s 0.51% 82.16% 6.19s 16.63% os.(*File).readdir
0.17s 0.46% 82.62% 31.24s 83.91% path/filepath.walk
0.12s 0.32% 82.94% 0.20s 0.54% runtime.exitsyscall
0.08s 0.21% 83.16% 0.24s 0.64% runtime.scanObject
0.08s 0.21% 83.37% 0.24s 0.64% runtime.sweepone
0.08s 0.21% 83.59% 0.29s 0.78% slices.pdqsortOrdered[go.shape.string]
0.07s 0.19% 83.78% 0.27s 0.73% runtime.makeslicecopy
0.06s 0.16% 83.94% 28.48s 76.50% syscall.RawSyscall6
0.05s 0.13% 84.07% 0.27s 0.73% github.com/google/cadvisor/fs.GetDirUsage.func1
0.05s 0.13% 84.21% 19.99s 53.69% os.lstatNolog
0.05s 0.13% 84.34% 1.26s 3.38% runtime.mallocgc
0.05s 0.13% 84.47% 0.43s 1.15% runtime.mallocgcSmallNoscan
0.04s 0.11% 84.58% 0.46s 1.24% runtime.newobject
0.04s 0.11% 84.69% 0.19s 0.51% runtime.selectgo
0.04s 0.11% 84.80% 1.18s 3.17% runtime.systemstack
```
Reviewed-on: coop-cloud/monitoring-ng#29
Reviewed-by: Danny Groenewegen <247+dannygroenewegen@noreply.git.coopcloud.tech>
Co-authored-by: Simon <s.thiessen@local-it.org>
109 lines
3.9 KiB
Bash
109 lines
3.9 KiB
Bash
export GF_DATASOURCES_VERSION=v1
|
|
export GF_DASHBOARDS_VERSION=v3
|
|
export GF_SWARM_DASH_VERSION=v3
|
|
export GF_STACKS_DASH_VERSION=v3
|
|
export GF_TRAEFIK_DASH_VERSION=v3
|
|
export GF_BACKUP_DASH_VERSION=v1
|
|
export GF_CUSTOM_INI_VERSION=v4
|
|
export LOKI_YML_VERSION=v3
|
|
export PROMETHEUS_YML_VERSION=v2
|
|
export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
|
|
export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
|
|
export GF_ALERTS_NODE_VERSION=v3
|
|
export CONFIG_ALLOY_VERSION=v2
|
|
|
|
# migrates secrets from old names to new names by reading values from the
|
|
# running containers on the server and re-inserting them under the new names.
|
|
# preview changes: abra app cmd --local <app> migrate_secret_names
|
|
# execute changes: abra app cmd --local <app> migrate_secret_names execute
|
|
migrate_secret_names() {
|
|
if ! command -v jq &> /dev/null; then
|
|
echo "jq is required on your local machine to migrate secret names"
|
|
echo "It could not be found in your PATH, please install jq to proceed."
|
|
echo "For example: On a debian/ubuntu system, run `apt install jq`"
|
|
exit 1
|
|
fi
|
|
|
|
# Hardcoded migration mappings: old_secret_name|new_secret_name
|
|
MIGRATIONS="
|
|
grafana_admin_password|gf_adminpasswd
|
|
grafana_smtp_password|gf_smtp_pass
|
|
grafana_oidc_client_secret|gf_oidc_secret
|
|
matrix_access_token|matrix_token
|
|
loki_aws_secret_access_key|loki_aws_key
|
|
"
|
|
|
|
# Determine which server the app is deployed on
|
|
SERVER=$(abra app ls -m | jq -r --arg domain "$APP_NAME" '[.[].apps[] | select(.domain == $domain) | .server] | first' 2>/dev/null)
|
|
|
|
if [ -z "$SERVER" ]; then
|
|
echo "Error: could not determine server for app '$APP_NAME'"
|
|
exit 1
|
|
fi
|
|
|
|
# Build a lookup table of all secrets currently mounted in this stack.
|
|
# Each line: <secretID> <containerID> <secretName>
|
|
LOOKUP=$(ssh "$SERVER" "
|
|
docker stack services ${STACK_NAME} --format '{{.Name}}' | while read svc; do
|
|
CID=\$(docker ps --no-trunc -q --filter \"name=\${svc}\" | head -1)
|
|
docker service inspect \"\$svc\" --format '{{json .Spec.TaskTemplate.ContainerSpec.Secrets}}' | \
|
|
jq -r --arg cid \"\$CID\" '.[]? | .SecretID + \" \" + \$cid + \" \" + .SecretName'
|
|
done | sort -k3 -r
|
|
" 2>/dev/null)
|
|
|
|
echo "Secret migration plan for: $APP_NAME (server: $SERVER)"
|
|
echo ""
|
|
printf " %-24s %-8s %s\n" "OLD NAME" "FOUND" "ACTION"
|
|
printf " %-24s %-8s %s\n" "--------" "-----" "------"
|
|
|
|
# Check each old name against the lookup table and display the plan
|
|
ANY_FOUND=false
|
|
while IFS='|' read -r OLD_NAME NEW_NAME; do
|
|
[ -z "$OLD_NAME" ] && continue
|
|
MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
|
|
if [ -n "$MATCH" ]; then
|
|
printf " %-24s %-8s %s\n" "$OLD_NAME" "yes" "recreate as '$NEW_NAME' version V1"
|
|
ANY_FOUND=true
|
|
else
|
|
printf " %-24s %-8s %s\n" "$OLD_NAME" "no" "nothing (not found on server)"
|
|
fi
|
|
done <<< "$MIGRATIONS"
|
|
|
|
echo ""
|
|
|
|
if [ "$ANY_FOUND" = false ]; then
|
|
echo "No old secrets found on server. Nothing to migrate."
|
|
return 0
|
|
fi
|
|
|
|
if [ "$1" != "execute" ]; then
|
|
echo "To apply the above changes, run:"
|
|
echo " abra app cmd --local $APP_NAME migrate_secret_names execute"
|
|
return 0
|
|
fi
|
|
|
|
# read each found secret from its container and re-insert with the new name
|
|
while IFS='|' read -r OLD_NAME NEW_NAME; do
|
|
[ -z "$OLD_NAME" ] && continue
|
|
|
|
MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
|
|
[ -z "$MATCH" ] && continue
|
|
|
|
SECRET_ID=$(echo "$MATCH" | awk '{print $1}')
|
|
CID=$(echo "$MATCH" | awk '{print $2}')
|
|
SECRET_VALUE=$(ssh "$SERVER" "cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null || sudo cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null")
|
|
|
|
if [ -z "$SECRET_VALUE" ]; then
|
|
echo "Error: could not read value for '$OLD_NAME', skipping"
|
|
continue
|
|
fi
|
|
|
|
echo "Migrating: '$OLD_NAME' -> '$NEW_NAME' (v1)"
|
|
printf '%s' "$SECRET_VALUE" | abra app secret insert -C "$APP_NAME" "$NEW_NAME" v1
|
|
|
|
done <<< "$MIGRATIONS"
|
|
|
|
echo ""
|
|
echo "Done."
|
|
}
|