Files
simon f48aa64193 make disk usage per container metric optional (#29)
Following the discussion from the matrix channel, I made the disk usage metrics optional, as it currently causes ~35% of cpu usage on our system.

[Old cadvisor-config](02b01e5c23/compose.yml) from this repo had a `- "--housekeeping_interval=120s"` option enabled which seems currently not configurable for alloy - maybe this allows us reenabling disk metrics at a later point again.

dropped from 35% to 5% when removing `disk` from
`  enabled_metrics = ["cpu", "cpuLoad", "diskIO", "memory", "network"]` for cadvisor.

go profiling:

```
File: alloy
Build ID: de7ba8978f6753c25c13f6886ab3b896d3b05d3f
Type: cpu
Time: Sep 7, 2026 at 12:53pm (CEST)
Duration: 120s, Total samples = 37.23s (31.02%)
Showing nodes accounting for 32.20s, 86.49% of 37.23s total
Dropped 790 nodes (cum <= 0.19s)
      flat  flat%   sum%        cum   cum%
    28.72s 77.14% 77.14%     28.72s 77.14%  internal/runtime/syscall/linux.Syscall6
     0.47s  1.26% 78.40%      0.48s  1.29%  internal/filepathlite.(*lazybuf).append (inline)
     0.30s  0.81% 79.21%      0.94s  2.52%  internal/filepathlite.Clean
     0.26s   0.7% 79.91%      0.26s   0.7%  runtime.nextFreeFast (inline)
     0.23s  0.62% 80.53%      0.23s  0.62%  runtime.futex
     0.21s  0.56% 81.09%      0.21s  0.56%  runtime.memclrNoHeapPointers
     0.21s  0.56% 81.65%      0.21s  0.56%  runtime.memmove
     0.19s  0.51% 82.16%      6.19s 16.63%  os.(*File).readdir
     0.17s  0.46% 82.62%     31.24s 83.91%  path/filepath.walk
     0.12s  0.32% 82.94%      0.20s  0.54%  runtime.exitsyscall
     0.08s  0.21% 83.16%      0.24s  0.64%  runtime.scanObject
     0.08s  0.21% 83.37%      0.24s  0.64%  runtime.sweepone
     0.08s  0.21% 83.59%      0.29s  0.78%  slices.pdqsortOrdered[go.shape.string]
     0.07s  0.19% 83.78%      0.27s  0.73%  runtime.makeslicecopy
     0.06s  0.16% 83.94%     28.48s 76.50%  syscall.RawSyscall6
     0.05s  0.13% 84.07%      0.27s  0.73%  github.com/google/cadvisor/fs.GetDirUsage.func1
     0.05s  0.13% 84.21%     19.99s 53.69%  os.lstatNolog
     0.05s  0.13% 84.34%      1.26s  3.38%  runtime.mallocgc
     0.05s  0.13% 84.47%      0.43s  1.15%  runtime.mallocgcSmallNoscan
     0.04s  0.11% 84.58%      0.46s  1.24%  runtime.newobject
     0.04s  0.11% 84.69%      0.19s  0.51%  runtime.selectgo
     0.04s  0.11% 84.80%      1.18s  3.17%  runtime.systemstack
```

Reviewed-on: coop-cloud/monitoring-ng#29
Reviewed-by: Danny Groenewegen <247+dannygroenewegen@noreply.git.coopcloud.tech>
Co-authored-by: Simon <s.thiessen@local-it.org>
2026-09-08 14:25:47 +00:00

109 lines
3.9 KiB
Bash

export GF_DATASOURCES_VERSION=v1
export GF_DASHBOARDS_VERSION=v3
export GF_SWARM_DASH_VERSION=v3
export GF_STACKS_DASH_VERSION=v3
export GF_TRAEFIK_DASH_VERSION=v3
export GF_BACKUP_DASH_VERSION=v1
export GF_CUSTOM_INI_VERSION=v4
export LOKI_YML_VERSION=v3
export PROMETHEUS_YML_VERSION=v2
export MATRIX_ALERTMANAGER_CONFIG_VERSION=v1
export MATRIX_ALERTMANAGER_ENTRYPOINT_VERSION=v1
export GF_ALERTS_NODE_VERSION=v3
export CONFIG_ALLOY_VERSION=v2
# migrates secrets from old names to new names by reading values from the
# running containers on the server and re-inserting them under the new names.
# preview changes: abra app cmd --local <app> migrate_secret_names
# execute changes: abra app cmd --local <app> migrate_secret_names execute
migrate_secret_names() {
if ! command -v jq &> /dev/null; then
echo "jq is required on your local machine to migrate secret names"
echo "It could not be found in your PATH, please install jq to proceed."
echo "For example: On a debian/ubuntu system, run `apt install jq`"
exit 1
fi
# Hardcoded migration mappings: old_secret_name|new_secret_name
MIGRATIONS="
grafana_admin_password|gf_adminpasswd
grafana_smtp_password|gf_smtp_pass
grafana_oidc_client_secret|gf_oidc_secret
matrix_access_token|matrix_token
loki_aws_secret_access_key|loki_aws_key
"
# Determine which server the app is deployed on
SERVER=$(abra app ls -m | jq -r --arg domain "$APP_NAME" '[.[].apps[] | select(.domain == $domain) | .server] | first' 2>/dev/null)
if [ -z "$SERVER" ]; then
echo "Error: could not determine server for app '$APP_NAME'"
exit 1
fi
# Build a lookup table of all secrets currently mounted in this stack.
# Each line: <secretID> <containerID> <secretName>
LOOKUP=$(ssh "$SERVER" "
docker stack services ${STACK_NAME} --format '{{.Name}}' | while read svc; do
CID=\$(docker ps --no-trunc -q --filter \"name=\${svc}\" | head -1)
docker service inspect \"\$svc\" --format '{{json .Spec.TaskTemplate.ContainerSpec.Secrets}}' | \
jq -r --arg cid \"\$CID\" '.[]? | .SecretID + \" \" + \$cid + \" \" + .SecretName'
done | sort -k3 -r
" 2>/dev/null)
echo "Secret migration plan for: $APP_NAME (server: $SERVER)"
echo ""
printf " %-24s %-8s %s\n" "OLD NAME" "FOUND" "ACTION"
printf " %-24s %-8s %s\n" "--------" "-----" "------"
# Check each old name against the lookup table and display the plan
ANY_FOUND=false
while IFS='|' read -r OLD_NAME NEW_NAME; do
[ -z "$OLD_NAME" ] && continue
MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
if [ -n "$MATCH" ]; then
printf " %-24s %-8s %s\n" "$OLD_NAME" "yes" "recreate as '$NEW_NAME' version V1"
ANY_FOUND=true
else
printf " %-24s %-8s %s\n" "$OLD_NAME" "no" "nothing (not found on server)"
fi
done <<< "$MIGRATIONS"
echo ""
if [ "$ANY_FOUND" = false ]; then
echo "No old secrets found on server. Nothing to migrate."
return 0
fi
if [ "$1" != "execute" ]; then
echo "To apply the above changes, run:"
echo " abra app cmd --local $APP_NAME migrate_secret_names execute"
return 0
fi
# read each found secret from its container and re-insert with the new name
while IFS='|' read -r OLD_NAME NEW_NAME; do
[ -z "$OLD_NAME" ] && continue
MATCH=$(echo "$LOOKUP" | grep " ${STACK_NAME}_${OLD_NAME}_" | head -1)
[ -z "$MATCH" ] && continue
SECRET_ID=$(echo "$MATCH" | awk '{print $1}')
CID=$(echo "$MATCH" | awk '{print $2}')
SECRET_VALUE=$(ssh "$SERVER" "cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null || sudo cat /var/lib/docker/containers/${CID}/mounts/secrets/${SECRET_ID} 2>/dev/null")
if [ -z "$SECRET_VALUE" ]; then
echo "Error: could not read value for '$OLD_NAME', skipping"
continue
fi
echo "Migrating: '$OLD_NAME' -> '$NEW_NAME' (v1)"
printf '%s' "$SECRET_VALUE" | abra app secret insert -C "$APP_NAME" "$NEW_NAME" v1
done <<< "$MIGRATIONS"
echo ""
echo "Done."
}