From 7e8571306a5e208019a5efa161d76ba4067b307f Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 15:58:33 -0400 Subject: [PATCH 01/11] add secrets --- compose.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compose.yml b/compose.yml index 3d7a3b6..26be3e2 100644 --- a/compose.yml +++ b/compose.yml @@ -11,6 +11,10 @@ services: secrets: - source: rpc_secret mode: 0600 + - source: metrics_token + mode: 0600 + - source: admin_token + mode: 0600 networks: - proxy - internal @@ -49,6 +53,12 @@ secrets: rpc_secret: name: ${STACK_NAME}_rpc_secret_${SECRET_RPC_SECRET_VERSION} external: true + admin_token: + name: ${STACK_NAME}_admin_token_${SECRET_ADMIN_TOKEN_SECRET_VERSION} + external: true + metrics_token: + name: ${STACK_NAME}_metrics_token_${SECRET_METRICS_TOKEN_SECRET_VERSION} + external: true volumes: meta: -- 2.49.0 From 7d1b4ddc771d15afe009ea6cc651bdbf50b3af82 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 15:59:09 -0400 Subject: [PATCH 02/11] add monitoring conditional and secrets to .env --- .env.sample | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env.sample b/.env.sample index 043e3d8..65b5387 100644 --- a/.env.sample +++ b/.env.sample @@ -6,6 +6,10 @@ LETS_ENCRYPT_ENV=production COMPOSE_FILE="compose.yml" SECRET_RPC_SECRET_VERSION=v1 # length=64 charset=hex +SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex +SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex + +MONITORING_ENABLED="false" # Changing the replication factor after initial deployment is not # supported and requires deleting the existing cluster layout metadata. -- 2.49.0 From aeb6675be422c80c1e55373c5ca8cbce179fa484 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 15:59:27 -0400 Subject: [PATCH 03/11] setup conditional for garage.toml, set require metrics token to true --- garage.toml.tmpl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/garage.toml.tmpl b/garage.toml.tmpl index d54cc2f..caeb375 100644 --- a/garage.toml.tmpl +++ b/garage.toml.tmpl @@ -27,3 +27,11 @@ bootstrap_peers = [ s3_region = "garage" api_bind_addr = "[::]:3900" root_domain = ".s3.garage" + +{{ if eq (env "MONITORING_ENABLED") "true" }} +[admin] +api_bind_addr = "[::]:3903" +admin_token_file = "/run/secrets/admin_token" +metrics_require_token = true +metrics_token_file = "/run/secrets/metrics_token" +{{ end }} \ No newline at end of file -- 2.49.0 From ff778eab627c504a2c68e630cac9521b38d24319 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 15:59:40 -0400 Subject: [PATCH 04/11] update readme with basic configuration --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4195d5c..5298a1c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# garage +# Garage > An open-source distributed object storage service tailored for selfhosting at a small-to-medium scale. @@ -52,7 +52,19 @@ You can optionally add this alias to your `.bashrc` (or similar) file to avoid h ### Garage Quick Start Guide Once `garage status` works, you can follow the guide here: https://garagehq.deuxfleurs.fr/documentation/quick-start/#checking-that-garage-runs-correctly +## Monitoring +By default monitoring is disabled and must be enabled in the config. + +To enable set `MONITORING` to `true`. + +> If you've deployed garage before ver `0.0.2+v2.3.0` then you will need to add the following lines to your config: +> ``` +> SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex +>SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex +> +> MONITORING="true" +> ``` ## Backups -- 2.49.0 From 46ba380e45d4ef254763d9b2981753175b580d96 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 16:10:15 -0400 Subject: [PATCH 05/11] use compose_file env instead of a conditional in the .env file. --- .env.sample | 2 +- compose.monitoring.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 compose.monitoring.yml diff --git a/.env.sample b/.env.sample index 65b5387..2656d88 100644 --- a/.env.sample +++ b/.env.sample @@ -9,7 +9,7 @@ SECRET_RPC_SECRET_VERSION=v1 # length=64 charset=hex SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex -MONITORING_ENABLED="false" +#COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml" # Changing the replication factor after initial deployment is not # supported and requires deleting the existing cluster layout metadata. diff --git a/compose.monitoring.yml b/compose.monitoring.yml new file mode 100644 index 0000000..eeea8a2 --- /dev/null +++ b/compose.monitoring.yml @@ -0,0 +1,12 @@ +--- +version: "3.8" + +services: + app: + environment: + - MONITORING_ENABLED=true + deploy: + labels: + - "traefik.tcp.routers.garage-monitoring.rule=HostSNI(`*`)" + - "traefik.tcp.routers.garage-monitoring.entrypoints=garage-monitoring" + - "traefik.tcp.services.garage-monitoring.loadbalancer.server.port=3903" -- 2.49.0 From 3741560b13e2202863bd19d1d6505a68d76b8a1c Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 16:54:30 -0400 Subject: [PATCH 06/11] move secrets to compose.monitoring.yml --- compose.monitoring.yml | 13 +++++++++++++ compose.yml | 10 ---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/compose.monitoring.yml b/compose.monitoring.yml index eeea8a2..4549657 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -5,8 +5,21 @@ services: app: environment: - MONITORING_ENABLED=true + secrets: + - source: metrics_token + mode: 0600 + - source: admin_token + mode: 0600 deploy: labels: - "traefik.tcp.routers.garage-monitoring.rule=HostSNI(`*`)" - "traefik.tcp.routers.garage-monitoring.entrypoints=garage-monitoring" - "traefik.tcp.services.garage-monitoring.loadbalancer.server.port=3903" + +secrets: + admin_token: + name: ${STACK_NAME}_admin_token_${SECRET_ADMIN_TOKEN_SECRET_VERSION} + external: true + metrics_token: + name: ${STACK_NAME}_metrics_token_${SECRET_METRICS_TOKEN_SECRET_VERSION} + external: true diff --git a/compose.yml b/compose.yml index 26be3e2..3d7a3b6 100644 --- a/compose.yml +++ b/compose.yml @@ -11,10 +11,6 @@ services: secrets: - source: rpc_secret mode: 0600 - - source: metrics_token - mode: 0600 - - source: admin_token - mode: 0600 networks: - proxy - internal @@ -53,12 +49,6 @@ secrets: rpc_secret: name: ${STACK_NAME}_rpc_secret_${SECRET_RPC_SECRET_VERSION} external: true - admin_token: - name: ${STACK_NAME}_admin_token_${SECRET_ADMIN_TOKEN_SECRET_VERSION} - external: true - metrics_token: - name: ${STACK_NAME}_metrics_token_${SECRET_METRICS_TOKEN_SECRET_VERSION} - external: true volumes: meta: -- 2.49.0 From baa4167a89e5f80ddf8a417cdf50a6d96a45b752 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 16:58:37 -0400 Subject: [PATCH 07/11] no idea if this actually works, pushing so that I can test later --- .env.sample | 1 + compose.monitoring.yml | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.env.sample b/.env.sample index 2656d88..82154fb 100644 --- a/.env.sample +++ b/.env.sample @@ -1,6 +1,7 @@ TYPE=garage DOMAIN=garage.example.com +MONITORING_DOMAIN=monitoring.garage.example.com LETS_ENCRYPT_ENV=production COMPOSE_FILE="compose.yml" diff --git a/compose.monitoring.yml b/compose.monitoring.yml index 4549657..79ab9e1 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -12,9 +12,10 @@ services: mode: 0600 deploy: labels: - - "traefik.tcp.routers.garage-monitoring.rule=HostSNI(`*`)" - - "traefik.tcp.routers.garage-monitoring.entrypoints=garage-monitoring" - - "traefik.tcp.services.garage-monitoring.loadbalancer.server.port=3903" + - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=3903" + - "traefik.http.routers.${STACK_NAME}.rule=Host(`${MONITORING_DOMAIN}`)" + - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure" + - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}" secrets: admin_token: -- 2.49.0 From 88d46b767e651171661e5adc096700c440d1c4e6 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 8 May 2026 20:54:02 -0400 Subject: [PATCH 08/11] set unique stack name --- compose.monitoring.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compose.monitoring.yml b/compose.monitoring.yml index 79ab9e1..9dc7c91 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -12,10 +12,10 @@ services: mode: 0600 deploy: labels: - - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=3903" - - "traefik.http.routers.${STACK_NAME}.rule=Host(`${MONITORING_DOMAIN}`)" - - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure" - - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.services.${STACK_NAME}-monitoring.loadbalancer.server.port=3903" + - "traefik.http.routers.${STACK_NAME}-monitoring.rule=Host(`${MONITORING_DOMAIN}`)" + - "traefik.http.routers.${STACK_NAME}-monitoring.entrypoints=web-secure" + - "traefik.http.routers.${STACK_NAME}-monitoring.tls.certresolver=${LETS_ENCRYPT_ENV}" secrets: admin_token: -- 2.49.0 From 97e96ed1d3015d035f54b55779af4f9cf2387c58 Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 15 May 2026 15:09:28 -0400 Subject: [PATCH 09/11] move monitoring_enabled to .env file --- .env.sample | 1 + compose.monitoring.yml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index 82154fb..c4a9f25 100644 --- a/.env.sample +++ b/.env.sample @@ -11,6 +11,7 @@ SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex #COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml" +#MONITORING_ENABLED=true # Changing the replication factor after initial deployment is not # supported and requires deleting the existing cluster layout metadata. diff --git a/compose.monitoring.yml b/compose.monitoring.yml index 9dc7c91..0000080 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -3,8 +3,6 @@ version: "3.8" services: app: - environment: - - MONITORING_ENABLED=true secrets: - source: metrics_token mode: 0600 -- 2.49.0 From 04463dffe8f451108bd0e2c69d6594f1c4e6a19d Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 15 May 2026 16:09:13 -0400 Subject: [PATCH 10/11] scrap monitoring domain as it was too much of a headache to configure, use /metrics endpoint which seems fairly standard(?) --- .env.sample | 1 - compose.monitoring.yml | 9 +++++---- compose.yml | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.env.sample b/.env.sample index c4a9f25..d3d4a9c 100644 --- a/.env.sample +++ b/.env.sample @@ -1,7 +1,6 @@ TYPE=garage DOMAIN=garage.example.com -MONITORING_DOMAIN=monitoring.garage.example.com LETS_ENCRYPT_ENV=production COMPOSE_FILE="compose.yml" diff --git a/compose.monitoring.yml b/compose.monitoring.yml index 0000080..ad1ae35 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -10,10 +10,11 @@ services: mode: 0600 deploy: labels: - - "traefik.http.services.${STACK_NAME}-monitoring.loadbalancer.server.port=3903" - - "traefik.http.routers.${STACK_NAME}-monitoring.rule=Host(`${MONITORING_DOMAIN}`)" - - "traefik.http.routers.${STACK_NAME}-monitoring.entrypoints=web-secure" - - "traefik.http.routers.${STACK_NAME}-monitoring.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.routers.${STACK_NAME}-metrics.rule=Host(`${DOMAIN}`) && Path(`/metrics`)" + - "traefik.http.routers.${STACK_NAME}-metrics.entrypoints=web-secure" + - "traefik.http.routers.${STACK_NAME}-metrics.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.routers.${STACK_NAME}-metrics.service=${STACK_NAME}-metrics" + - "traefik.http.services.${STACK_NAME}-metrics.loadbalancer.server.port=3903" secrets: admin_token: diff --git a/compose.yml b/compose.yml index 3d7a3b6..502cba1 100644 --- a/compose.yml +++ b/compose.yml @@ -23,6 +23,7 @@ services: - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.routers.${STACK_NAME}.service=${STACK_NAME}" - "traefik.tcp.routers.${STACK_NAME}-rpc.rule=HostSNI(`*`)" - "traefik.tcp.routers.${STACK_NAME}-rpc.entrypoints=garage-rpc" - "traefik.tcp.services.${STACK_NAME}-rpc.loadbalancer.server.port=3901" -- 2.49.0 From 37dbb6caa81645c3b2d8c50adc395aa88c52a4ab Mon Sep 17 00:00:00 2001 From: Brooke Date: Fri, 15 May 2026 16:11:39 -0400 Subject: [PATCH 11/11] update readme for clarity --- README.md | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5298a1c..64a623a 100644 --- a/README.md +++ b/README.md @@ -54,18 +54,34 @@ Once `garage status` works, you can follow the guide here: https://garagehq.deux ## Monitoring -By default monitoring is disabled and must be enabled in the config. +### Enabling -To enable set `MONITORING` to `true`. +By default monitoring is disabled and must be enabled in your config. + +To enable, set `MONITORING` to `true` and uncomment the line `#COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml"`. > If you've deployed garage before ver `0.0.2+v2.3.0` then you will need to add the following lines to your config: > ``` +> MONITORING_DOMAIN=monitoring.garage.example.com > SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex ->SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex -> +> SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex +> +> #COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml" > MONITORING="true" > ``` +### Deploying + +Now, undeploy the service, generate the new secrets, and finally re-deploy: +``` +abra app undeploy +abra app secret generate --all +abra app deploy +``` +### Utilizing metrics + +Within your chosen monitoring software (ie. Telegraf, Prometheus, etc.), you'll need to make sure it interprets the correct scheme (https), and point it at /metrics as the monitoring endpoint. The secret you copied earlier called metrics_token will be used to authenticate the request. + ## Backups > In development, not currently reliable -- 2.49.0