Merge branch 'main' into monitoring

Reviewed-on: #19
Reviewed-by: Brooke <brooke@myco.systems>

.drone.yml

@@ -6,7 +6,7 @@ steps:
   image: git.coopcloud.tech/coop-cloud/stack-ssh-deploy:latest
   settings:
     host: swarm-test.autonomic.zone
-      stack: example_com  # UPDATE ME
+    stack: garage
     generate_secrets: true
     purge: true
     deploy_key:
@@ -14,9 +14,12 @@ steps:
     networks:
     - proxy
   environment:
-      DOMAIN: example.swarm-test.autonomic.zone # UPDATE ME
+    DOMAIN: garage.swarm-test.autonomic.zone
-      STACK_NAME: example_com  # UPDATE ME
+    STACK_NAME: garage
     LETS_ENCRYPT_ENV: production
+    SECRET_RPC_SECRET_VERSION: v1 # length=64 charset=hex
+    SECRET_ADMIN_TOKEN_SECRET_VERSION: v1 # length=64 charset=hex
+    SECRET_METRICS_TOKEN_SECRET_VERSION: v1 # length=64 charset=hex
 trigger:
   branch:
   - main
@@ -32,7 +35,7 @@ steps:
       from_secret: drone_abra-bot_token
     fork: true
     repositories:
-        - coop-cloud/auto-recipes-catalogue-json
+    - toolshed/auto-recipes-catalogue-json
 
 trigger:
   event: tag

.env.sample

@@ -5,7 +5,12 @@ DOMAIN=garage.example.com
 LETS_ENCRYPT_ENV=production
 COMPOSE_FILE="compose.yml"
 
-SECRET_RPC_SECRET_VERSION=v1 # length=32 charset=hex
+SECRET_RPC_SECRET_VERSION=v1 # length=64 charset=hex
+SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex
+SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex
+
+#COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml"
+#MONITORING_ENABLED=true
 
 # Changing the replication factor after initial deployment is not 
 # supported and requires deleting the existing cluster layout metadata.

README.md

@@ -1,16 +1,16 @@
-# garage
+# Garage
 
 > An open-source distributed object storage service tailored for selfhosting at a small-to-medium scale.
 
 <!-- metadata -->
 
 * **Category**: Apps
-* **Status**: wip
+* **Status**: 3
 * **Image**: [`garage`](https://hub.docker.com/r/dxflrs/garage), 4, upstream
-* **Healthcheck**: No
+* **Healthcheck**: Yes
 * **Backups**: No
 * **Email**: N/A
-* **Tests**: No
+* **Tests**: 3
 * **SSO**: N/A
 
 <!-- endmetadata -->
@@ -52,7 +52,43 @@ You can optionally add this alias to your `.bashrc` (or similar) file to avoid h
 ### Garage Quick Start Guide
 Once `garage status` works, you can follow the guide here: https://garagehq.deuxfleurs.fr/documentation/quick-start/#checking-that-garage-runs-correctly
 
+## Monitoring
 
+### Enabling
+
+By default monitoring is disabled and must be enabled in your config.
+
+To enable, set `MONITORING` to `true` and uncomment the line `#COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml"`.
+
+> If you've deployed garage before ver `0.0.2+v2.3.0` then you will need to add the following lines to your config:
+> ```
+> MONITORING_DOMAIN=monitoring.garage.example.com
+> SECRET_ADMIN_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex
+> SECRET_METRICS_TOKEN_SECRET_VERSION=v1 # length=64 charset=hex
+> 
+> #COMPOSE_FILE="$COMPOSE_FILE:compose.monitoring.yml"
+> MONITORING="true"
+> ```
+
+If you're using the
+[monitoring-ng](https://recipes.coopcloud.tech/monitoring-ng) recipe,
+insert the `metrics_token` with the value of `basic_auth`:
+
+```sh
+abra app secret insert garage.example.coop v1 metrics_token BASIC_AUTH
+```
+
+### Deploying
+
+Now, undeploy the service, generate the new secrets, and finally re-deploy:
+```
+abra app undeploy <app-domain>
+abra app secret generate --all <app-domain>
+abra app deploy <app-domain>
+```
+### Utilizing metrics
+
+Within your chosen monitoring software (ie. Telegraf, Prometheus, etc.), you'll need to make sure it interprets the correct scheme (https), and point it at <app-domain>/metrics as the monitoring endpoint. The secret you copied earlier called metrics_token will be used to authenticate the request.
 
 ## Backups
 
@@ -60,12 +96,11 @@ Once `garage status` works, you can follow the guide here: https://garagehq.deux
 
 By default, backups will only capture a snapshot of the metadata directory, which includes bucket names, hashed secrets, and other related information. 
 By default, the actual data will not be backed up!
-If you're running Garage in a cluster, when you restore the metadata, other nodes will provide any missing data.
+If you're running Garage in a cluster, when you restore the metadata, other nodes will provide any missing data (assuming a replication factor >1).
 
 ### To enable full data backups
 * `abra app config <app domain>`
 * Uncomment the block that starts with `## Enable Full Data Backups`
 * Re-deploy Garage: `abra app undeploy -n <app domain> && sleep 5 && abra app deploy -n <app domain>`
 
-
 For more, see [`garagehq.deuxfleurs.fr`](https://garagehq.deuxfleurs.fr/documentation/cookbook/real-world/).

abra.sh

@@ -1 +1 @@
-export GARAGE_CONF_VERSION=v5
+export GARAGE_CONF_VERSION=v7

compose.monitoring.yml

@@ -0,0 +1,29 @@
+---
+version: "3.8"
+
+services:
+  app:
+    secrets:
+      - source: metrics_token
+        mode: 0600
+      - source: admin_token
+        mode: 0600
+    deploy:
+      labels:
+        - "traefik.http.routers.${STACK_NAME}-metrics.rule=Host(`${DOMAIN}`) && Path(`/metrics`)"
+        - "traefik.http.routers.${STACK_NAME}-metrics.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}-metrics.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}-metrics.service=${STACK_NAME}-metrics"
+        - "traefik.http.services.${STACK_NAME}-metrics.loadbalancer.server.port=3903"
+        - "prometheus.io/scrape=true"
+        - "prometheus.io/port=3903"
+        - "prometheus.io/path=/metrics"
+        - "prometheus.io/auth=bearer"
+
+secrets:
+  admin_token:
+    name: ${STACK_NAME}_admin_token_${SECRET_ADMIN_TOKEN_SECRET_VERSION}
+    external: true
+  metrics_token:
+    name: ${STACK_NAME}_metrics_token_${SECRET_METRICS_TOKEN_SECRET_VERSION}
+    external: true

compose.yml

@@ -3,12 +3,14 @@ version: "3.8"
 
 services:
   app:
-    image: dxflrs/garage:v2.1.0
+    image: dxflrs/garage:v2.3.0
+    hostname: "${DOMAIN}"
     configs:
       - source: garage_conf
         target: /etc/garage.toml
     secrets:
-      - rpc_secret
+      - source: rpc_secret
+        mode: 0600
     networks:
       - proxy
       - internal
@@ -21,16 +23,23 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.routers.${STACK_NAME}.service=${STACK_NAME}"
         - "traefik.tcp.routers.${STACK_NAME}-rpc.rule=HostSNI(`*`)"
         - "traefik.tcp.routers.${STACK_NAME}-rpc.entrypoints=garage-rpc"
         - "traefik.tcp.services.${STACK_NAME}-rpc.loadbalancer.server.port=3901"
-        - "coop-cloud.${STACK_NAME}.version=0.0.1+2.1.0"
+        - "coop-cloud.${STACK_NAME}.version=0.0.2+v2.3.0"
         - "backupbot.backup=true"
         - "backupbot.backup.pre-hook=/garage meta snapshot --all"
         - "backupbot.backup.path=/var/lib/garage/meta/snapshots/,/var/lib/garage/meta/cluster_layout,/var/lib/garage/meta/data_layout,/var/lib/garage/meta/node_key,/var/lib/garage/meta/node_key.pub"
     volumes:
       - "${LOCAL_FOLDER_META:-meta}:/var/lib/garage/meta"
       - "${LOCAL_FOLDER_DATA:-data}:/var/lib/garage/data"
+    healthcheck:
+      test: ["CMD", "/garage", "status"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 10s
 
 networks:
   proxy:

garage.toml.tmpl

@@ -15,7 +15,7 @@ compression_level = 2
 rpc_bind_addr = "[::]:3901"
 rpc_public_addr = "{{ env "DOMAIN" }}:3901"
 rpc_addr = "[::]:3901"
-rpc_secret = "{{ secret "rpc_secret" }}"
+rpc_secret_file = "/run/secrets/rpc_secret"
 
 {{ if ne (env "BOOTSTRAP_ID") "" }}
 bootstrap_peers = [
@@ -27,3 +27,11 @@ bootstrap_peers = [
 s3_region = "garage"
 api_bind_addr = "[::]:3900"
 root_domain = ".s3.garage"
+
+{{ if eq (env "MONITORING_ENABLED") "true" }}
+[admin]
+api_bind_addr = "[::]:3903"
+admin_token_file = "/run/secrets/admin_token"
+metrics_require_token = true
+metrics_token_file = "/run/secrets/metrics_token"
+{{ end }}