shortened secrets, added healthcheck, removed debug log

Reviewed-on: #21

.drone.yml

@@ -18,10 +18,11 @@ steps:
       STACK_NAME: outline
       LETS_ENCRYPT_ENV: production
       APP_ENTRYPOINT_VERSION: v1
+      DB_ENTRYPOINT_VERSION: v1
+      PG_BACKUP_VERSION: v1
       SECRET_DB_PASSWORD_VERSION: v1
       SECRET_SECRET_KEY_VERSION: v1  # length=64
       SECRET_UTILS_SECRET_VERSION: v1  # length=64
-      SECRET_AWS_SECRET_KEY_VERSION: v1
 trigger:
   branch:
     - main
@@ -37,7 +38,7 @@ steps:
         from_secret: drone_abra-bot_token
       fork: true
       repositories:
-        - coop-cloud/auto-recipes-catalogue-json
+        - toolshed/auto-recipes-catalogue-json
 
 trigger:
   event: tag

.env.sample

@@ -4,10 +4,14 @@ TYPE=outline
 
 DOMAIN=outline.example.com
 
+#TIMEOUT=
+
 ## Domain aliases
 #EXTRA_DOMAINS=', `www.outline.example.com`'
 LETS_ENCRYPT_ENV=production
 
+ENABLE_BACKUPS=true
+
 COMPOSE_FILE="compose.yml"
 
 # –––––––––––––––– REQUIRED ––––––––––––––––
@@ -15,15 +19,9 @@ COMPOSE_FILE="compose.yml"
 SECRET_DB_PASSWORD_VERSION=v1
 SECRET_SECRET_KEY_VERSION=v1  # length=64
 SECRET_UTILS_SECRET_VERSION=v1  # length=64
-SECRET_AWS_SECRET_KEY_VERSION=v1
 
-AWS_ACCESS_KEY_ID=
-AWS_REGION=
-AWS_S3_UPLOAD_BUCKET_URL=
-AWS_S3_UPLOAD_BUCKET_NAME=
-AWS_S3_UPLOAD_MAX_SIZE=26214400
-AWS_S3_FORCE_PATH_STYLE=true
-AWS_S3_ACL=private
+# Set to s3 to use AWS S3 bucket
+FILE_STORAGE=local
 
 # –––––––––––––––– OPTIONAL ––––––––––––––––
 
@@ -45,7 +43,7 @@ WEB_CONCURRENCY=1
 
 # Override the maxium size of document imports, could be required if you have
 # especially large Word documents with embedded imagery
-MAXIMUM_IMPORT_SIZE=5120000
+FILE_STORAGE_IMPORT_MAX_SIZE=5120000
 
 # You can remove this line if your reverse proxy already logs incoming http
 # requests and this ends up being duplicative
@@ -85,3 +83,26 @@ ALLOWED_DOMAINS=
 #GOOGLE_ENABLED=1
 #GOOGLE_CLIENT_ID=
 #SECRET_GOOGLE_CLIENT_SECRET_VERSION=v1
+
+COMPOSE_FILE="$COMPOSE_FILE:compose.local.yml"
+FILE_STORAGE_UPLOAD_MAX_SIZE=26214400
+
+#COMPOSE_FILE="$COMPOSE_FILE:compose.aws.yml"
+#AWS_ACCESS_KEY_ID=
+#AWS_REGION=
+#AWS_S3_UPLOAD_BUCKET_URL=
+#AWS_S3_UPLOAD_BUCKET_NAME=
+#AWS_S3_UPLOAD_MAX_SIZE=26214400
+#AWS_S3_FORCE_PATH_STYLE=true
+#AWS_S3_ACL=private
+#SECRET_AWS_SECRET_KEY_VERSION=v1
+
+# COMPOSE_FILE="$COMPOSE_FILE:compose.authentikgroupsync.yml"
+# # details on env here: https://github.com/burritosoftware/Outline-Authentik-Connector/blob/master/.env.example
+# AGS_AUTHENTIK_URL=
+# SECRET_AGS_OUTLINE_TOKEN_VERSION=v1 
+# SECRET_AGS_AUTHENTIK_TOKEN_VERSION=v1
+# SECRET_AGS_WEBHOOK_SECRET_VERSION=v1
+# AGS_AUTO_CREATE_GROUPS=True
+# #AGS_SYNC_GROUP_REGEX=
+# #AGS_DEBUG=True

README.md

@@ -22,13 +22,12 @@ Wiki and knowledge base for growing teams
 3. `abra app new ${REPO_NAME}`
    - **WARNING**: Choose "n" when `abra` asks if you'd like to generate secrets
 4. `abra app config YOURAPPNAME` - be sure to change `$DOMAIN` to something that resolves to
-   your Docker swarm box. For Minio, you'll want:
-   - `AWS_ACCESS_KEY_ID=<minio username>`
-   - `AWS_REGION="us-east-1"`
-   - `AWS_S3_UPLOAD_BUCKET_URL=https://minio.example.com`
-   - `AWS_S3_UPLOAD_BUCKET_NAME=
-5. `abra app deploy YOURAPPNAME`
-7. Open the configured domain in your browser to finish set-up
+   your Docker swarm box
+5. Insert secrets:
+    - `abra app secret insert YOURAPPNAME secret_key v1 $(openssl rand -hex 32)` #12
+    - `abra app secret generate -a YOURAPPNAME`
+6. `abra app deploy YOURAPPNAME`
+8. Open the configured domain in your browser to finish set-up
 
 [`abra`]: https://git.coopcloud.tech/coop-cloud/abra
 [`coop-cloud/traefik`]: https://git.coopcloud.tech/coop-cloud/traefik
@@ -41,14 +40,6 @@ Wiki and knowledge base for growing teams
 abra app cmd YOURAPPNAME app create_email_user test@example.com
 ```
 
-### Post-deploy migration
-
-```
-abra app cmd YOURAPPNAME app migrate
-```
-
-_As of 2022-03-30, this requires `abra` RC version, run `abra upgrade --rc`._
-
 ### Setting up your `.env` config
 
 Avoid the use of quotes (`"..."`) as much as possible, the NodeJS scripts flip out for some reason on some vars.
@@ -61,14 +52,43 @@ Where `<username-to-delete>` is the username of the user to be removed, and
 `<username-to-replace>` is the username of another user, to assign documents and
 revisions to (instead of deleting them).
 
-_As of 2022-03-30, this requires `abra` RC version, run `abra upgrade --rc`._
+### Migrate from S3 to local storage
 
-## Single Sign On with Keycloak
+- `abra app config <domain>`, add
+  - `COMPOSE_FILE="$COMPOSE_FILE:compose.local.yml"`
+  - `FILE_STORAGE_UPLOAD_MAX_SIZE=26214400`
+- `abra app deploy <domain> -f`
+  - compose.aws.yml should still be deployed!
+- `abra app undeploy <domain>`
+- on the docker host, find mountpoint of newly created volume via `docker volume ls` and `docker volume inspect`
+  - volume name is smth like `<domain>_storage-data`
+- take note which linux user owns `<storage_mountpoint>` (likely `1001`)
+- use s3cmd/rclone/... to sync your bucket to `<storage_mountpoint>`
+- `chown -R <storage_user>:<storage_user> <storage_mountpoint>`
+- `abra app config <domain>`, switch storage backend
+  - remove `AWS_*` vars, `SECRET_AWS_SECRET_KEY_VERSION` and `COMPOSE_FILE="$COMPOSE_FILE:compose.aws.yml"`
+  - set `FILE_STORAGE=local`
+- `abra app deploy <domain> -f`
+- enjoy getting rid of S3 🥳
 
-`abra app config YOURAPPNAME`, then uncomment everything in the `OIDC_` section.
+## Single Sign On with Keycloak/Authentik
 
-Create a new client in Keycloak:
+- Create an OIDC client in Keycloak (in Authentik this is called a provider and application)
+- Run `abra app config YOURAPPNAME`, then uncomment everything in the `OIDC_` section.
+  - **Valid Redirect URIs**: `https://YOURAPPDOMAIN/auth/oidc.callback`
+  - Reference the client/provider info to populate the `_AUTH_URI` `_TOKEN_URI` and `_USERINFO_URI` values
+- Set the OIDC secret using the value from the client/provider `abra app secret insert YOURAPPNAME oidc_client_secret v1 SECRETVALUE`
+- `abra app deploy YOURAPPDOMAIN`
 
-- **Valid Redirect URIs**: `https://YOURAPPDOMAIN/auth/oidc.callback`
+### Advanced: Group Sync with Authentik
+- As `outline` doesn't support group sync, you can make use of an [extra service, the Outline-Authentik-Connector,](https://github.com/burritosoftware/Outline-Authentik-Connector) to do so.
+- Just uncomment the respective section in your `.env`, and set the necessary envs.
+- Then [follow these instructions](https://github.com/burritosoftware/Outline-Authentik-Connector?tab=readme-ov-file#outline-setup) to create the needed user and tokens
+  - ! for the authentik-token make sure you don't use the token it shows when creating the user (that is a password), create as the user (it will expire) but in the admin interface (path: `https://login..../if/admin/#/core/tokens`). Also setting the needed global permissions was not possible on the user directly, but I had to create a role for this.
 
-`abra app deploy YOURAPPDOMAIN`
+- and insert them as secrets:
+```
+abra app secret insert YOURAPPNAME agsoutline v1 SECRETVALUE
+abra app secret insert YOURAPPNAME agsauthentik v1 SECRETVALUE
+abra app secret insert YOURAPPNAME agswebhook v1 SECRETVALUE
+```

abra.sh

@@ -1,5 +1,7 @@
-export APP_ENTRYPOINT_VERSION=v8
+export APP_ENTRYPOINT_VERSION=v11
 export DB_ENTRYPOINT_VERSION=v2
+export PG_BACKUP_VERSION=v1
+export AGS_ENTRYPOINT_VERSION=v1
 
 create_email_user() {
 	if [ -z "$1" ]; then
@@ -19,6 +21,10 @@ migrate() {
 	yarn db:migrate --env=production-ssl-disabled
 }
 
+generate_secret() {
+    abra app secret insert $DOMAIN secret_key v1 $(openssl rand -hex 32)
+}
+
 delete_user_by_id() {
 	if [ -z "$1" ] || [ -z "$2" ]; then
 		echo "Usage: ... delete_user_by_id <userid-to-delete> <userid-to-replace>"
@@ -93,3 +99,10 @@ delete_duplicate_users() {
 		delete_user_by_id "$USERID_OLD" "$USERID_NEW"
 	done
 }
+
+fix_collation_mismatch() {
+  psql -U ${POSTGRES_USER} -d outline -c "ALTER DATABASE outline REFRESH COLLATION VERSION;"
+  psql -U ${POSTGRES_USER} -d outline -c "REINDEX DATABASE outline;"
+  psql -U ${POSTGRES_USER} -d postgres -c "ALTER DATABASE postgres REFRESH COLLATION VERSION;"
+  psql -U ${POSTGRES_USER} -d postgres -c "REINDEX DATABASE postgres;"
+}

alaconnect.yml

@@ -0,0 +1,15 @@
+authentik:
+    env: 
+        OIDC_CLIENT_ID: outline
+        OIDC_AUTH_URI: https://authentik.example.com/application/o/authorize/
+        OIDC_TOKEN_URI: https://authentik.example.com/application/o/token/
+        OIDC_USERINFO_URI: https://authentik.example.com/application/o/userinfo/
+        OIDC_DISPLAY_NAME: "Authentik"
+    uncomment:
+        - compose.oidc.yml
+        - OIDC_ENABLED
+        - OIDC_USERNAME_CLAIM
+        - OIDC_SCOPES
+        - SECRET_OIDC_CLIENT_SECRET_VERSION
+    shared_secrets:
+        outline_secret: oidc_client_secret

compose.authentikgroupsync.yml

@@ -0,0 +1,62 @@
+---
+version: "3.8"
+
+services:
+  outline-authentik-connector:
+    image: burritosoftware/outline-authentik-connector:1.2
+
+    secrets:
+      - agsoutline
+      - agsauthentik
+      - agswebhook
+      
+    environment:
+      - AUTHENTIK_URL=${AGS_AUTHENTIK_URL}
+      - OUTLINE_URL=https://${DOMAIN}
+      - OUTLINE_TOKEN_FILE=/var/run/secrets/agsoutline
+      - OUTLINE_WEBHOOK_SECRET_FILE=/var/run/secrets/agswebhook
+      - AUTHENTIK_TOKEN_FILE=/var/run/secrets/agsauthentik
+      - AUTO_CREATE_GROUPS=${AGS_AUTO_CREATE_GROUPS:-True}
+      - DEBUG=${AGS_DEBUG:-False}
+      - SYNC_GROUP_REGEX=${AGS_SYNC_GROUP_REGEX:-.*}
+
+    deploy:
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.routers.sync-router.rule=Host(`groupsync.${DOMAIN}`)"
+        - "traefik.http.routers.sync-router.entrypoints=web-secure"
+        - "traefik.http.routers.sync-router.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - "traefik.http.services.sync-router.loadbalancer.server.port=80"
+
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://0.0.0.0:80/ | grep -q '\"status\":\"running\"' || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+
+    networks:
+      - backend
+      - proxy
+
+    configs:
+      - source: ags_entrypoint
+        target: /docker-entrypoint.sh
+        mode: 0555
+
+    entrypoint: /docker-entrypoint.sh
+
+secrets:
+  agsoutline:
+    name: ${STACK_NAME}_agsoutline_${SECRET_AGS_OUTLINE_TOKEN_VERSION}
+    external: true
+  agsauthentik:
+    name: ${STACK_NAME}_agsauthentik_${SECRET_AGS_AUTHENTIK_TOKEN_VERSION}
+    external: true
+  agswebhook:
+    name: ${STACK_NAME}_agswebhook_${SECRET_AGS_WEBHOOK_SECRET_VERSION}
+    external: true
+
+configs:
+  ags_entrypoint:
+    name: ${STACK_NAME}_ags_entrypoint_${AGS_ENTRYPOINT_VERSION}
+    file: entrypoint.authentikgroupsync.sh

compose.aws.yml

@@ -0,0 +1,22 @@
+---
+version: "3.8"
+
+services:
+  app:
+    secrets:
+      - aws_secret_key
+    environment:
+      - AWS_ACCESS_KEY_ID
+      - AWS_REGION
+      - AWS_S3_ACL
+      - AWS_S3_FORCE_PATH_STYLE
+      - AWS_S3_UPLOAD_BUCKET_NAME
+      - AWS_S3_UPLOAD_BUCKET_URL
+      - AWS_S3_UPLOAD_MAX_SIZE
+      - AWS_SDK_LOAD_CONFIG=0
+      - AWS_SECRET_KEY_FILE=/run/secrets/aws_secret_key
+
+secrets:
+  aws_secret_key:
+    name: ${STACK_NAME}_aws_secret_key_${SECRET_AWS_SECRET_KEY_VERSION}
+    external: true

compose.local.yml

@@ -0,0 +1,13 @@
+---
+version: "3.8"
+
+services:
+  app:
+    volumes:
+      - storage-data:/var/lib/outline/data
+    environment:
+      - FILE_STORAGE
+      - FILE_STORAGE_UPLOAD_MAX_SIZE
+
+volumes:
+  storage-data:

compose.yml

@@ -6,9 +6,8 @@ services:
     networks:
       - backend
       - proxy
-    image: outlinewiki/outline:0.71.0
+    image: outlinewiki/outline:1.6.0
     secrets:
-      - aws_secret_key
       - db_password
       - secret_key
       - utils_secret
@@ -17,15 +16,7 @@ services:
         target: /docker-entrypoint.sh
         mode: 0555
     environment:
-      - AWS_ACCESS_KEY_ID
-      - AWS_REGION
-      - AWS_S3_ACL
-      - AWS_S3_FORCE_PATH_STYLE
-      - AWS_S3_UPLOAD_BUCKET_NAME
-      - AWS_S3_UPLOAD_BUCKET_URL
-      - AWS_S3_UPLOAD_MAX_SIZE
-      - AWS_SDK_LOAD_CONFIG=0
-      - AWS_SECRET_KEY_FILE=/run/secrets/aws_secret_key
+      - FILE_STORAGE
       - DATABASE_PASSWORD_FILE=/run/secrets/db_password
       - FORCE_HTTPS=true
       - PGSSLMODE=disable
@@ -43,19 +34,20 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=0.13.0+0.71.0"
-        ## Redirect from EXTRA_DOMAINS to DOMAIN
-        #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
-        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
-        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
+        - "coop-cloud.${STACK_NAME}.version=2.18.0+1.6.0"
+        # Redirect from EXTRA_DOMAINS to DOMAIN
+        - "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
+        - "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
+        - "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
+        - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT}"
 
   cache:
-    image: redis:7.2.0
+    image: redis:8.6.1
     networks:
       - backend
 
   db:
-    image: postgres:15.4
+    image: postgres:17.9
     networks:
       - backend
     secrets:
@@ -64,6 +56,9 @@ services:
       - source: db_entrypoint
         target: /docker-entrypoint.sh
         mode: 0555
+      - source: pg_backup
+        target: /pg_backup.sh
+        mode: 0555
     environment:
       POSTGRES_DB: outline
       POSTGRES_PASSWORD_FILE: /run/secrets/db_password
@@ -73,10 +68,10 @@ services:
     entrypoint: /docker-entrypoint.sh
     deploy:
       labels:
-        backupbot.backup: "true"
-        backupbot.backup.path: "/tmp/dump.sql.gz"
-        backupbot.backup.post-hook: "rm -f /tmp/dump.sql.gz"
-        backupbot.backup.pre-hook: "sh -c 'PGPASSWORD=$$(cat $${POSTGRES_PASSWORD_FILE}) pg_dump -U outline outline | gzip > /tmp/dump.sql.gz'"
+        backupbot.backup: "${ENABLE_BACKUPS:-true}"
+        backupbot.backup.pre-hook: "/pg_backup.sh backup"
+        backupbot.backup.volumes.postgres_data.path: "backup.sql"
+        backupbot.restore.post-hook: '/pg_backup.sh restore'
 
 secrets:
   secret_key:
@@ -85,9 +80,6 @@ secrets:
   utils_secret:
     name: ${STACK_NAME}_utils_secret_${SECRET_UTILS_SECRET_VERSION}
     external: true
-  aws_secret_key:
-    name: ${STACK_NAME}_aws_secret_key_${SECRET_AWS_SECRET_KEY_VERSION}
-    external: true
   db_password:
     name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}
     external: true
@@ -109,3 +101,6 @@ configs:
     name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
     file: entrypoint.postgres.sh.tmpl
     template_driver: golang
+  pg_backup:
+    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION}
+    file: pg_backup.sh

entrypoint.authentikgroupsync.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+
+load_secret() {
+    env_var="$1"
+    secret_file="$2"
+
+    if [ -f "$secret_file" ]; then
+        value=$(cat "$secret_file")
+        if [ -z "$value" ]; then
+            echo >&2 "error: $secret_file is empty"
+            exit 1
+        fi
+        export "$env_var"="$value"
+    else
+        echo >&2 "[info] didn't set $env_var because $secret_file does not exist. If you don't use the secret or it is no secret at all you can safely ignore this message."
+    fi
+}
+
+# load every env-var that ends on _FILE
+for var in $(env | grep "_FILE="); do
+    key=$(echo "$var" | sed 's/_FILE=.*//')
+    value=$(echo "$var" | sed 's/.*_FILE=//')
+    load_secret "$key" "$value"
+done
+
+exec uvicorn connect:app --host=0.0.0.0 --port=80

entrypoint.sh.tmpl

@@ -1,6 +1,8 @@
 #!/bin/sh
 
+{{ if eq (env "FILE_STORAGE")  "s3" }}
 export AWS_SECRET_ACCESS_KEY=$(cat /run/secrets/aws_secret_key)
+{{ end }}
 
 {{ if eq (env "SMTP_ENABLED") "1" }}
 export SMTP_PASSWORD=$(cat /run/secrets/smtp_password)
@@ -16,10 +18,9 @@ export GOOGLE_CLIENT_SECRET=$(cat /run/secrets/google_client_secret)
 
 export UTILS_SECRET=$(cat /run/secrets/utils_secret)
 export SECRET_KEY=$(cat /run/secrets/secret_key)
-export DATABASE_PASSWORD=$(cat /run/secrets/db_password)
+DATABASE_PASSWORD=$(cat /run/secrets/db_password)
 export DATABASE_URL="postgres://outline:${DATABASE_PASSWORD}@${STACK_NAME}_db:5432/outline"
 
 if [ ! "$1" = "-e" ]; then
-	/usr/local/bin/yarn db:migrate --env=production-ssl-disabled
-	/usr/local/bin/yarn start "$@"
+  node build/server/index.js
 fi

pg_backup.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+BACKUP_FILE='/var/lib/postgresql/data/backup.sql'
+
+function backup {
+  export PGPASSWORD=$(cat $POSTGRES_PASSWORD_FILE)
+  pg_dump -U ${POSTGRES_USER} ${POSTGRES_DB} > $BACKUP_FILE
+}
+
+function restore {
+    cd /var/lib/postgresql/data/
+    restore_config(){
+        # Restore allowed connections
+        cat pg_hba.conf.bak > pg_hba.conf
+        su postgres -c 'pg_ctl reload'
+    }
+    # Don't allow any other connections than local
+    cp pg_hba.conf pg_hba.conf.bak
+    echo "local all all trust" > pg_hba.conf
+    su postgres -c 'pg_ctl reload'
+    trap restore_config EXIT INT TERM
+
+    # Recreate Database
+    psql -U ${POSTGRES_USER} -d postgres -c "DROP DATABASE ${POSTGRES_DB} WITH (FORCE);" 
+    createdb -U ${POSTGRES_USER} ${POSTGRES_DB}
+    psql -U ${POSTGRES_USER} -d ${POSTGRES_DB} -1 -f $BACKUP_FILE
+
+    trap - EXIT INT TERM
+    restore_config
+}
+
+$@

release/2.0.0+0.74.0

@@ -0,0 +1,4 @@
+Due to the introduction of local storage, you need to adapt your config to continue using S3 storage. Just add the following lines to your config:
+
+FILE_STORAGE=s3
+COMPOSE_FILE="$COMPOSE_FILE:compose.aws.yml"

release/2.17.0+1.5.0

@@ -0,0 +1 @@
+y

release/2.9.1+0.82.0

@@ -0,0 +1 @@
+Fixes a problem where deployments were consistently giving a timeout response even though they were successful