make pg upgrade more safe

.drone.yml

@@ -0,0 +1,43 @@
+---
+kind: pipeline
+name: deploy to swarm-test.autonomic.zone
+steps:
+  - name: deployment
+    image: git.coopcloud.tech/coop-cloud/stack-ssh-deploy:latest
+    settings:
+      host: swarm-test.autonomic.zone
+      stack: outline
+      generate_secrets: true
+      purge: true
+      deploy_key:
+        from_secret: drone_ssh_swarm_test
+      networks:
+        - proxy
+    environment:
+      DOMAIN: outline.swarm-test.autonomic.zone
+      STACK_NAME: outline
+      LETS_ENCRYPT_ENV: production
+      APP_ENTRYPOINT_VERSION: v1
+      SECRET_DB_PASSWORD_VERSION: v1
+      SECRET_SECRET_KEY_VERSION: v1  # length=64
+      SECRET_UTILS_SECRET_VERSION: v1  # length=64
+      SECRET_AWS_SECRET_KEY_VERSION: v1
+trigger:
+  branch:
+    - main
+---
+kind: pipeline
+name: generate recipe catalogue
+steps:
+  - name: release a new version
+    image: plugins/downstream
+    settings:
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - coop-cloud/auto-recipes-catalogue-json
+
+trigger:
+  event: tag

.env.sample

@@ -9,8 +9,6 @@ DOMAIN=outline.example.com
 LETS_ENCRYPT_ENV=production
 
 COMPOSE_FILE="compose.yml"
-#COMPOSE_YML="compose.yml:compose.oidc.yml"
-#COMPOSE_YML="compose.yml:compose.google.yml"
 
 # –––––––––––––––– REQUIRED ––––––––––––––––
 
@@ -70,6 +68,7 @@ ALLOWED_DOMAINS=
 #SMTP_TLS_CIPHERS=
 #SMTP_SECURE=true
 
+#COMPOSE_FILE="$COMPOSE_FILE:compose.oidc.yml"
 #OIDC_ENABLED=1
 #OIDC_CLIENT_ID=
 #OIDC_AUTH_URI=
@@ -80,6 +79,7 @@ ALLOWED_DOMAINS=
 #OIDC_SCOPES="openid profile email"
 #SECRET_OIDC_CLIENT_SECRET_VERSION=v1
 
+#COMPOSE_FILE="$COMPOSE_FILE:compose.google.yml"
 #GOOGLE_ENABLED=1
 #GOOGLE_CLIENT_ID=
 #SECRET_GOOGLE_CLIENT_SECRET_VERSION=v1

README.md

@@ -52,3 +52,13 @@ Where `<username-to-delete>` is the username of the user to be removed, and
 revisions to (instead of deleting them).
 
 _As of 2022-03-30, this requires `abra` RC version, run `abra upgrade --rc`._
+
+## Single Sign On with Keycloak
+
+`abra app config YOURAPPNAME`, then uncomment everything in the `OIDC_` section.
+
+Create a new client in Keycloak:
+
+- **Valid Redirect URIs**: `https://YOURAPPDOMAIN/auth/oidc.callback`
+
+`abra app deploy YOURAPPDOMAIN`

abra.sh

@@ -1,4 +1,5 @@
 export APP_ENTRYPOINT_VERSION=v6
+export DB_ENTRYPOINT_VERSION=v1
 
 migrate() {
 	export DATABASE_PASSWORD=$(cat /run/secrets/db_password)
@@ -22,10 +23,14 @@ delete_user_by_id() {
 	UPDATE group_users SET "createdById" = '$USERID_REPLACE' WHERE "createdById" = '$USERID_REMOVE';
 	UPDATE collections SET "createdById" = '$USERID_REPLACE' WHERE "createdById" = '$USERID_REMOVE';
 	UPDATE collection_users SET "createdById" = '$USERID_REPLACE' WHERE "createdById" = '$USERID_REMOVE';
+	UPDATE collection_groups SET "createdById" = '$USERID_REPLACE' WHERE "createdById" = '$USERID_REMOVE';
 	UPDATE documents SET "lastModifiedById" = '$USERID_REPLACE' WHERE "lastModifiedById" = '$USERID_REMOVE';
 	UPDATE documents SET "createdById" = '$USERID_REPLACE' WHERE "createdById" = '$USERID_REMOVE';
 	UPDATE revisions SET "userId" = '$USERID_REPLACE' WHERE "userId" = '$USERID_REMOVE';
 	UPDATE attachments SET "userId" = '$USERID_REPLACE' WHERE "userId" = '$USERID_REMOVE';
+	UPDATE backlinks SET "userId" = '$USERID_REPLACE' WHERE "userId" = '$USERID_REMOVE';
+	UPDATE file_operations SET "userId" = '$USERID_REPLACE' WHERE "userId" = '$USERID_REMOVE';
+	UPDATE users SET "suspendedById" = '$USERID_REPLACE' WHERE "suspendedById" = '$USERID_REMOVE';
 	DELETE FROM search_queries WHERE "userId" = '$USERID_REMOVE';
 	DELETE FROM shares WHERE "userId" = '$USERID_REMOVE';
 	DELETE FROM notification_settings WHERE "userId" = '$USERID_REMOVE';

compose.yml

@@ -6,7 +6,7 @@ services:
     networks:
       - backend
       - proxy
-    image: outlinewiki/outline:0.67.2
+    image: outlinewiki/outline:0.69.2
     secrets:
       - aws_secret_key
       - db_password
@@ -43,29 +43,40 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=0.5.0+0.67.2"
+        - "coop-cloud.${STACK_NAME}.version=0.8.0+0.69.2"
         ## Redirect from EXTRA_DOMAINS to DOMAIN
         #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
         #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
         #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
 
   cache:
-    image: redis:6.2.6
+    image: redis:7.0.11
     networks:
       - backend
 
   db:
-    image: postgres:11
+    image: postgres:15.3
     networks:
       - backend
     secrets:
       - db_password
+    configs:
+      - source: db_entrypoint
+        target: /docker-entrypoint.sh
+        mode: 0555
     environment:
       POSTGRES_DB: outline
       POSTGRES_PASSWORD_FILE: /run/secrets/db_password
       POSTGRES_USER: outline
     volumes:
       - "postgres_data:/var/lib/postgresql/data"
+    entrypoint: /docker-entrypoint.sh
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.path: "/tmp/dump.sql.gz"
+        backupbot.backup.post-hook: "rm -f /tmp/dump.sql.gz"
+        backupbot.backup.pre-hook: "sh -c 'PGPASSWORD=$$(cat $${POSTGRES_PASSWORD_FILE}) pg_dump -U outline outline | gzip > /tmp/dump.sql.gz'"
 
 secrets:
   secret_key:
@@ -94,3 +105,7 @@ configs:
     name: ${STACK_NAME}_app_entrypoint_${APP_ENTRYPOINT_VERSION}
     file: entrypoint.sh.tmpl
     template_driver: golang
+  db_entrypoint:
+    name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
+    file: entrypoint.postgres.sh.tmpl
+    template_driver: golang

entrypoint.postgres.sh.tmpl

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+MIGRATION_MARKER=$PGDATA/migration_in_progress
+OLDDATA=$PGDATA/old_data
+NEWDATA=$PGDATA/new_data
+
+if [ -e $MIGRATION_MARKER ]; then
+  echo "FATAL: previous migration not completed. manual restore necessary"
+  exit 1
+fi
+
+DATA_VERSION=$(cat $PGDATA/PG_VERSION)
+
+if [ -n "$DATA_VERSION" -a "$PG_MAJOR" != "$DATA_VERSION" ]; then
+  echo "postgres data version $DATA_VERSION found, but need $PG_MAJOR. Starting migration"
+  echo "Installing postgres $DATA_VERSION"
+  sed -i "s/$/ $DATA_VERSION/" /etc/apt/sources.list.d/pgdg.list
+  apt-get update && apt-get install -y --no-install-recommends \
+    postgresql-$DATA_VERSION \
+    && rm -rf /var/lib/apt/lists/*
+  echo "shuffling around"
+  gosu postgres mkdir $OLDDATA $NEWDATA
+  chmod 700 $OLDDATA $NEWDATA
+  mv $PGDATA/* $OLDDATA/ || true
+  touch $MIGRATION_MARKER
+  echo "running initdb"
+  # abuse entrypoint script for initdb by making server error out
+  gosu postgres bash -c "export PGDATA=$NEWDATA ; /usr/local/bin/docker-entrypoint.sh --invalid-arg"
+  echo "running pg_upgrade"
+  cd /tmp
+  gosu postgres pg_upgrade --link -b /usr/lib/postgresql/$DATA_VERSION/bin -d $OLDDATA -D $NEWDATA -U $POSTGRES_USER
+  cp $OLDDATA/pg_hba.conf $NEWDATA/
+  mv $NEWDATA/* $PGDATA
+  rm -rf $OLDDATA
+  rmdir $NEWDATA
+  rm $MIGRATION_MARKER
+  echo "migration complete"
+fi
+
+/usr/local/bin/docker-entrypoint.sh postgres