Upgrade to upstream v4.5.6+1.2.0

Applies the changes defined in https://github.com/hometown-fork/hometown/releases/tag/v4.5.6%2Bhometown-1.2.0
 (with one exception: I ignored the optional fasp Sidekiq queue)

There was a long gap in upstream releases,
 and the hometown fork skipped over 4.3.x and 4.4.x
 so there are a lot of changes here.

NB: this isn't the current up-to-date version of hometown,
 but the later releases suggest doing the 1.2.0 upgrade
 before moving on. So I'm giving Coop-Cloud operators that option

.drone.yml

@@ -38,7 +38,7 @@ steps:
         from_secret: drone_abra-bot_token
       fork: true
       repositories:
-        - coop-cloud/auto-recipes-catalogue-json
+        - toolshed/auto-recipes-catalogue-json
 
 trigger:
   event: tag

.env.sample

@@ -1,6 +1,6 @@
 TYPE=hometown
 
-DOMAIN={{ .Domain }}
+DOMAIN=hometown.example.com
 # Enables WEB_DOMAIN if set (FOR FUTURE USE)
 # USER_DOMAIN=
 
@@ -54,11 +54,9 @@ DB_PORT=5432
 REDIS_HOST=redis
 REDIS_PORT=6379
 # REDIS_URL=
-# REDIS_NAMESPACE=
 # CACHE_REDIS_HOST=
 # CACHE_REDIS_PORT=
 # CACHE_REDIS_URL=
-# CACHE_REDIS_NAMESPACE=
 
 # ElasticSearch
 # --------------------------------------
@@ -79,6 +77,9 @@ SECRET_VAPID_PRIVATE_KEY_VERSION=v1
 SECRET_DB_PASSWORD_VERSION=v1
 SECRET_SMTP_PASSWORD_VERSION=v1
 SECRET_AWS_SECRET_ACCESS_KEY_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_VERSION=v1
 
 # Web Push
 # ========

abra.sh

@@ -1,70 +1,92 @@
 #!/bin/bash
 
-export ENTRYPOINT_CONF_VERSION=v7
+export ENTRYPOINT_CONF_VERSION=v9
 
-assets() {
-  export OTP_SECRET=$(cat /run/secrets/otp_secret)
-  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
-  export DB_PASS=$(cat /run/secrets/db_password)
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
 
-  RAILS_ENV=production bundle exec rails assets:precompile
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  declare -x -g "$var"="$val"
+  unset "$fileVar"
 }
 
-setup() {
-  export OTP_SECRET=$(cat /run/secrets/otp_secret)
-  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
-  export DB_PASS=$(cat /run/secrets/db_password)
+environment() {
+    # for sidekiq service bundle exec env var threading
+    file_env "OTP_SECRET"
+    file_env "SECRET_KEY_BASE"
+    file_env "DB_PASS"
+    file_env "SMTP_PASSWORD"
+    file_env "VAPID_PRIVATE_KEY"
 
-  RAILS_ENV=production bundle exec rake db:setup
+    declare -x RAILS_ENV=production
 }
 
-admin() {
-  export OTP_SECRET=$(cat /run/secrets/otp_secret)
-  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
-  export DB_PASS=$(cat /run/secrets/db_password)
-
-  RAILS_ENV=production bin/tootctl accounts create "$1" --email "$2" --confirmed --role admin
+setup_admin() {
+    ## Create an admin user
+    environment
+    accounts create "$1" --email "$2" --confirmed --role admin
 }
 
-secrets() {
-  docker context use default > /dev/null 2>&1
-
-  echo "Generating secrets for new Hometown deployment..."
-  echo ""
-
-  SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
-  abra app secret insert "$APP_NAME" secret_key_base v1 "$SECRET_KEY_BASE"
-  echo "SECRET_KEY_BASE = $SECRET_KEY_BASE"
-  echo ""
-
-  OTP_SECRET=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
-  abra app secret insert "$APP_NAME" otp_secret v1 "$OTP_SECRET"
-  echo "OTP_SECRET = $OTP_SECRET"
-  echo ""
-
-  docker run \
-    -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \
-    -e OTP_SECRET="$OTP_SECRET" \
-    --rm tootsuite/mastodon:v3.4.0 \
-    bundle exec rake mastodon:webpush:generate_vapid_key \
-    > /tmp/key.txt
-
-  VAPID_PRIVATE_KEY=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" "/tmp/key.txt")
-  VAPID_PUBLIC_KEY=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" "/tmp/key.txt")
-  rm -rf /tmp/key.txt
-
-  echo "VAPID_PUBLIC_KEY = $VAPID_PUBLIC_KEY"
-  echo "!IMPORTANT! you MUST insert this VAPID_PUBLIC_KEY into your app .env config !IMPORTANT!"
-  echo ""
-
-  abra app secret insert "$APP_NAME" vapid_private_key v1 "$VAPID_PRIVATE_KEY"
-  echo "VAPID_PRIVATE_KEY = $VAPID_PRIVATE_KEY"
-  echo ""
-
-  abra app secret generate "$APP_NAME" db_password v1
-  echo ""
-
-  echo "don't forget to insert your smtp_password! your deployment won't work without it"
-  echo "run \"abra app secret insert $APP_NAME smtp_password v1 YOURSMTPPASSWORD\""
-  echo ""
+shell() {
+    ## Run a shell with proper environment
+    environment
+    bash $@
+}
+
+generate_secrets() {
+    ## Run `abra app cmd -l <yourdomain> generate_secrets` to use Docker to generate secrets you'll need to deploy
+    ## your new instance (and create the secrets on target app).
+    docker context use default > /dev/null 2>&1
+
+    echo "Generating secrets for new Hometown deployment..."
+    echo ""
+
+    SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:v4.2.0 bundle exec rake secret)
+    abra app secret insert "$APP_NAME" secret_key_base v1 "$SECRET_KEY_BASE"
+    echo "SECRET_KEY_BASE = $SECRET_KEY_BASE"
+    echo ""
+
+    OTP_SECRET=$(docker run --rm tootsuite/mastodon:v4.2.0 bundle exec rake secret)
+    abra app secret insert "$APP_NAME" otp_secret v1 "$OTP_SECRET"
+    echo "OTP_SECRET = $OTP_SECRET"
+    echo ""
+
+    docker run \
+           -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \
+           -e OTP_SECRET="$OTP_SECRET" \
+           --rm tootsuite/mastodon:v3.4.0 \
+           bundle exec rake mastodon:webpush:generate_vapid_key \
+           > /tmp/key.txt
+
+    VAPID_PRIVATE_KEY=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" "/tmp/key.txt")
+    VAPID_PUBLIC_KEY=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" "/tmp/key.txt")
+    rm -rf /tmp/key.txt
+
+    echo "VAPID_PUBLIC_KEY = $VAPID_PUBLIC_KEY"
+    echo "!IMPORTANT! you MUST insert this VAPID_PUBLIC_KEY into your app .env config !IMPORTANT!"
+    echo ""
+
+    abra app secret insert "$APP_NAME" vapid_private_key v1 "$VAPID_PRIVATE_KEY"
+    echo "VAPID_PRIVATE_KEY = $VAPID_PRIVATE_KEY"
+    echo ""
+
+    abra app secret generate "$APP_NAME" db_password v1
+    echo ""
+
+    echo "don't forget to insert your smtp_password! your deployment won't work without it"
+    echo "run \"abra app secret insert $APP_NAME smtp_password v1 YOURSMTPPASSWORD\""
+    echo ""
 }

compose.yml

@@ -3,8 +3,8 @@ version: "3.8"
 
 services:
   app:
-    image: git.coopcloud.tech/coop-cloud-chaos-patchs/hometown:v4.0.10-hometown-1.1.1
-    command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
+    image: git.coopcloud.tech/coop-cloud-chaos-patchs/hometown:v4.5.6-hometown-1.2.0
+    command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rake db:migrate; bundle exec rails s -p 3000"
     networks: &bothNetworks
       - proxy
       - internal_network
@@ -19,7 +19,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}_web.rule=Host(`${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}_web.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}_web.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=1.1.3+v4.0.10-hometown-1.1.1"
+        - "coop-cloud.${STACK_NAME}.version=3.0.0+v4.5.6-hometown-1.2.0"
     configs: &configs
       - source: entrypoint_sh
         target: /usr/local/bin/entrypoint.sh
@@ -33,18 +33,24 @@ services:
       - secret_key_base
       - smtp_password
       - vapid_private_key
+      - enc_det_key
+      - enc_salt
+      - enc_pri_key
     environment: &env
+      - ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_FILE=/run/secrets/enc_det_key
+      - ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_FILE=/run/secrets/enc_salt
+      - ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_FILE=/run/secrets/enc_pri_key
       - ALLOW_ACCESS_TO_HIDDEN_SERVICE
       - ALTERNATE_DOMAINS
       - AUTHORIZED_FETCH
       - CACHE_REDIS_HOST
-      - CACHE_REDIS_NAMESPACE
       - CACHE_REDIS_PORT
       - CACHE_REDIS_URL
       - DB_HOST
       - DB_NAME
       - DB_PORT
       - DB_USER
+      - DB_PASS_FILE=/run/secrets/db_password
       - DEFAULT_LOCALE
       - DISALLOW_UNAUTHENTICATED_API_ACCESS
       - EMAIL_DOMAIN_ALLOWLIST
@@ -96,7 +102,6 @@ services:
       - RAILS_ENV
       - RAILS_SERVE_STATIC_FILES
       - REDIS_HOST
-      - REDIS_NAMESPACE
       - REDIS_PORT
       - REDIS_URL
       - SAML_ACS_URL
@@ -142,7 +147,7 @@ services:
       - WEB_DOMAIN
 
   streaming:
-    image: git.coopcloud.tech/coop-cloud-chaos-patchs/hometown:v4.0.10-hometown-1.1.1
+    image: git.coopcloud.tech/coop-cloud-chaos-patchs/hometown:v4.5.6-hometown-1.2.0-streaming
     command: node ./streaming
     configs: *configs
     entrypoint: *entrypoint
@@ -163,7 +168,7 @@ services:
     volumes: *appVolume # used to make sure this volume is created
 
   sidekiq:
-    image: git.coopcloud.tech/coop-cloud-chaos-patchs/hometown:v4.0.10-hometown-1.1.1
+    image: git.coopcloud.tech/coop-cloud-chaos-patchs/hometown:v4.5.6-hometown-1.2.0
     secrets: *secrets
     command: bundle exec sidekiq
     configs: *configs
@@ -177,7 +182,7 @@ services:
     environment: *env
 
   db:
-    image: postgres:14.5-alpine
+    image: postgres:14.10-alpine
     networks: &internalNetwork
       - internal_network
     volumes:
@@ -188,9 +193,17 @@ services:
       - POSTGRES_DB=${DB_NAME}
       - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
       - POSTGRES_USER=${DB_USER}
+    deploy:
+      labels:
+        - "backupbot.backup=true"
+        - "backupbot.backup.pre-hook=sh -c \"mkdir -p /var/backup/hometown; /usr/local/bin/pg_dump -Fc -U ${DB_USER} ${DB_NAME} > /var/backup/hometown/backupbot.dump\""
+        - "backupbot.backup.path=/var/backup/hometown"
+        - "backupbot.backup.post-hook=rm -f /var/backup/hometown/backupbot.dump"
+        - "backupbot.restore.post-hook=sh -c \"/usr/local/bin/pg_restore -U ${DB_USER} -d ${DB_NAME} -1 /var/backup/hometown/backupbot.dump\""
+
 
   redis:
-    image: redis:7.0-alpine
+    image: redis:7.4-alpine
     networks: *internalNetwork
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
@@ -213,6 +226,15 @@ secrets:
   smtp_password:
     name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
     external: true
+  enc_det_key:
+    name: ${STACK_NAME}_enc_det_key_${SECRET_ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_VERSION}
+    external: true
+  enc_salt:
+    name: ${STACK_NAME}_enc_salt_${SECRET_ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_VERSION}
+    external: true
+  enc_pri_key:
+    name: ${STACK_NAME}_enc_pri_key_${SECRET_ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_VERSION}
+    external: true
 
 volumes:
   app:

entrypoint.sh.tmpl

@@ -23,14 +23,16 @@ file_env() {
   unset "$fileVar"
 }
 
-export DB_PASS=$(cat /run/secrets/db_password)
-
 # for sidekiq service bundle exec env var threading
+file_env "DB_PASS"
 file_env "OTP_SECRET"
 file_env "SECRET_KEY_BASE"
 file_env "SMTP_PASSWORD"
 file_env "VAPID_PRIVATE_KEY"
 file_env "AWS_SECRET_ACCESS_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
 
 {{ if eq (env "OIDC_ENABLED") "true" }}
 file_env "OIDC_CLIENT_SECRET"

release/3.0.0+v4.5.6-hometown-1.2.0

@@ -0,0 +1,20 @@
+!!! It is recommended to update to this version before moving on to later updates !!!
+
+- Redis Namespaces are no longer supported! If you use REDIS_NAMESPACE or CACHE_REDIS_NAMESPACE in your config, you'll have to remove it.
+
+- Three new secrets have been introduced:
+  * Add the keys to your env:
+SECRET_ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_VERSION=v1
+  * Create the keys, e.g. with `abra app secret generate -a your.app.domain`
+
+- As with every mastodon upgrade, you'll need to migrate databases by running:
+```
+abra app run your.app.domain app bash -c "SKIP_POST_DEPLOYMENT_MIGRATIONS=true rails db:migrate"
+abra app undeploy your.app.domain
+abra app deploy your.app.domain
+abra app run your.app.domain app rails db:migrate
+```
+
+See the full release notes for details: https://github.com/hometown-fork/hometown/releases/tag/v4.5.6%2Bhometown-1.2.0