OIDC secret storage, separate streaming entrypoint

2024-11-21 22:44:45 -05:00 · 2024-11-21 22:44:45 -05:00 · 95f295ed5c
commit 95f295ed5c
parent f70ec17004
7 changed files with 205 additions and 25 deletions
--- a/.env.sample
+++ b/.env.sample
@ -79,6 +79,9 @@ SECRET_OTP_SECRET_VERSION=v1
 SECRET_VAPID_PRIVATE_KEY_VERSION=v1
 SECRET_DB_PASSWORD_VERSION=v1
 SECRET_SMTP_PASSWORD_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_VERSION=v1

 # Web Push
 # ========
@ -187,4 +190,5 @@ DEFAULT_LOCALE=en
 # OIDC_REDIRECT_URI=https://mastodon.company/auth/auth/openid_connect/callback
 # OIDC_SECURITY_ASSUME_EMAIL_IS_VERIFIED=true
 # OMNIAUTH_ONLY=true
-# ONE_CLICK_SSO_LOGIN
+# ONE_CLICK_SSO_LOGIN=true
+# SECRET_OIDC_CLIENT_SECRET_VERSION=v1
--- a/abra.sh
+++ b/abra.sh
@ -1,25 +1,60 @@
-export ENTRYPOINT_CONF_VERSION=v3
+export ENTRYPOINT_CONF_VERSION=v5
+export ENTRYPOINT_STREAMING_CONF_VERSION=v1
+
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  declare -x -g "$var"="$val"
+  unset "$fileVar"
+}
+
+environment() {
+    # for sidekiq service bundle exec env var threading
+    file_env "DB_PASS"
+    file_env "OTP_SECRET"
+    file_env "SECRET_KEY_BASE"
+    file_env "VAPID_PRIVATE_KEY"
+    file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+    file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+    file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+
+    declare -x RAILS_ENV=production
+}

 assets() {
-  export OTP_SECRET=$(cat /run/secrets/otp_secret)
-  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
-  export DB_PASS=$(cat /run/secrets/db_password)
+  environment

-  RAILS_ENV=production bundle exec rails assets:precompile
+  bundle exec rails assets:precompile
 }

 admin() {
-  export OTP_SECRET=$(cat /run/secrets/otp_secret)
-  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
-  export DB_PASS=$(cat /run/secrets/db_password)
+  environment

-  RAILS_ENV=production bin/tootctl accounts create "$1" --email "$2" --confirmed --role Owner
+  bin/tootctl accounts create "$1" --email "$2" --confirmed --role Owner
+}
+
+shell() {
+    ## Run a shell with proper environment
+    environment
+    bash $@
 }

 setup() {
-  export OTP_SECRET=$(cat /run/secrets/otp_secret)
-  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
-  export DB_PASS=$(cat /run/secrets/db_password)
+  environment

  RAILS_ENV=production bundle exec rake db:setup
 }
@ -29,7 +64,7 @@ secrets() {

  docker context use default > /dev/null 2>&1

-  MASTO_VERSION="v4.1.0"
+  MASTO_VERSION="v4.3.1"

  echo "Generating secrets for a new Mastodon deployment..."
  echo ""
@ -70,3 +105,38 @@ secrets() {
  echo "run \"abra app secret insert $APP_NAME smtp_password v1 YOURSMTPPASSWORD\""
  echo ""
 }
+
+secrets_activerecord() {
+  set -e
+
+  docker context use default > /dev/null 2>&1
+
+  MASTO_VERSION="v4.3.1"
+
+  echo "Generating activerecord secrets for an updated deployment"
+  echo ""
+
+  docker run \
+    -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \
+    -e OTP_SECRET="$OTP_SECRET" \
+    --rm tootsuite/mastodon:$MASTO_VERSION \
+    bundle exec rake db:encryption:init \
+    > /tmp/activerecord.txt
+
+  ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=$(grep ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY /tmp/activerecord.txt | cut -d'=' -f2)
+  abra app secret insert "$APP_NAME" active_record_encryption_deterministic_key v1 "$ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+  echo "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY = $ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+  echo ""
+
+  ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=$(grep ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT /tmp/activerecord.txt | cut -d'=' -f2)
+  abra app secret insert "$APP_NAME" active_record_encryption_key_derivation_salt v1 "$ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+  echo "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT = $ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+  echo ""
+
+  ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=$(grep ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY /tmp/activerecord.txt | cut -d'=' -f2)
+  abra app secret insert "$APP_NAME" active_record_encryption_primary_key v1 "$ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+  echo "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY = $ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+  echo ""
+
+  rm -rf /tmp/activerecord.txt
+}
--- a/compose.oidc.yml
+++ b/compose.oidc.yml
@ -12,8 +12,15 @@ services:
      - OIDC_SCOPE
      - OIDC_UID_FIELD
      - OIDC_CLIENT_ID
-      - OIDC_CLIENT_SECRET
      - OIDC_REDIRECT_URI
      - OIDC_SECURITY_ASSUME_EMAIL_IS_VERIFIED
      - OMNIAUTH_ONLY
      - ONE_CLICK_SSO_LOGIN
+      - OIDC_CLIENT_SECRET_FILE=/run/secrets/oidc_client_secret
+    secrets:
+      - oidc_client_secret
+
+secrets:
+  oidc_client_secret:
+    name: ${STACK_NAME}_oidc_client_secret_${SECRET_OIDC_CLIENT_SECRET_VERSION}
+    external: true
--- a/compose.yml
+++ b/compose.yml
@ -3,7 +3,7 @@ version: "3.8"

 services:
  app:
-    image: tootsuite/mastodon:v4.2.13
+    image: tootsuite/mastodon:v4.3.1
    command: |
      bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
    networks: &bothNetworks
@ -25,6 +25,9 @@ services:
      - source: entrypoint_sh
        target: /usr/local/bin/entrypoint.sh
        mode: 0555
+      - source: entrypoint_streaming_sh
+        target: /usr/local/bin/entrypoint_streaming.sh
+        mode: 0555
    entrypoint: &entrypoint /usr/local/bin/entrypoint.sh
    volumes: &appVolume
      - app:/opt/mastodon/public/system
@ -34,7 +37,13 @@ services:
      - secret_key_base
      - smtp_password
      - vapid_private_key
+      - active_record_encryption_deterministic_key
+      - active_record_encryption_key_derivation_salt
+      - active_record_encryption_primary_key
    environment: &env
+      - ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_FILE=/run/secrets/active_record_encryption_deterministic_key
+      - ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_FILE=/run/secrets/active_record_encryption_key_derivation_salt
+      - ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_FILE=/run/secrets/active_record_encryption_primary_key
      - ALLOW_ACCESS_TO_HIDDEN_SERVICE
      - ALTERNATE_DOMAINS
      - AUTHORIZED_FETCH
@ -46,6 +55,7 @@ services:
      - DB_NAME
      - DB_PORT
      - DB_USER
+      - DB_PASS_FILE=/run/secrets/db_password
      - DEFAULT_LOCALE
      - EMAIL_DOMAIN_ALLOWLIST
      - EMAIL_DOMAIN_DENYLIST
@ -119,10 +129,10 @@ services:
      - WEB_DOMAIN

  streaming:
-    image: tootsuite/mastodon:v4.2.13
-    command: node ./streaming
+    image: tootsuite/mastodon-streaming:v4.3.1
+    command: node ./streaming/index.js
    configs: *configs
-    entrypoint: *entrypoint
+    entrypoint: /usr/local/bin/entrypoint_streaming.sh
    secrets: *secrets
    networks: *bothNetworks
    deploy:
@ -140,7 +150,7 @@ services:
    volumes: *appVolume

  sidekiq:
-    image: tootsuite/mastodon:v4.2.13
+    image: tootsuite/mastodon:v4.3.1
    secrets: *secrets
    command: bundle exec sidekiq
    configs: *configs
@ -206,6 +216,15 @@ secrets:
  smtp_password:
    name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
    external: true
+  active_record_encryption_deterministic_key:
+    name: ${STACK_NAME}_active_record_encryption_deterministic_key_${SECRET_ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_VERSION}
+    external: true
+  active_record_encryption_key_derivation_salt:
+    name: ${STACK_NAME}_active_record_encryption_key_derivation_salt_${SECRET_ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_VERSION}
+    external: true
+  active_record_encryption_primary_key:
+    name: ${STACK_NAME}_active_record_encryption_primary_key_${SECRET_ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_VERSION}
+    external: true

 volumes:
  app:
@ -223,3 +242,7 @@ configs:
    name: ${STACK_NAME}_entrypoint_conf_${ENTRYPOINT_CONF_VERSION}
    file: entrypoint.sh.tmpl
    template_driver: golang
+  entrypoint_streaming_sh:
+    name: ${STACK_NAME}_entrypoint_streaming_conf_${ENTRYPOINT_STREAMING_CONF_VERSION}
+    file: entrypoint-streaming.sh.tmpl
+    template_driver: golang
--- a/entrypoint-streaming.sh.tmpl
+++ b/entrypoint-streaming.sh.tmpl
@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -eu
+
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  declare -x -g "$var"="$val"
+  unset "$fileVar"
+}
+
+file_env "DB_PASS"
+file_env "OTP_SECRET"
+file_env "SECRET_KEY_BASE"
+file_env "VAPID_PRIVATE_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+
+/usr/local/bin/docker-entrypoint.sh "$@"
--- a/entrypoint.sh.tmpl
+++ b/entrypoint.sh.tmpl
@ -2,11 +2,42 @@

 set -eu

-export DB_PASS=$(cat /run/secrets/db_password)
-export OTP_SECRET=$(cat $OTP_SECRET_FILE)
-export SECRET_KEY_BASE=$(cat $SECRET_KEY_BASE_FILE)
-export VAPID_PRIVATE_KEY=$(cat $VAPID_PRIVATE_KEY_FILE)
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"

-sed -i '/- admin$/d' /opt/mastodon/config/settings.yml
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  declare -x -g "$var"="$val"
+  unset "$fileVar"
+}
+
+file_env "DB_PASS"
+file_env "SMTP_PASSWORD"
+file_env "OTP_SECRET"
+file_env "SECRET_KEY_BASE"
+file_env "VAPID_PRIVATE_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+
+{{ if eq (env "OIDC_ENABLED") "true" }}
+file_env "OIDC_CLIENT_SECRET"
+{{ end }}
+
+
+# NOTE: this was working in mastodon 4.2 but breaks in 4.3
+# sed -i '/- admin$/d' /opt/mastodon/config/settings.yml

 /usr/bin/tini -s -- "$@"
--- a/release/next
+++ b/release/next
@ -0,0 +1,11 @@
+WARNING❗ This release requires adding new secrets, and will not deploy successfully without them.
+
+Run `abra app cmd --local <domain> secrets_activerecord` to generate and store these secrets.
+
+You will also need to add this to your config (`abra app config <domain>`):
+
+```
+SECRET_ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_VERSION=v1
+SECRET_ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_VERSION=v1
+```