Pass through AWS_SECRET_ACCESS_KEY

Reviewed-on: #33
Reviewed-by: ammaratef45 <ammaratef45@proton.me>

.drone.yml

@@ -3,10 +3,13 @@ kind: pipeline
 name: deploy to swarm-test.autonomic.zone
 steps:
   - name: deployment
-    image: decentral1se/stack-ssh-deploy:latest
+    image: git.coopcloud.tech/coop-cloud/stack-ssh-deploy:latest
     settings:
       host: swarm-test.autonomic.zone
       stack: mastodon
+      generate_secrets: true
+      networks:
+       - proxy
       purge: true
       deploy_key:
         from_secret: drone_ssh_swarm_test
@@ -14,16 +17,28 @@ steps:
       DOMAIN: mastodon.swarm-test.autonomic.zone
       STACK_NAME: mastodon
       LETS_ENCRYPT_ENV: production
+      ENTRYPOINT_CONF_VERSION: v1
+      SECRET_KEY_BASE_VERSION: v1
+      SECRET_OTP_SECRET_VERSION: v1
+      SECRET_VAPID_PRIVATE_KEY_VERSION: v1
+      SECRET_DB_PASSWORD_VERSION: v1
+      SECRET_SMTP_PASSWORD_VERSION: v1
 trigger:
   branch:
     - main
 ---
 kind: pipeline
-name: recipe release
+name: generate recipe catalogue
 steps:
   - name: release a new version
-    image: thecoopcloud/drone-abra:latest
+    image: plugins/downstream
     settings:
-      command: recipe mastodon release
-      deploy_key:
-        from_secret: abra_bot_deploy_key
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - toolshed/auto-recipes-catalogue-json
+
+trigger:
+  event: tag

.env.sample

@@ -1,6 +1,6 @@
 TYPE=mastodon
 
-DOMAIN=mastodon.swarm-test.autonomic.zone
+DOMAIN=mastodon.example.com
 # Enables WEB_DOMAIN if set (FOR FUTURE USE)
 # USER_DOMAIN=
 
@@ -13,6 +13,12 @@ LETS_ENCRYPT_ENV=production
 # Variables you *need* to change will me marked as such.
 # Most optional features are commented out/disabled and will need to be enabled by you after checking the documentation.
 
+COMPOSE_FILE="compose.yml"
+
+# Set the maximum length for toots (posts). Longer posts from other servers will still be displayed, this limit only applies to users on this instance.
+#MAX_CHARS=500
+#COMPOSE_FILE="$COMPOSE_FILE:compose.character-limit.yml"
+
 # Federation
 # ----------
 # DO NOT CHANGE DOMAIN VARIABLES AFTER DEPLOYMENT! WILL BREAK FEDERATION!!
@@ -61,9 +67,7 @@ REDIS_PORT=6379
 
 # ElasticSearch
 # --------------------------------------
-ES_ENABLED=true
-ES_HOST=es
-ES_PORT=9200
+# COMPOSE_FILE="$COMPOSE_FILE:compose.elasticsearch.yml"
 
 # StatsD (CURRENTLY NOT SUPPORTED)
 # -------------------------------
@@ -72,11 +76,15 @@ ES_PORT=9200
 
 # Secrets
 # =======
-SECRET_KEY_BASE_VERSION=v1
+SECRET_SECRET_KEY_BASE_VERSION=v1
 SECRET_OTP_SECRET_VERSION=v1
 SECRET_VAPID_PRIVATE_KEY_VERSION=v1
 SECRET_DB_PASSWORD_VERSION=v1
 SECRET_SMTP_PASSWORD_VERSION=v1
+SECRET_ARE_DETERMINISTIC_KEY_VERSION=v1
+SECRET_ARE_KEY_DERIVATION_SALT_VERSION=v1
+SECRET_ARE_PRIMARY_KEY_VERSION=v1
+SECRET_AWS_SECRET_ACCESS_KEY_VERSION=v1
 
 # Web Push
 # ========
@@ -117,10 +125,9 @@ DEFAULT_LOCALE=en
 
 # S3 and AWS
 # ----------
-# S3_ENABLED=
-# S3_BUCKET=
+# COMPOSE_FILE="$COMPOSE_FILE:compose.s3.yml"
 # AWS_ACCESS_KEY_ID=
-# AWS_SECRET_ACCESS_KEY=
+# S3_BUCKET=
 # S3_REGION=
 # S3_PROTOCOL=
 # S3_HOSTNAME=
@@ -129,6 +136,15 @@ DEFAULT_LOCALE=en
 # S3_OVERRIDE_PATH_STYLE=
 # S3_OPEN_TIMEOUT=
 # S3_READ_TIMEOUT=
+# S3_RETRY_LIMIT=
+# S3_FORCE_SINGLE_REQUEST=
+# S3_ENABLE_CHECKSUM_MODE=
+# S3_STORAGE_CLASS=
+# S3_MULTIPART_THRESHOLD=
+# S3_PERMISSION=
+# S3_BATCH_DELETE_LIMIT=
+# S3_BATCH_DELETE_RETRY=
+# S3_ALIAS_HOST=
 
 # External Authentication
 # =======================
@@ -170,7 +186,20 @@ DEFAULT_LOCALE=en
 # SAML_ATTRIBUTES_STATEMENTS_VERIFIED=
 # SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL=
 
-# Hidden services (Not Supported)
-# ===============================
-# http_proxy=  # yes, this should be lowercase
-# ALLOW_ACCESS_TO_HIDDEN_SERVICE=
+# OpenID Connect
+# ----
+# COMPOSE_FILE="$COMPOSE_FILE:compose.oidc.yml"
+# OIDC_ENABLED=true
+# OIDC_DISPLAY_NAME=authentik
+# OIDC_DISCOVERY=true
+# OIDC_ISSUER=<OpenID Configuration Issuer>
+# OIDC_AUTH_ENDPOINT=https://authentik.company/application/o/authorize/
+# OIDC_SCOPE=openid,profile,email
+# OIDC_UID_FIELD=sub
+# OIDC_CLIENT_ID=<Client ID>
+# OIDC_CLIENT_SECRET=<Client Secret>
+# OIDC_REDIRECT_URI=https://mastodon.company/auth/auth/openid_connect/callback
+# OIDC_SECURITY_ASSUME_EMAIL_IS_VERIFIED=true
+# OMNIAUTH_ONLY=true
+# ONE_CLICK_SSO_LOGIN=true
+# SECRET_OIDC_CLIENT_SECRET_VERSION=v1

README.md

@@ -1,62 +1,65 @@
 # Mastodon
 
-Your self-hosted, globally interconnected microblogging community
+> Your self-hosted, globally interconnected microblogging community
 
 <!-- metadata -->
+* **Maintainers**: `@3wordchant` (Matrix: `@3wc:autonomic.zone`), `Benjamin` (Matrix: `@benjaminlj:matrix.org`)
+* **Status**: `stable`
 * **Category**: Apps
-* **Status**: 
+* **Features**: 1
 * **Image**: [`tootsuite/mastodon`](https://hub.docker.com/r/tootsuite/mastodon)
-* **Healthcheck**: 
-* **Backups**: 
-* **Email**: 
-* **Tests**: 
-* **SSO**: 
+* **Healthcheck**: No
+* **Backups**:  No
+* **Email**: Yes
+* **Tests**: No
+* **SSO**: Yes
 <!-- endmetadata -->
 
-## Basic usage
+## Quick start
+
+Mastodon expects secrets to be formatted in a very specific way, so please
+choose "No" when prompted to generate secrets for `abra app new mastodon`. The
+secrets must be generated outside of `abra` and that is achieved in step 2. See
+the [`abra.sh`](./abra.sh) for more.
 
-1. Set up Docker Swarm and [`abra`]
-1. Deploy [`coop-cloud/traefik`]
 1. `abra app new mastodon`
-1. Follow the [secrets setup docs](#secrets-setup)
-1. `abra app YOURAPPDOMAIN config` - be sure to change `DOMAIN` to something that resolves to your Docker swarm box
-1. `abra app YOURAPPDOMAIN setup` to setup the database and create the admin account (services will stop flapping shortly after)
+1. `abra app cmd --local <domain> secrets`
+1. `abra app cmd --local <domain> secrets_activerecord`
+1. `abra app secret insert <domain> smtp_password v1 <password>`
+1. `abra app config <domain>` (uncomment SMTP details)
+1. `abra app deploy <domain>`
 
-[`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra
-[`coop-cloud/traefik`]: https://git.autonomic.zone/coop-cloud/traefik
-
-## Secrets setup
-
-Because Mastodon expects secrets generated by specific tools, we don't support that in `abra` yet. However, you can run these commands yourself using the underlying Docker CLI. You can then load them in as secrets to the swarm using `abra` though and then they will be picked up on the deployment.
-
-First, generate the `SECRET_KEY_BASE` and `OTP_SECRET` and store them in your local shell environment, you'll need them for subsequent commands.
+Then, on your host (outside of the containers), you'll need to fix permissions
+for the volume (see [#10](https://git.coopcloud.tech/coop-cloud/mastodon/issues/10)):
 
 ```
-$ SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
-$ OTP_SECRET=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
-$ abra app YOURAPPDOMAIN secret insert secret_key_base v1 $SECRET_KEY_BASE
-$ abra app YOURAPPDOMAIN secret insert otp_secret v1 $OTP_SECRET
+chown -R 991:991 /var/lib/docker/volumes/<domain>_app/_data
 ```
 
-Then you need to generate the `VAPID_{PUBLIC/PRIVATE}_KEY` values using the `SECRET_KEY_BASE`/`OTP_SECRET`:
+And finally, within the `app` container, create an admin account:
 
 ```
-$ docker run \
-  -e SECRET_KEY_BASE=$SECRET_KEY_BASE \
-  -e OTP_SECRET=$OTP_SECRET \
-  --rm tootsuite/mastodon:v3.4.0 \
-  bundle exec rake mastodon:webpush:generate_vapid_key
+abra app cmd <domain> app admin -- <username> <email>
 ```
 
-Once you see the values generated, you can load the `VAPID_PUBLIC_KEY` into your `.env` file and `VAPID_PRIVATE_KEY` into a secret.
+## Tips & tricks
+
+### Auto-complete is not working?
+
+Check the sidekiq logs (`/sidekiq/retries`), is a bunch of stuff failing? What
+is the error?
+
+If it looks anything like `blocked by: [FORBIDDEN/12/index read-only / allow
+delete (api)];` then it might mean that your elastic search service has put
+itself into "read-only" state. This could be due to running close to no free
+disk space one time. ES doesn't undo this state, even when you have more free
+disk space once more, so you need to handle this manually:
 
 ```
-$ abra app YOURDOMAIN secret insert vapid_private_key v1 YOURVAPIDPRIVATEKEY
+abra app run <domain> es bash
+curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}'
 ```
 
-And finally, to end your whirlwind secrets loading adventure, get the `DB_PASS` and `SMTP_PASSWORD` loaded.
-
-```
-$ abra app YOURAPPDOMAIN secret generate db_password v1
-$ abra app YOURDOMAIN secret insert smtp_password v1 YOURSMTPPASSWORD
-```
+Then head back to the sidekiq retries panel and retry one job. You should see
+the ticket of retries go down by one if if passed. Then you can "retry all" and
+they should get scheduled & run.

abra.sh

@@ -1 +1,150 @@
-export ENTRYPOINT_CONF_VERSION=v1
+export ENTRYPOINT_CONF_VERSION=v7
+export ENTRYPOINT_STREAMING_CONF_VERSION=v2
+
+grep=grep
+if ! $grep -P --version 2>/dev/null 1>/dev/null
+then
+    echo "$grep doesn't have -P option, trying ggrep"
+    grep=ggrep
+    if ! $grep -P --version 2>/dev/null 1>/dev/null
+    then
+      echo "If you're on a mac try running \`brew install grep\`"
+      exit 1
+    fi
+fi
+
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  declare -x -g "$var"="$val"
+  unset "$fileVar"
+}
+
+environment() {
+    # for sidekiq service bundle exec env var threading
+    file_env "DB_PASS"
+    file_env "OTP_SECRET"
+    file_env "SECRET_KEY_BASE"
+    file_env "VAPID_PRIVATE_KEY"
+    file_env "AWS_SECRET_ACCESS_KEY"
+    file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+    file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+    file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+
+    declare -x RAILS_ENV=production
+}
+
+assets() {
+  environment
+
+  bundle exec rails assets:precompile
+}
+
+admin() {
+  environment
+
+  bin/tootctl accounts create "$1" --email "$2" --confirmed --role Owner
+  bin/tootctl accounts approve "$1"
+}
+
+shell() {
+    ## Run a shell with proper environment
+    environment
+    bash $@
+}
+
+secrets() {
+  set -e
+
+  docker context use default > /dev/null 2>&1
+
+  MASTO_VERSION="v4.3.8"
+
+  echo "Generating secrets for a new Mastodon deployment..."
+  echo ""
+
+  SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:$MASTO_VERSION bundle exec rails secret)
+  abra app secret insert "$APP_NAME" secret_key_base v1 "$SECRET_KEY_BASE"
+  echo "SECRET_KEY_BASE = $SECRET_KEY_BASE"
+  echo ""
+
+  OTP_SECRET=$(docker run --rm tootsuite/mastodon:$MASTO_VERSION bundle exec rails secret)
+  abra app secret insert "$APP_NAME" otp_secret v1 "$OTP_SECRET"
+  echo "OTP_SECRET = $OTP_SECRET"
+  echo ""
+
+  docker run \
+    -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \
+    -e OTP_SECRET="$OTP_SECRET" \
+    --rm tootsuite/mastodon:$MASTO_VERSION \
+    bundle exec rake mastodon:webpush:generate_vapid_key \
+    > /tmp/key.txt
+
+  VAPID_PRIVATE_KEY=$($grep -oP "VAPID_PRIVATE_KEY=\K.+" "/tmp/key.txt")
+  VAPID_PUBLIC_KEY=$($grep -oP "VAPID_PUBLIC_KEY=\K.+" "/tmp/key.txt")
+  rm -rf /tmp/key.txt
+
+  echo "VAPID_PUBLIC_KEY = $VAPID_PUBLIC_KEY"
+  echo "!IMPORTANT! you MUST insert this VAPID_PUBLIC_KEY into your app .env config !IMPORTANT!"
+  echo ""
+
+  abra app secret insert "$APP_NAME" vapid_private_key v1 "$VAPID_PRIVATE_KEY"
+  echo "VAPID_PRIVATE_KEY = $VAPID_PRIVATE_KEY"
+  echo ""
+
+  abra app secret generate "$APP_NAME" db_password v1
+  echo ""
+
+  echo "don't forget to insert your smtp_password! your deployment won't work without it"
+  echo "run \"abra app secret insert $APP_NAME smtp_password v1 YOURSMTPPASSWORD\""
+  echo ""
+}
+
+secrets_activerecord() {
+  set -e
+
+  docker context use default > /dev/null 2>&1
+
+  MASTO_VERSION="v4.3.8"
+
+  echo "Generating activerecord secrets for an updated deployment"
+  echo ""
+
+  docker run \
+    -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \
+    -e OTP_SECRET="$OTP_SECRET" \
+    --rm tootsuite/mastodon:$MASTO_VERSION \
+    bundle exec rake db:encryption:init \
+    > /tmp/activerecord.txt
+
+  ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY=$($grep ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY /tmp/activerecord.txt | cut -d'=' -f2)
+  abra app secret insert "$APP_NAME" are_deterministic_key v1 "$ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+  echo "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY = $ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+  echo ""
+
+  ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT=$($grep ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT /tmp/activerecord.txt | cut -d'=' -f2)
+  abra app secret insert "$APP_NAME" are_key_derivation_salt v1 "$ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+  echo "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT = $ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+  echo ""
+
+  ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY=$($grep ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY /tmp/activerecord.txt | cut -d'=' -f2)
+  abra app secret insert "$APP_NAME" are_primary_key v1 "$ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+  echo "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY = $ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+  echo ""
+
+  rm -rf /tmp/activerecord.txt
+}

compose.character-limit.yml

@@ -0,0 +1,12 @@
+---
+version: "3.8"
+
+services:
+  app:
+    environment:
+    - MAX_CHARS
+    # NOTE: See [0] for background on why this is necessary
+    # [0]: See https://github.com/mastodon/mastodon/pull/30091
+    user: root
+    command: >
+      /bin/sh -c 'set -x && ls && sed -i -e "s/500/$MAX_CHARS/g" app/javascript/mastodon/features/compose/components/compose_form.jsx && sed -i -e "s/500/$MAX_CHARS/g" app/validators/status_length_validator.rb && rm -f /mastodon/tmp/pids/server.pid && su -c "RAILS_ENV=production bundle exec rails s -p 3000"'

compose.elasticsearch.yml

@@ -0,0 +1,34 @@
+---
+version: "3.8"
+
+services:
+  es:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "cluster.name=es-mastodon"
+      - "discovery.type=single-node"
+      - "bootstrap.memory_lock=true"
+    networks:
+      - internal
+    volumes:
+      - es:/usr/share/elasticsearch/data
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
+  app:
+    environment: &es-env
+      - "ES_ENABLED=true"
+      - "ES_HOST=es"
+      - "ES_PORT=9200"
+
+  streaming:
+    environment: *es-env
+
+  sidekiq:
+    environment: *es-env
+
+volumes:
+  es:

compose.oidc.yml

@@ -0,0 +1,26 @@
+---
+version: "3.8"
+
+services:
+  app:
+    environment:
+      - OIDC_ENABLED
+      - OIDC_DISPLAY_NAME
+      - OIDC_DISCOVERY
+      - OIDC_ISSUER
+      - OIDC_AUTH_ENDPOINT
+      - OIDC_SCOPE
+      - OIDC_UID_FIELD
+      - OIDC_CLIENT_ID
+      - OIDC_REDIRECT_URI
+      - OIDC_SECURITY_ASSUME_EMAIL_IS_VERIFIED
+      - OMNIAUTH_ONLY
+      - ONE_CLICK_SSO_LOGIN
+      - OIDC_CLIENT_SECRET_FILE=/run/secrets/oidc_client_secret
+    secrets:
+      - oidc_client_secret
+
+secrets:
+  oidc_client_secret:
+    name: ${STACK_NAME}_oidc_client_secret_${SECRET_OIDC_CLIENT_SECRET_VERSION}
+    external: true

compose.s3.yml

@@ -0,0 +1,42 @@
+---
+version: "3.8"
+
+services:
+  app:
+    environment: &s3-env
+      - S3_ENABLED=true
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY_FILE=/run/secrets/aws_secret_access_key
+      - S3_BUCKET
+      - S3_REGION
+      - S3_PROTOCOL
+      - S3_HOSTNAME
+      - S3_ENDPOINT
+      - S3_SIGNATURE_VERSION
+      - S3_OVERRIDE_PATH_STYLE
+      - S3_OPEN_TIMEOUT
+      - S3_READ_TIMEOUT
+      - S3_RETRY_LIMIT
+      - S3_FORCE_SINGLE_REQUEST
+      - S3_ENABLE_CHECKSUM_MODE
+      - S3_STORAGE_CLASS
+      - S3_MULTIPART_THRESHOLD
+      - S3_PERMISSION
+      - S3_BATCH_DELETE_LIMIT
+      - S3_BATCH_DELETE_RETRY
+      - S3_ALIAS_HOST
+    secrets: &s3-secrets
+      - aws_secret_access_key
+
+  streaming:
+    environment: *s3-env
+    secrets: *s3-secrets
+
+  sidekiq:
+    environment: *s3-env
+    secrets: *s3-secrets
+
+secrets:
+  aws_secret_access_key:
+    name: ${STACK_NAME}_aws_secret_access_key_${SECRET_AWS_SECRET_ACCESS_KEY_VERSION}
+    external: true

compose.yml

@@ -3,11 +3,12 @@ version: "3.8"
 
 services:
   app:
-    image: tootsuite/mastodon:v3.4.6
-    command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
+    image: tootsuite/mastodon:v4.5.3
+    command: |
+      bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
     networks: &bothNetworks
       - proxy
-      - internal_network
+      - internal
     deploy:
       update_config:
         failure_action: rollback
@@ -19,11 +20,14 @@ services:
         - "traefik.http.routers.${STACK_NAME}_web.rule=Host(`${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}_web.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}_web.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=0.1.1+3.4.6"
+        - "coop-cloud.${STACK_NAME}.version=2.1.3+v4.5.3"
     configs: &configs
       - source: entrypoint_sh
         target: /usr/local/bin/entrypoint.sh
         mode: 0555
+      - source: entrypoint_streaming_sh
+        target: /usr/local/bin/entrypoint_streaming.sh
+        mode: 0555
     entrypoint: &entrypoint /usr/local/bin/entrypoint.sh
     volumes: &appVolume
       - app:/opt/mastodon/public/system
@@ -33,134 +37,102 @@ services:
       - secret_key_base
       - smtp_password
       - vapid_private_key
+      - are_deterministic_key
+      - are_key_derivation_salt
+      - are_primary_key
     environment: &env
-      - DB_HOST
-      - DB_USER
-      - DB_NAME
-      - DB_PASS_FILE=/run/secrets/db_password
-      - DB_PORT
-      - REDIS_HOST
-      - REDIS_PORT
-      - REDIS_URL
-      - REDIS_NAMESPACE
+      - ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY_FILE=/run/secrets/are_deterministic_key
+      - ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT_FILE=/run/secrets/are_key_derivation_salt
+      - ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY_FILE=/run/secrets/are_primary_key
+      - ALLOW_ACCESS_TO_HIDDEN_SERVICE
+      - ALTERNATE_DOMAINS
+      - AUTHORIZED_FETCH
       - CACHE_REDIS_HOST
+      - CACHE_REDIS_NAMESPACE
       - CACHE_REDIS_PORT
       - CACHE_REDIS_URL
-      - CACHE_REDIS_NAMESPACE
+      - DB_HOST
+      - DB_NAME
+      - DB_PORT
+      - DB_USER
+      - DB_PASS_FILE=/run/secrets/db_password
+      - DEFAULT_LOCALE
+      - EMAIL_DOMAIN_ALLOWLIST
+      - EMAIL_DOMAIN_DENYLIST
       - ES_ENABLED
       - ES_HOST
       - ES_PORT
-      - STATSD_ADDR
-      - STATSD_NAMESPACE
-      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
-      - VAPID_PUBLIC_KEY
-      - OTP_SECRET_FILE=/run/secrets/otp_secret
-      - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base
-      - LOCAL_DOMAIN
-      - WEB_DOMAIN
-      - ALTERNATE_DOMAINS
-      - AUTHORIZED_FETCH
-      - LIMITED_FEDERATION_MODE
-      - RAILS_ENV
-      - RAILS_SERVE_STATIC_FILES
-      - SINGLE_USER_MODE
-      - EMAIL_DOMAIN_ALLOWLIST
-      - EMAIL_DOMAIN_DENYLIST
-      - DEFAULT_LOCALE
-      - MAX_SESSION_ACTIVATIONS
-      - USER_ACTIVE_DAYS
-      - SMTP_SERVER
-      - SMTP_PORT
-      - SMTP_LOGIN
-      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
-      - SMTP_FROM_ADDRESS
-      - SMTP_DOMAIN
-      - SMTP_DELIVERY_METHOD
-      - SMTP_AUTH_METHOD
-      - SMTP_CA_FILE
-      - SMTP_OPENSSL_VERIFY_MODE
-      - SMTP_ENABLE_STARTTLS_AUTO
-      - SMTP_TLS
-      - SMTP_SSL
-      - PAPERCLIP_ROOT_PATH
-      - PAPERCLIP_ROOT_URL
-      - OAUTH_REDIRECT_AT_SIGN_IN
-      - LDAP_ENABLED
-      - LDAP_HOST
-      - LDAP_PORT
-      - LDAP_METHOD
       - LDAP_BASE
       - LDAP_BIND_DN
-      - LDAP_PASSWORD
-      - LDAP_UID
-      - LDAP_SEARCH_FILTER
+      - LDAP_ENABLED
+      - LDAP_HOST
       - LDAP_MAIL
+      - LDAP_METHOD
+      - LDAP_PASSWORD
+      - LDAP_PORT
+      - LDAP_SEARCH_FILTER
+      - LDAP_UID
       - LDAP_UID_CONVERSTION_ENABLED
-      - SAML_ENABLED
+      - LIMITED_FEDERATION_MODE
+      - LOCAL_DOMAIN
+      - MAX_SESSION_ACTIVATIONS
+      - OAUTH_REDIRECT_AT_SIGN_IN
+      - OTP_SECRET_FILE=/run/secrets/otp_secret
+      - PAPERCLIP_ROOT_PATH
+      - PAPERCLIP_ROOT_URL
+      - RAILS_ENV
+      - RAILS_SERVE_STATIC_FILES
+      - REDIS_HOST
+      - REDIS_NAMESPACE
+      - REDIS_PORT
+      - REDIS_URL
       - SAML_ACS_URL
-      - SAML_ISSUER
-      - SAML_IDP_SSO_TARGET_URL
-      - SAML_IDP_CERT
-      - SAML_IDP_CERT_FINGERPRINT
-      - SAML_NAME_IDENTIFIER_FORMAT
-      - SAML_CERT
-      - SAML_PRIVATE_KEY
-      - SAML_SECURITY_WANT_ASSERTION_SIGNED
-      - SAML_SECURITY_WANT_ASSERTION_ENCRYPTED
-      - SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED
-      - SAML_ATTRIBUTES_STATEMENTS_UID
       - SAML_ATTRIBUTES_STATEMENTS_EMAIL
-      - SAML_ATTRIBUTES_STATEMENTS_FULL_NAME
       - SAML_ATTRIBUTES_STATEMENTS_FIRST_NAME
+      - SAML_ATTRIBUTES_STATEMENTS_FULL_NAME
       - SAML_ATTRIBUTES_STATEMENTS_LAST_NAME
-      - SAML_UID_ATTRIBUTE
+      - SAML_ATTRIBUTES_STATEMENTS_UID
       - SAML_ATTRIBUTES_STATEMENTS_VERIFIED
       - SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL
-      - http_proxy # yes, this should be lowercase
-      - ALLOW_ACCESS_TO_HIDDEN_SERVICE
-
-  db:
-    image: postgres:9.6-alpine
-    networks: &internalNetwork
-      - internal_network
-    volumes:
-      - postgres:/var/lib/postgresql/data
-    secrets:
-      - db_password
-    environment:
-      - POSTGRES_DB=${DB_NAME}
-      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
-      - POSTGRES_USER=${DB_USER}
-
-  redis:
-    image: redis:6.2-alpine
-    networks: *internalNetwork
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-    volumes:
-      - redis:/data
-
-  es:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.17
-    environment:
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "cluster.name=es-mastodon"
-      - "discovery.type=single-node"
-      - "bootstrap.memory_lock=true"
-    networks:
-      - internal_network
-    volumes:
-      - es:/usr/share/elasticsearch/data
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
+      - SAML_CERT
+      - SAML_ENABLED
+      - SAML_IDP_CERT
+      - SAML_IDP_CERT_FINGERPRINT
+      - SAML_IDP_SSO_TARGET_URL
+      - SAML_ISSUER
+      - SAML_NAME_IDENTIFIER_FORMAT
+      - SAML_PRIVATE_KEY
+      - SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED
+      - SAML_SECURITY_WANT_ASSERTION_ENCRYPTED
+      - SAML_SECURITY_WANT_ASSERTION_SIGNED
+      - SAML_UID_ATTRIBUTE
+      - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base
+      - SINGLE_USER_MODE
+      - SMTP_AUTH_METHOD
+      - SMTP_CA_FILE
+      - SMTP_DELIVERY_METHOD
+      - SMTP_DOMAIN
+      - SMTP_ENABLE_STARTTLS_AUTO
+      - SMTP_FROM_ADDRESS
+      - SMTP_LOGIN
+      - SMTP_OPENSSL_VERIFY_MODE
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
+      - SMTP_PORT
+      - SMTP_SERVER
+      - SMTP_SSL
+      - SMTP_TLS
+      - STATSD_ADDR
+      - STATSD_NAMESPACE
+      - USER_ACTIVE_DAYS
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
+      - VAPID_PUBLIC_KEY
+      - WEB_DOMAIN
 
   streaming:
-    image: tootsuite/mastodon:v3.4.6
-    command: node ./streaming
+    image: tootsuite/mastodon-streaming:v4.5.3
+    command: node ./streaming/index.js
     configs: *configs
-    entrypoint: *entrypoint
+    entrypoint: /usr/local/bin/entrypoint_streaming.sh
     secrets: *secrets
     networks: *bothNetworks
     deploy:
@@ -175,10 +147,10 @@ services:
         - "traefik.http.routers.${STACK_NAME}_streaming.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}_streaming.tls.certresolver=${LETS_ENCRYPT_ENV}"
     environment: *env
-    volumes: *appVolume # used to make sure this volume is created
+    volumes: *appVolume
 
   sidekiq:
-    image: tootsuite/mastodon:v3.4.6
+    image: tootsuite/mastodon:v4.5.3
     secrets: *secrets
     command: bundle exec sidekiq
     configs: *configs
@@ -191,9 +163,31 @@ services:
     volumes: *appVolume
     environment: *env
 
+  db:
+    image: pgautoupgrade/pgautoupgrade:18-alpine
+    networks: &internalNetwork
+      - internal
+    volumes:
+      - postgres:/var/lib/postgresql
+
+    secrets:
+      - db_password
+    environment:
+      - POSTGRES_DB=${DB_NAME}
+      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
+      - POSTGRES_USER=${DB_USER}
+
+  redis:
+    image: redis:8.4-alpine
+    networks: *internalNetwork
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+    volumes:
+      - redis:/data
+
 secrets:
   secret_key_base:
-    name: ${STACK_NAME}_secret_key_base_${SECRET_KEY_BASE_VERSION}
+    name: ${STACK_NAME}_secret_key_base_${SECRET_SECRET_KEY_BASE_VERSION}
     external: true
   otp_secret:
     name: ${STACK_NAME}_otp_secret_${SECRET_OTP_SECRET_VERSION}
@@ -207,21 +201,32 @@ secrets:
   smtp_password:
     name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
     external: true
+  are_deterministic_key:
+    name: ${STACK_NAME}_are_deterministic_key_${SECRET_ARE_DETERMINISTIC_KEY_VERSION}
+    external: true
+  are_key_derivation_salt:
+    name: ${STACK_NAME}_are_key_derivation_salt_${SECRET_ARE_KEY_DERIVATION_SALT_VERSION}
+    external: true
+  are_primary_key:
+    name: ${STACK_NAME}_are_primary_key_${SECRET_ARE_PRIMARY_KEY_VERSION}
+    external: true
 
 volumes:
   app:
   redis:
   postgres:
-  es:
 
 networks:
+  internal:
   proxy:
     external: true
-  internal_network:
-    internal: true
 
 configs:
   entrypoint_sh:
     name: ${STACK_NAME}_entrypoint_conf_${ENTRYPOINT_CONF_VERSION}
     file: entrypoint.sh.tmpl
     template_driver: golang
+  entrypoint_streaming_sh:
+    name: ${STACK_NAME}_entrypoint_streaming_conf_${ENTRYPOINT_STREAMING_CONF_VERSION}
+    file: entrypoint-streaming.sh.tmpl
+    template_driver: golang

entrypoint-streaming.sh.tmpl

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -eu
+
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  declare -x -g "$var"="$val"
+  unset "$fileVar"
+}
+
+file_env "DB_PASS"
+file_env "OTP_SECRET"
+file_env "SECRET_KEY_BASE"
+file_env "VAPID_PRIVATE_KEY"
+file_env "AWS_SECRET_ACCESS_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
+
+/usr/local/bin/docker-entrypoint.sh "$@"

entrypoint.sh.tmpl

@@ -19,14 +19,28 @@ file_env() {
     val="$(< "${!fileVar}")"
   fi
 
-  export "$var"="$val"
+  declare -x -g "$var"="$val"
   unset "$fileVar"
 }
 
 file_env "DB_PASS"
+file_env "SMTP_PASSWORD"
 file_env "OTP_SECRET"
 file_env "SECRET_KEY_BASE"
-file_env "SMTP_PASSWORD"
 file_env "VAPID_PRIVATE_KEY"
+file_env "AWS_SECRET_ACCESS_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY"
+file_env "ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT"
+file_env "ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY"
 
-/usr/bin/tini -- "$@"
+{{ if eq (env "OIDC_ENABLED") "true" }}
+file_env "OIDC_CLIENT_SECRET"
+{{ end }}
+
+
+# NOTE: this was working in mastodon 4.2 but breaks in 4.3
+# sed -i '/- admin$/d' /opt/mastodon/config/settings.yml
+
+RAILS_ENV=production bundle exec rake db:migrate
+
+/usr/bin/tini -s -- "$@"

release/0.2.0+v4.3.1

@@ -0,0 +1 @@
+upgrade apps

release/1.0.0+v4.3.1

@@ -0,0 +1,11 @@
+WARNING❗ This release requires adding new secrets, and will not deploy successfully without them.
+
+Run `abra app cmd --local <domain> secrets_activerecord` to generate and store these secrets.
+
+You will also need to add this to your config (`abra app config <domain>`):
+
+```
+SECRET_ARE_DETERMINISTIC_KEY_VERSION=v1
+SECRET_ARE_KEY_DERIVATION_SALT_VERSION=v1
+SECRET_ARE_PRIMARY_KEY_VERSION=v1
+```

release/1.1.0+v4.3.8

@@ -0,0 +1 @@
+MAX_CHARS now supported; see ~/.abra/recipes/.env.sample

release/2.0.0+v4.4.7

@@ -0,0 +1 @@
+This release includes several Postgres major version upgrades; this should work automatically, but please take extra care to make a database backup before upgrading!

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}