wip: v4.x & major config refactor

2023-02-15 01:30:24 +01:00 · 2023-02-15 01:30:24 +01:00 · 2f8dcad8aa
parent 4e9990f351
commit 2f8dcad8aa
4 changed files with 187 additions and 168 deletions
--- a/.env.sample
+++ b/.env.sample
@ -72,7 +72,7 @@ ES_PORT=9200

 # Secrets
 # =======
-SECRET_KEY_BASE_VERSION=v1
+SECRET_SECRET_KEY_BASE_VERSION=v1
 SECRET_OTP_SECRET_VERSION=v1
 SECRET_VAPID_PRIVATE_KEY_VERSION=v1
 SECRET_DB_PASSWORD_VERSION=v1
@ -169,8 +169,3 @@ DEFAULT_LOCALE=en
 # SAML_UID_ATTRIBUTE=
 # SAML_ATTRIBUTES_STATEMENTS_VERIFIED=
 # SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL=
-
-# Hidden services (Not Supported)
-# ===============================
-# http_proxy=  # yes, this should be lowercase
-# ALLOW_ACCESS_TO_HIDDEN_SERVICE=
--- a/README.md
+++ b/README.md
@ -1,62 +1,32 @@
 # Mastodon

-Your self-hosted, globally interconnected microblogging community
+> Your self-hosted, globally interconnected microblogging community

 <!-- metadata -->
 * **Category**: Apps
-* **Status**: 
+* **Status**: 1
 * **Image**: [`tootsuite/mastodon`](https://hub.docker.com/r/tootsuite/mastodon)
-* **Healthcheck**: 
-* **Backups**: 
-* **Email**: 
-* **Tests**: 
-* **SSO**: 
+* **Healthcheck**: No
+* **Backups**:  No
+* **Email**: Yes
+* **Tests**: No
+* **SSO**: Yes
 <!-- endmetadata -->

-## Basic usage
+## Quick start
+
+Mastodon expects secrets to be formatted in a very specific way, so please
+choose "No" when prompted to generate secrets for `abra app new mastodon`. The
+secrets must be generated outside of `abra` and that is achieved in step 2. See
+the [`abra.sh`](./abra.sh) for more.

-1. Set up Docker Swarm and [`abra`]
-1. Deploy [`coop-cloud/traefik`]
 1. `abra app new mastodon`
-1. Follow the [secrets setup docs](#secrets-setup)
-1. `abra app config YOURAPPDOMAIN` - be sure to change `DOMAIN` to something that resolves to your Docker swarm box
-1. `abra app setup YOURAPPDOMAIN` to setup the database and create the admin account (services will stop flapping shortly after)
+1. `abra app cmd <domain> secrets --local`
+1. `abra app config <domain>`
+1. `abra app deploy <domain>`

-[`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra
-[`coop-cloud/traefik`]: https://git.autonomic.zone/coop-cloud/traefik
-
-## Secrets setup
-
-Because Mastodon expects secrets generated by specific tools, we don't support that in `abra` yet. However, you can run these commands yourself using the underlying Docker CLI. You can then load them in as secrets to the swarm using `abra` though and then they will be picked up on the deployment.
-
-First, generate the `SECRET_KEY_BASE` and `OTP_SECRET` and store them in your local shell environment, you'll need them for subsequent commands.
+You can insert the `smtp_password` with the following:

 ```
-$ SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
-$ OTP_SECRET=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
-$ abra app secret insert YOURAPPDOMAIN secret_key_base v1 $SECRET_KEY_BASE
-$ abra app secret insert YOURAPPDOMAIN otp_secret v1 $OTP_SECRET
-```
-
-Then you need to generate the `VAPID_{PUBLIC/PRIVATE}_KEY` values using the `SECRET_KEY_BASE`/`OTP_SECRET`:
-
-```
-$ docker run \
-  -e SECRET_KEY_BASE=$SECRET_KEY_BASE \
-  -e OTP_SECRET=$OTP_SECRET \
-  --rm tootsuite/mastodon:v3.4.0 \
-  bundle exec rake mastodon:webpush:generate_vapid_key
-```
-
-Once you see the values generated, you can load the `VAPID_PUBLIC_KEY` into your `.env` file and `VAPID_PRIVATE_KEY` into a secret.
-
-```
-$ abra app secret insert YOURDOMAIN vapid_private_key v1 YOURVAPIDPRIVATEKEY
-```
-
-And finally, to end your whirlwind secrets loading adventure, get the `DB_PASS` and `SMTP_PASSWORD` loaded.
-
-```
-$ abra app secret generate YOURAPPDOMAIN db_password v1
-$ abra app secret insert YOURDOMAIN smtp_password v1 YOURSMTPPASSWORD
+$ abra app secret insert <domain> smtp_password v1 <YOUR-SMTP-PASSWORD>
 ```
--- a/abra.sh
+++ b/abra.sh
@ -1 +1,56 @@
 export ENTRYPOINT_CONF_VERSION=v1
+
+admin() {
+  export OTP_SECRET=$(cat /run/secrets/otp_secret)
+  export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base)
+  export DB_PASS=$(cat /run/secrets/db_password)
+
+  tootctl accounts create "$1" --email "$2" --confirmed --role admin
+}
+
+secrets() {
+  set -e
+
+  docker context use default > /dev/null 2>&1
+
+  MASTO_VERSION="v4.1.0"
+
+  echo "Generating secrets for a new Mastodon deployment..."
+  echo ""
+
+  SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:$MASTO_VERSION bundle exec rake secret)
+  abra app secret insert "$APP_NAME" secret_key_base v1 "$SECRET_KEY_BASE"
+  echo "SECRET_KEY_BASE = $SECRET_KEY_BASE"
+  echo ""
+
+  OTP_SECRET=$(docker run --rm tootsuite/mastodon:$MASTO_VERSION bundle exec rake secret)
+  abra app secret insert "$APP_NAME" otp_secret v1 "$OTP_SECRET"
+  echo "OTP_SECRET = $OTP_SECRET"
+  echo ""
+
+  docker run \
+    -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \
+    -e OTP_SECRET="$OTP_SECRET" \
+    --rm tootsuite/mastodon:$MASTO_VERSION \
+    bundle exec rake mastodon:webpush:generate_vapid_key \
+    > /tmp/key.txt
+
+  VAPID_PRIVATE_KEY=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" "/tmp/key.txt")
+  VAPID_PUBLIC_KEY=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" "/tmp/key.txt")
+  rm -rf /tmp/key.txt
+
+  echo "VAPID_PUBLIC_KEY = $VAPID_PUBLIC_KEY"
+  echo "!IMPORTANT! you MUST insert this VAPID_PUBLIC_KEY into your app .env config !IMPORTANT!"
+  echo ""
+
+  abra app secret insert "$APP_NAME" vapid_private_key v1 "$VAPID_PRIVATE_KEY"
+  echo "VAPID_PRIVATE_KEY = $VAPID_PRIVATE_KEY"
+  echo ""
+
+  abra app secret generate "$APP_NAME" db_password v1
+  echo ""
+
+  echo "don't forget to insert your smtp_password! your deployment won't work without it"
+  echo "run \"abra app secret insert $APP_NAME smtp_password v1 YOURSMTPPASSWORD\""
+  echo ""
+}
--- a/compose.yml
+++ b/compose.yml
@ -3,11 +3,12 @@ version: "3.8"

 services:
  app:
-    image: tootsuite/mastodon:v3.4.6
-    command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
+    image: tootsuite/mastodon:v4.1.0
+    command: |
+      bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
    networks: &bothNetworks
      - proxy
-      - internal_network
+      - internal
    deploy:
      update_config:
        failure_action: rollback
@ -19,7 +20,7 @@ services:
        - "traefik.http.routers.${STACK_NAME}_web.rule=Host(`${DOMAIN}`)"
        - "traefik.http.routers.${STACK_NAME}_web.entrypoints=web-secure"
        - "traefik.http.routers.${STACK_NAME}_web.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=0.1.1+3.4.6"
+        - "coop-cloud.${STACK_NAME}.version=1.0.0+v4.1.0"
    configs: &configs
      - source: entrypoint_sh
        target: /usr/local/bin/entrypoint.sh
@ -34,130 +35,92 @@ services:
      - smtp_password
      - vapid_private_key
    environment: &env
+      - ALLOW_ACCESS_TO_HIDDEN_SERVICE
+      - ALTERNATE_DOMAINS
+      - AUTHORIZED_FETCH
+      - CACHE_REDIS_HOST
+      - CACHE_REDIS_NAMESPACE
+      - CACHE_REDIS_PORT
+      - CACHE_REDIS_URL
      - DB_HOST
-      - DB_USER
      - DB_NAME
      - DB_PASS_FILE=/run/secrets/db_password
      - DB_PORT
-      - REDIS_HOST
-      - REDIS_PORT
-      - REDIS_URL
-      - REDIS_NAMESPACE
-      - CACHE_REDIS_HOST
-      - CACHE_REDIS_PORT
-      - CACHE_REDIS_URL
-      - CACHE_REDIS_NAMESPACE
+      - DB_USER
+      - DEFAULT_LOCALE
+      - EMAIL_DOMAIN_ALLOWLIST
+      - EMAIL_DOMAIN_DENYLIST
      - ES_ENABLED
      - ES_HOST
      - ES_PORT
-      - STATSD_ADDR
-      - STATSD_NAMESPACE
-      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
-      - VAPID_PUBLIC_KEY
-      - OTP_SECRET_FILE=/run/secrets/otp_secret
-      - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base
-      - LOCAL_DOMAIN
-      - WEB_DOMAIN
-      - ALTERNATE_DOMAINS
-      - AUTHORIZED_FETCH
-      - LIMITED_FEDERATION_MODE
-      - RAILS_ENV
-      - RAILS_SERVE_STATIC_FILES
-      - SINGLE_USER_MODE
-      - EMAIL_DOMAIN_ALLOWLIST
-      - EMAIL_DOMAIN_DENYLIST
-      - DEFAULT_LOCALE
-      - MAX_SESSION_ACTIVATIONS
-      - USER_ACTIVE_DAYS
-      - SMTP_SERVER
-      - SMTP_PORT
-      - SMTP_LOGIN
-      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
-      - SMTP_FROM_ADDRESS
-      - SMTP_DOMAIN
-      - SMTP_DELIVERY_METHOD
-      - SMTP_AUTH_METHOD
-      - SMTP_CA_FILE
-      - SMTP_OPENSSL_VERIFY_MODE
-      - SMTP_ENABLE_STARTTLS_AUTO
-      - SMTP_TLS
-      - SMTP_SSL
-      - PAPERCLIP_ROOT_PATH
-      - PAPERCLIP_ROOT_URL
-      - OAUTH_REDIRECT_AT_SIGN_IN
-      - LDAP_ENABLED
-      - LDAP_HOST
-      - LDAP_PORT
-      - LDAP_METHOD
      - LDAP_BASE
      - LDAP_BIND_DN
-      - LDAP_PASSWORD
-      - LDAP_UID
-      - LDAP_SEARCH_FILTER
+      - LDAP_ENABLED
+      - LDAP_HOST
      - LDAP_MAIL
+      - LDAP_METHOD
+      - LDAP_PASSWORD
+      - LDAP_PORT
+      - LDAP_SEARCH_FILTER
+      - LDAP_UID
      - LDAP_UID_CONVERSTION_ENABLED
-      - SAML_ENABLED
+      - LIMITED_FEDERATION_MODE
+      - LOCAL_DOMAIN
+      - MAX_SESSION_ACTIVATIONS
+      - OAUTH_REDIRECT_AT_SIGN_IN
+      - OTP_SECRET_FILE=/run/secrets/otp_secret
+      - PAPERCLIP_ROOT_PATH
+      - PAPERCLIP_ROOT_URL
+      - RAILS_ENV
+      - RAILS_SERVE_STATIC_FILES
+      - REDIS_HOST
+      - REDIS_NAMESPACE
+      - REDIS_PORT
+      - REDIS_URL
      - SAML_ACS_URL
-      - SAML_ISSUER
-      - SAML_IDP_SSO_TARGET_URL
-      - SAML_IDP_CERT
-      - SAML_IDP_CERT_FINGERPRINT
-      - SAML_NAME_IDENTIFIER_FORMAT
-      - SAML_CERT
-      - SAML_PRIVATE_KEY
-      - SAML_SECURITY_WANT_ASSERTION_SIGNED
-      - SAML_SECURITY_WANT_ASSERTION_ENCRYPTED
-      - SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED
-      - SAML_ATTRIBUTES_STATEMENTS_UID
      - SAML_ATTRIBUTES_STATEMENTS_EMAIL
-      - SAML_ATTRIBUTES_STATEMENTS_FULL_NAME
      - SAML_ATTRIBUTES_STATEMENTS_FIRST_NAME
+      - SAML_ATTRIBUTES_STATEMENTS_FULL_NAME
      - SAML_ATTRIBUTES_STATEMENTS_LAST_NAME
-      - SAML_UID_ATTRIBUTE
+      - SAML_ATTRIBUTES_STATEMENTS_UID
      - SAML_ATTRIBUTES_STATEMENTS_VERIFIED
      - SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL
-      - http_proxy # yes, this should be lowercase
-      - ALLOW_ACCESS_TO_HIDDEN_SERVICE
-
-  db:
-    image: postgres:9.6-alpine
-    networks: &internalNetwork
-      - internal_network
-    volumes:
-      - postgres:/var/lib/postgresql/data
-    secrets:
-      - db_password
-    environment:
-      - POSTGRES_DB=${DB_NAME}
-      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
-      - POSTGRES_USER=${DB_USER}
-
-  redis:
-    image: redis:6.2-alpine
-    networks: *internalNetwork
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-    volumes:
-      - redis:/data
-
-  es:
-    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.17
-    environment:
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "cluster.name=es-mastodon"
-      - "discovery.type=single-node"
-      - "bootstrap.memory_lock=true"
-    networks:
-      - internal_network
-    volumes:
-      - es:/usr/share/elasticsearch/data
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
+      - SAML_CERT
+      - SAML_ENABLED
+      - SAML_IDP_CERT
+      - SAML_IDP_CERT_FINGERPRINT
+      - SAML_IDP_SSO_TARGET_URL
+      - SAML_ISSUER
+      - SAML_NAME_IDENTIFIER_FORMAT
+      - SAML_PRIVATE_KEY
+      - SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED
+      - SAML_SECURITY_WANT_ASSERTION_ENCRYPTED
+      - SAML_SECURITY_WANT_ASSERTION_SIGNED
+      - SAML_UID_ATTRIBUTE
+      - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base
+      - SINGLE_USER_MODE
+      - SMTP_AUTH_METHOD
+      - SMTP_CA_FILE
+      - SMTP_DELIVERY_METHOD
+      - SMTP_DOMAIN
+      - SMTP_ENABLE_STARTTLS_AUTO
+      - SMTP_FROM_ADDRESS
+      - SMTP_LOGIN
+      - SMTP_OPENSSL_VERIFY_MODE
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
+      - SMTP_PORT
+      - SMTP_SERVER
+      - SMTP_SSL
+      - SMTP_TLS
+      - STATSD_ADDR
+      - STATSD_NAMESPACE
+      - USER_ACTIVE_DAYS
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
+      - VAPID_PUBLIC_KEY
+      - WEB_DOMAIN

  streaming:
-    image: tootsuite/mastodon:v3.4.6
+    image: tootsuite/mastodon:v4.1.0
    command: node ./streaming
    configs: *configs
    entrypoint: *entrypoint
@ -175,10 +138,10 @@ services:
        - "traefik.http.routers.${STACK_NAME}_streaming.entrypoints=web-secure"
        - "traefik.http.routers.${STACK_NAME}_streaming.tls.certresolver=${LETS_ENCRYPT_ENV}"
    environment: *env
-    volumes: *appVolume # used to make sure this volume is created
+    volumes: *appVolume

  sidekiq:
-    image: tootsuite/mastodon:v3.4.6
+    image: tootsuite/mastodon:v4.1.0
    secrets: *secrets
    command: bundle exec sidekiq
    configs: *configs
@ -191,9 +154,46 @@ services:
    volumes: *appVolume
    environment: *env

+  db:
+    image: postgres:15.2-alpine
+    networks: &internalNetwork
+      - internal
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    secrets:
+      - db_password
+    environment:
+      - POSTGRES_DB=${DB_NAME}
+      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
+      - POSTGRES_USER=${DB_USER}
+
+  redis:
+    image: redis:7.0-alpine
+    networks: *internalNetwork
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+    volumes:
+      - redis:/data
+
+  es:
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "cluster.name=es-mastodon"
+      - "discovery.type=single-node"
+      - "bootstrap.memory_lock=true"
+    networks:
+      - internal
+    volumes:
+      - es:/usr/share/elasticsearch/data
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
 secrets:
  secret_key_base:
-    name: ${STACK_NAME}_secret_key_base_${SECRET_KEY_BASE_VERSION}
+    name: ${STACK_NAME}_secret_key_base_${SECRET_SECRET_KEY_BASE_VERSION}
    external: true
  otp_secret:
    name: ${STACK_NAME}_otp_secret_${SECRET_OTP_SECRET_VERSION}
@ -215,10 +215,9 @@ volumes:
  es:

 networks:
+  internal:
  proxy:
    external: true
-  internal_network:
-    internal: true

 configs:
  entrypoint_sh: