From 2f8dcad8aa70649f547e39081dbd5e6f94de209b Mon Sep 17 00:00:00 2001 From: decentral1se Date: Wed, 15 Feb 2023 01:30:24 +0100 Subject: [PATCH] wip: v4.x & major config refactor --- .env.sample | 7 +- README.md | 66 +++++---------- abra.sh | 55 +++++++++++++ compose.yml | 227 ++++++++++++++++++++++++++-------------------------- 4 files changed, 187 insertions(+), 168 deletions(-) diff --git a/.env.sample b/.env.sample index 1a9b402..586c7e1 100644 --- a/.env.sample +++ b/.env.sample @@ -72,7 +72,7 @@ ES_PORT=9200 # Secrets # ======= -SECRET_KEY_BASE_VERSION=v1 +SECRET_SECRET_KEY_BASE_VERSION=v1 SECRET_OTP_SECRET_VERSION=v1 SECRET_VAPID_PRIVATE_KEY_VERSION=v1 SECRET_DB_PASSWORD_VERSION=v1 @@ -169,8 +169,3 @@ DEFAULT_LOCALE=en # SAML_UID_ATTRIBUTE= # SAML_ATTRIBUTES_STATEMENTS_VERIFIED= # SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL= - -# Hidden services (Not Supported) -# =============================== -# http_proxy= # yes, this should be lowercase -# ALLOW_ACCESS_TO_HIDDEN_SERVICE= diff --git a/README.md b/README.md index 80da3ea..6089fdd 100644 --- a/README.md +++ b/README.md @@ -1,62 +1,32 @@ # Mastodon -Your self-hosted, globally interconnected microblogging community +> Your self-hosted, globally interconnected microblogging community * **Category**: Apps -* **Status**: +* **Status**: 1 * **Image**: [`tootsuite/mastodon`](https://hub.docker.com/r/tootsuite/mastodon) -* **Healthcheck**: -* **Backups**: -* **Email**: -* **Tests**: -* **SSO**: +* **Healthcheck**: No +* **Backups**: No +* **Email**: Yes +* **Tests**: No +* **SSO**: Yes -## Basic usage +## Quick start + +Mastodon expects secrets to be formatted in a very specific way, so please +choose "No" when prompted to generate secrets for `abra app new mastodon`. The +secrets must be generated outside of `abra` and that is achieved in step 2. See +the [`abra.sh`](./abra.sh) for more. -1. Set up Docker Swarm and [`abra`] -1. Deploy [`coop-cloud/traefik`] 1. `abra app new mastodon` -1. Follow the [secrets setup docs](#secrets-setup) -1. `abra app config YOURAPPDOMAIN` - be sure to change `DOMAIN` to something that resolves to your Docker swarm box -1. `abra app setup YOURAPPDOMAIN` to setup the database and create the admin account (services will stop flapping shortly after) +1. `abra app cmd secrets --local` +1. `abra app config ` +1. `abra app deploy ` -[`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra -[`coop-cloud/traefik`]: https://git.autonomic.zone/coop-cloud/traefik - -## Secrets setup - -Because Mastodon expects secrets generated by specific tools, we don't support that in `abra` yet. However, you can run these commands yourself using the underlying Docker CLI. You can then load them in as secrets to the swarm using `abra` though and then they will be picked up on the deployment. - -First, generate the `SECRET_KEY_BASE` and `OTP_SECRET` and store them in your local shell environment, you'll need them for subsequent commands. +You can insert the `smtp_password` with the following: ``` -$ SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret) -$ OTP_SECRET=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret) -$ abra app secret insert YOURAPPDOMAIN secret_key_base v1 $SECRET_KEY_BASE -$ abra app secret insert YOURAPPDOMAIN otp_secret v1 $OTP_SECRET -``` - -Then you need to generate the `VAPID_{PUBLIC/PRIVATE}_KEY` values using the `SECRET_KEY_BASE`/`OTP_SECRET`: - -``` -$ docker run \ - -e SECRET_KEY_BASE=$SECRET_KEY_BASE \ - -e OTP_SECRET=$OTP_SECRET \ - --rm tootsuite/mastodon:v3.4.0 \ - bundle exec rake mastodon:webpush:generate_vapid_key -``` - -Once you see the values generated, you can load the `VAPID_PUBLIC_KEY` into your `.env` file and `VAPID_PRIVATE_KEY` into a secret. - -``` -$ abra app secret insert YOURDOMAIN vapid_private_key v1 YOURVAPIDPRIVATEKEY -``` - -And finally, to end your whirlwind secrets loading adventure, get the `DB_PASS` and `SMTP_PASSWORD` loaded. - -``` -$ abra app secret generate YOURAPPDOMAIN db_password v1 -$ abra app secret insert YOURDOMAIN smtp_password v1 YOURSMTPPASSWORD +$ abra app secret insert smtp_password v1 ``` diff --git a/abra.sh b/abra.sh index 5c54e9d..aecf983 100644 --- a/abra.sh +++ b/abra.sh @@ -1 +1,56 @@ export ENTRYPOINT_CONF_VERSION=v1 + +admin() { + export OTP_SECRET=$(cat /run/secrets/otp_secret) + export SECRET_KEY_BASE=$(cat /run/secrets/secret_key_base) + export DB_PASS=$(cat /run/secrets/db_password) + + tootctl accounts create "$1" --email "$2" --confirmed --role admin +} + +secrets() { + set -e + + docker context use default > /dev/null 2>&1 + + MASTO_VERSION="v4.1.0" + + echo "Generating secrets for a new Mastodon deployment..." + echo "" + + SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:$MASTO_VERSION bundle exec rake secret) + abra app secret insert "$APP_NAME" secret_key_base v1 "$SECRET_KEY_BASE" + echo "SECRET_KEY_BASE = $SECRET_KEY_BASE" + echo "" + + OTP_SECRET=$(docker run --rm tootsuite/mastodon:$MASTO_VERSION bundle exec rake secret) + abra app secret insert "$APP_NAME" otp_secret v1 "$OTP_SECRET" + echo "OTP_SECRET = $OTP_SECRET" + echo "" + + docker run \ + -e SECRET_KEY_BASE="$SECRET_KEY_BASE" \ + -e OTP_SECRET="$OTP_SECRET" \ + --rm tootsuite/mastodon:$MASTO_VERSION \ + bundle exec rake mastodon:webpush:generate_vapid_key \ + > /tmp/key.txt + + VAPID_PRIVATE_KEY=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" "/tmp/key.txt") + VAPID_PUBLIC_KEY=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" "/tmp/key.txt") + rm -rf /tmp/key.txt + + echo "VAPID_PUBLIC_KEY = $VAPID_PUBLIC_KEY" + echo "!IMPORTANT! you MUST insert this VAPID_PUBLIC_KEY into your app .env config !IMPORTANT!" + echo "" + + abra app secret insert "$APP_NAME" vapid_private_key v1 "$VAPID_PRIVATE_KEY" + echo "VAPID_PRIVATE_KEY = $VAPID_PRIVATE_KEY" + echo "" + + abra app secret generate "$APP_NAME" db_password v1 + echo "" + + echo "don't forget to insert your smtp_password! your deployment won't work without it" + echo "run \"abra app secret insert $APP_NAME smtp_password v1 YOURSMTPPASSWORD\"" + echo "" +} diff --git a/compose.yml b/compose.yml index 554dd3d..f5d32b2 100644 --- a/compose.yml +++ b/compose.yml @@ -3,11 +3,12 @@ version: "3.8" services: app: - image: tootsuite/mastodon:v3.4.6 - command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" + image: tootsuite/mastodon:v4.1.0 + command: | + bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" networks: &bothNetworks - proxy - - internal_network + - internal deploy: update_config: failure_action: rollback @@ -19,7 +20,7 @@ services: - "traefik.http.routers.${STACK_NAME}_web.rule=Host(`${DOMAIN}`)" - "traefik.http.routers.${STACK_NAME}_web.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}_web.tls.certresolver=${LETS_ENCRYPT_ENV}" - - "coop-cloud.${STACK_NAME}.version=0.1.1+3.4.6" + - "coop-cloud.${STACK_NAME}.version=1.0.0+v4.1.0" configs: &configs - source: entrypoint_sh target: /usr/local/bin/entrypoint.sh @@ -34,130 +35,92 @@ services: - smtp_password - vapid_private_key environment: &env + - ALLOW_ACCESS_TO_HIDDEN_SERVICE + - ALTERNATE_DOMAINS + - AUTHORIZED_FETCH + - CACHE_REDIS_HOST + - CACHE_REDIS_NAMESPACE + - CACHE_REDIS_PORT + - CACHE_REDIS_URL - DB_HOST - - DB_USER - DB_NAME - DB_PASS_FILE=/run/secrets/db_password - DB_PORT - - REDIS_HOST - - REDIS_PORT - - REDIS_URL - - REDIS_NAMESPACE - - CACHE_REDIS_HOST - - CACHE_REDIS_PORT - - CACHE_REDIS_URL - - CACHE_REDIS_NAMESPACE + - DB_USER + - DEFAULT_LOCALE + - EMAIL_DOMAIN_ALLOWLIST + - EMAIL_DOMAIN_DENYLIST - ES_ENABLED - ES_HOST - ES_PORT - - STATSD_ADDR - - STATSD_NAMESPACE - - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key - - VAPID_PUBLIC_KEY - - OTP_SECRET_FILE=/run/secrets/otp_secret - - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base - - LOCAL_DOMAIN - - WEB_DOMAIN - - ALTERNATE_DOMAINS - - AUTHORIZED_FETCH - - LIMITED_FEDERATION_MODE - - RAILS_ENV - - RAILS_SERVE_STATIC_FILES - - SINGLE_USER_MODE - - EMAIL_DOMAIN_ALLOWLIST - - EMAIL_DOMAIN_DENYLIST - - DEFAULT_LOCALE - - MAX_SESSION_ACTIVATIONS - - USER_ACTIVE_DAYS - - SMTP_SERVER - - SMTP_PORT - - SMTP_LOGIN - - SMTP_PASSWORD_FILE=/run/secrets/smtp_password - - SMTP_FROM_ADDRESS - - SMTP_DOMAIN - - SMTP_DELIVERY_METHOD - - SMTP_AUTH_METHOD - - SMTP_CA_FILE - - SMTP_OPENSSL_VERIFY_MODE - - SMTP_ENABLE_STARTTLS_AUTO - - SMTP_TLS - - SMTP_SSL - - PAPERCLIP_ROOT_PATH - - PAPERCLIP_ROOT_URL - - OAUTH_REDIRECT_AT_SIGN_IN - - LDAP_ENABLED - - LDAP_HOST - - LDAP_PORT - - LDAP_METHOD - LDAP_BASE - LDAP_BIND_DN - - LDAP_PASSWORD - - LDAP_UID - - LDAP_SEARCH_FILTER + - LDAP_ENABLED + - LDAP_HOST - LDAP_MAIL + - LDAP_METHOD + - LDAP_PASSWORD + - LDAP_PORT + - LDAP_SEARCH_FILTER + - LDAP_UID - LDAP_UID_CONVERSTION_ENABLED - - SAML_ENABLED + - LIMITED_FEDERATION_MODE + - LOCAL_DOMAIN + - MAX_SESSION_ACTIVATIONS + - OAUTH_REDIRECT_AT_SIGN_IN + - OTP_SECRET_FILE=/run/secrets/otp_secret + - PAPERCLIP_ROOT_PATH + - PAPERCLIP_ROOT_URL + - RAILS_ENV + - RAILS_SERVE_STATIC_FILES + - REDIS_HOST + - REDIS_NAMESPACE + - REDIS_PORT + - REDIS_URL - SAML_ACS_URL - - SAML_ISSUER - - SAML_IDP_SSO_TARGET_URL - - SAML_IDP_CERT - - SAML_IDP_CERT_FINGERPRINT - - SAML_NAME_IDENTIFIER_FORMAT - - SAML_CERT - - SAML_PRIVATE_KEY - - SAML_SECURITY_WANT_ASSERTION_SIGNED - - SAML_SECURITY_WANT_ASSERTION_ENCRYPTED - - SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED - - SAML_ATTRIBUTES_STATEMENTS_UID - SAML_ATTRIBUTES_STATEMENTS_EMAIL - - SAML_ATTRIBUTES_STATEMENTS_FULL_NAME - SAML_ATTRIBUTES_STATEMENTS_FIRST_NAME + - SAML_ATTRIBUTES_STATEMENTS_FULL_NAME - SAML_ATTRIBUTES_STATEMENTS_LAST_NAME - - SAML_UID_ATTRIBUTE + - SAML_ATTRIBUTES_STATEMENTS_UID - SAML_ATTRIBUTES_STATEMENTS_VERIFIED - SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL - - http_proxy # yes, this should be lowercase - - ALLOW_ACCESS_TO_HIDDEN_SERVICE - - db: - image: postgres:9.6-alpine - networks: &internalNetwork - - internal_network - volumes: - - postgres:/var/lib/postgresql/data - secrets: - - db_password - environment: - - POSTGRES_DB=${DB_NAME} - - POSTGRES_PASSWORD_FILE=/run/secrets/db_password - - POSTGRES_USER=${DB_USER} - - redis: - image: redis:6.2-alpine - networks: *internalNetwork - healthcheck: - test: ["CMD", "redis-cli", "ping"] - volumes: - - redis:/data - - es: - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.17 - environment: - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - "cluster.name=es-mastodon" - - "discovery.type=single-node" - - "bootstrap.memory_lock=true" - networks: - - internal_network - volumes: - - es:/usr/share/elasticsearch/data - ulimits: - memlock: - soft: -1 - hard: -1 + - SAML_CERT + - SAML_ENABLED + - SAML_IDP_CERT + - SAML_IDP_CERT_FINGERPRINT + - SAML_IDP_SSO_TARGET_URL + - SAML_ISSUER + - SAML_NAME_IDENTIFIER_FORMAT + - SAML_PRIVATE_KEY + - SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED + - SAML_SECURITY_WANT_ASSERTION_ENCRYPTED + - SAML_SECURITY_WANT_ASSERTION_SIGNED + - SAML_UID_ATTRIBUTE + - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base + - SINGLE_USER_MODE + - SMTP_AUTH_METHOD + - SMTP_CA_FILE + - SMTP_DELIVERY_METHOD + - SMTP_DOMAIN + - SMTP_ENABLE_STARTTLS_AUTO + - SMTP_FROM_ADDRESS + - SMTP_LOGIN + - SMTP_OPENSSL_VERIFY_MODE + - SMTP_PASSWORD_FILE=/run/secrets/smtp_password + - SMTP_PORT + - SMTP_SERVER + - SMTP_SSL + - SMTP_TLS + - STATSD_ADDR + - STATSD_NAMESPACE + - USER_ACTIVE_DAYS + - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key + - VAPID_PUBLIC_KEY + - WEB_DOMAIN streaming: - image: tootsuite/mastodon:v3.4.6 + image: tootsuite/mastodon:v4.1.0 command: node ./streaming configs: *configs entrypoint: *entrypoint @@ -175,10 +138,10 @@ services: - "traefik.http.routers.${STACK_NAME}_streaming.entrypoints=web-secure" - "traefik.http.routers.${STACK_NAME}_streaming.tls.certresolver=${LETS_ENCRYPT_ENV}" environment: *env - volumes: *appVolume # used to make sure this volume is created + volumes: *appVolume sidekiq: - image: tootsuite/mastodon:v3.4.6 + image: tootsuite/mastodon:v4.1.0 secrets: *secrets command: bundle exec sidekiq configs: *configs @@ -191,9 +154,46 @@ services: volumes: *appVolume environment: *env + db: + image: postgres:15.2-alpine + networks: &internalNetwork + - internal + volumes: + - postgres:/var/lib/postgresql/data + secrets: + - db_password + environment: + - POSTGRES_DB=${DB_NAME} + - POSTGRES_PASSWORD_FILE=/run/secrets/db_password + - POSTGRES_USER=${DB_USER} + + redis: + image: redis:7.0-alpine + networks: *internalNetwork + healthcheck: + test: ["CMD", "redis-cli", "ping"] + volumes: + - redis:/data + + es: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.10.2 + environment: + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "cluster.name=es-mastodon" + - "discovery.type=single-node" + - "bootstrap.memory_lock=true" + networks: + - internal + volumes: + - es:/usr/share/elasticsearch/data + ulimits: + memlock: + soft: -1 + hard: -1 + secrets: secret_key_base: - name: ${STACK_NAME}_secret_key_base_${SECRET_KEY_BASE_VERSION} + name: ${STACK_NAME}_secret_key_base_${SECRET_SECRET_KEY_BASE_VERSION} external: true otp_secret: name: ${STACK_NAME}_otp_secret_${SECRET_OTP_SECRET_VERSION} @@ -215,10 +215,9 @@ volumes: es: networks: + internal: proxy: external: true - internal_network: - internal: true configs: entrypoint_sh: