Merge branch 'main' of ssh://git.autonomic.zone:2222/coop-cloud/mastodon into main

2021-05-28 12:55:27 +02:00 · 2021-05-28 12:55:27 +02:00 · b0d0817bb7
parent 12eb79003b 580fc560c7
commit b0d0817bb7
5 changed files with 176 additions and 64 deletions
--- a/.env.sample
+++ b/.env.sample
@ -4,9 +4,8 @@ DOMAIN=mastodon.swarm-test.autonomic.zone
 # Enables WEB_DOMAIN if set (FOR FUTURE USE)
 # USER_DOMAIN=

-
 ## Domain aliases
-#EXTRA_DOMAINS=', `www.mastodon.example.com`'
+# EXTRA_DOMAINS=', `www.mastodon.example.com`'
 LETS_ENCRYPT_ENV=production

 # Please look at https://docs.joinmastodon.org/admin/config/ for the full documentation.
@ -27,9 +26,9 @@ LETS_ENCRYPT_ENV=production
 # fi

 LOCAL_DOMAIN=$DOMAIN
-#WEB_DOMAIN=$DOMAIN
+# WEB_DOMAIN=$DOMAIN

-#ALTERNATE_DOMAINS=$EXTRA_DOMAINS
+# ALTERNATE_DOMAINS=$EXTRA_DOMAINS
 AUTHORIZED_FETCH=false
 LIMITED_FEDERATION_MODE=false

@ -47,7 +46,6 @@ RAILS_SERVE_STATIC_FILES=true # might need this for traefik, need to test
 DB_HOST=db
 DB_USER=mastodon
 DB_NAME=mastodon_production
-DB_PASS=
 DB_PORT=5432

 # Redis
@ -61,12 +59,11 @@ REDIS_PORT=6379
 # CACHE_REDIS_URL=
 # CACHE_REDIS_NAMESPACE=

-# ElasticSearch (CURRENTLY NOT SUPPORTED)
+# ElasticSearch
 # --------------------------------------
-ES_ENABLED=false
-# ES_HOST=localhost
-# ES_PORT=9200
-# ES_PREFIX=
+ES_ENABLED=true
+ES_HOST=es
+ES_PORT=9200

 # StatsD (CURRENTLY NOT SUPPORTED)
 # -------------------------------
@ -75,13 +72,15 @@ ES_ENABLED=false

 # Secrets
 # =======
-SECRET_KEY_BASE=
-OTP_SECRET=
+SECRET_KEY_BASE_VERSION=v1
+SECRET_OTP_SECRET_VERSION=v1
+SECRET_VAPID_PRIVATE_KEY_VERSION=v1
+SECRET_DB_PASSWORD_VERSION=v1
+SECRET_SMTP_PASSWORD_VERSION=v1

 # Web Push
 # ========
-VAPID_PRIVATE_KEY=
-VAPID_PUBLIC_KEY=
+# VAPID_PUBLIC_KEY=

 # Limits
 # ======
@ -175,5 +174,5 @@ DEFAULT_LOCALE=en

 # Hidden services (Not Supported)
 # ===============================
-# http_proxy=
+# http_proxy=  # yes, this should be lowercase
 # ALLOW_ACCESS_TO_HIDDEN_SERVICE=
--- a/README.md
+++ b/README.md
@ -19,13 +19,50 @@ This repository is a copy of [coop-cloud/mastodon](https://git.autonomic.zone/co
 ## Basic usage

 1. Set up Docker Swarm and [`abra`]
-2. Deploy [`coop-cloud/traefik`]
-3. `abra app new hometown`
-4. `abra app YOURAPPDOMAIN config` - be sure to change `DOMAIN` to something that resolves to
-   your Docker swarm box. Install the npm package webpush to create VAPID keys `npm install web-push -g && web-push generate-vapid-keys`.
-5. `abra app YOURAPPDOMAIN deploy --no-domain-poll`. App will fail for now.
-6. `abra app hometown run streaming rake db:setup`
-7. Open the configured domain in your browser to finish set-up. To make an admin account `abra app hometown run web "bin/tootctl accounts create coolusername --email helo@autonomic.zone --confirmed --role admin"`
+1. Deploy [`coop-cloud/traefik`]
+1. `abra app new mastodon`
+1. `abra app YOURAPPDOMAIN config` - be sure to change `DOMAIN` to something that resolves to
+   your Docker swarm box.
+1. Follow the secrets setup documentation below.
+1. `abra app YOURAPPDOMAIN deploy --no-domain-poll`. App will fail for now.
+1. `abra app mastodon run streaming rake db:setup`
+1. Open the configured domain in your browser to finish set-up. To make an admin account `abra app mastodon run web "bin/tootctl accounts create coolusername --email helo@autonomic.zone --confirmed --role admin"`

 [`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra
 [`coop-cloud/traefik`]: https://git.autonomic.zone/coop-cloud/traefik
+
+## Secrets setup
+
+Because Mastodon expects secrets generated by specific tools, we don't support that in `abra` yet. However, you can run these commands yourself using the underlying Docker CLI. You can then load them in as secrets to the swarm using `abra` though and then they will be picked up on the deployment.
+
+First, generate the `SECRET_KEY_BASE` and `OTP_SECRET` and store them in your local shell environment, you'll need them for subsequent commands.
+
+```
+$ SECRET_KEY_BASE=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
+$ OTP_SECRET=$(docker run --rm tootsuite/mastodon:v3.4.0 bundle exec rake secret)
+$ printf $SECRET_KEY_BASE | abra app YOURAPPDOMAIN secret insert secret_key_base v1 -
+$ printf $OTP_SECRET | abra app YOURAPPDOMAIN secret insert otp_secret v1 -
+```
+
+Then you need to generate the `VAPID_{PUBLIC/PRIVATE}_KEY` values using the `SECRET_KEY_BASE`/`OTP_SECRET`:
+
+```
+$ docker run \
+  -e SECRET_KEY_BASE=$SECRET_KEY_BASE \
+  -e OTP_SECRET=$OTP_SECRET \
+  --rm tootsuite/mastodon:v3.4.0 \
+  bundle exec rake mastodon:webpush:generate_vapid_key
+```
+
+Once you see the values generated, you can load the `VAPID_PUBLIC_KEY` into your `.env` file and `VAPID_PRIVATE_KEY` into a secret.
+
+```
+$ printf YOURVAPIDPRIVATEKEY | abra app YOURDOMAIN secret insert vapid_private_key v1 -
+```
+
+And finally, to end your whirlwind secrets loading adventure, get the `DB_PASS` and `SMTP_PASSWORD` loaded.
+
+```
+$ abra app YOURAPPDOMAIN secret generate db_password v1
+$ printf YOURSMTPPASSWORD | abra app YOURDOMAIN secret insert smtp_password v1 -
+```
--- a/abra.sh
+++ b/abra.sh
@ -1,4 +1,5 @@
 # shellcheck disable=SC2148
+export ENTRYPOINT_CONF_VERSION=v1
 #MASTO_APP_DIR="mastodon/public"

 sub_rake() {
--- a/compose.yml
+++ b/compose.yml
@ -10,10 +10,13 @@ services:
      test: ["CMD", "pg_isready", "-U", "postgres"]
    volumes:
      - postgres:/var/lib/postgresql/data
+    secrets:
+      - db_password
    environment:
-      - POSTGRES_PASSWORD=${DB_PASS}
-      - POSTGRES_USER=${DB_USER}
      - POSTGRES_DB=${DB_NAME}
+      - POSTGRES_PASSWORD=${DB_PASS}
+      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
+      - POSTGRES_USER=${DB_USER}

  redis:
    image: redis:6.0-alpine
@ -23,24 +26,23 @@ services:
    volumes:
      - redis:/data

-  #  es:
-  #    restart: always
-  #    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.10
-  #    environment:
-  #      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-  #      - "cluster.name=es-mastodon"
-  #      - "discovery.type=single-node"
-  #      - "bootstrap.memory_lock=true"
-  #    networks:
-  #      - internal_network
-  #    healthcheck:
-  #      test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
-  #    volumes:
-  #      - ./elasticsearch:/usr/share/elasticsearch/data
-  #    ulimits:
-  #      memlock:
-  #        soft: -1
-  #        hard: -1
+   es:
+     image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.10
+     environment:
+       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+       - "cluster.name=es-mastodon"
+       - "discovery.type=single-node"
+       - "bootstrap.memory_lock=true"
+     networks:
+       - internal_network
+     healthcheck:
+       test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
+     volumes:
+       - es:/usr/share/elasticsearch/data
+     ulimits:
+       memlock:
+         soft: -1
+         hard: -1

  web:
    image: &image decentral1se/hometown:v1.0.5_3.4.0
@ -55,8 +57,9 @@ services:
          "wget -q --spider --proxy=off localhost:3000/health || exit 1",
        ]
    deploy:
-      restart_policy:
-        condition: on-failure
+      update_config:
+        failure_action: rollback
+        order: start-first
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=proxy"
@ -73,16 +76,30 @@ services:
        # - "traefik.http.routers.${STACK_NAME}_hack.entrypoints=websecure"
        # - "traefik.http.routers.${STACK_NAME}_hack.middlewares=mastodon-webfinger@docker"

+        ## Redirect from EXTRA_DOMAINS to DOMAIN
+        #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
+        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
+        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
+
+
+    configs: &configs
+      - source: entrypoint_sh
+        target: /usr/local/bin/entrypoint.sh
+        mode: 0555
+    entrypoint: &entrypoint /usr/local/bin/entrypoint.sh
    volumes: &appVolume
      - app:/mastodon
-    # secrets: &secrets
-    #   - secret_key_base
-    #   - otp_secret
+    secrets: &secrets
+      - db_password
+      - otp_secret
+      - secret_key_base
+      - smtp_password
+      - vapid_private_key
    environment: &env
      - DB_HOST
      - DB_USER
      - DB_NAME
-      - DB_PASS
+      - DB_PASS_FILE=/run/secrets/db_password
      - DB_PORT
      - REDIS_HOST
      - REDIS_PORT
@ -98,10 +115,10 @@ services:
      - ES_PREFIX
      - STATSD_ADDR
      - STATSD_NAMESPACE
-      - VAPID_PRIVATE_KEY
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
      - VAPID_PUBLIC_KEY
-      - OTP_SECRET
-      - SECRET_KEY_BASE
+      - OTP_SECRET_FILE=/run/secrets/otp_secret
+      - SECRET_KEY_BASE_FILE=/run/secrets/secret_key_base
      - LOCAL_DOMAIN
      - WEB_DOMAIN
      - ALTERNATE_DOMAINS
@ -118,7 +135,7 @@ services:
      - SMTP_SERVER
      - SMTP_PORT
      - SMTP_LOGIN
-      - SMTP_PASSWORD
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
      - SMTP_FROM_ADDRESS
      - SMTP_DOMAIN
      - SMTP_DELIVERY_METHOD
@ -162,12 +179,15 @@ services:
      - SAML_UID_ATTRIBUTE
      - SAML_ATTRIBUTES_STATEMENTS_VERIFIED
      - SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL
-      - http_proxy
+      - http_proxy # yes, this should be lowercase
      - ALLOW_ACCESS_TO_HIDDEN_SERVICE

  streaming:
    image: *image
    command: node ./streaming
+    configs: *configs
+    entrypoint: *entrypoint
+    secrets: *secrets
    networks: *bothNetworks
    healthcheck:
      test:
@ -176,8 +196,9 @@ services:
          "wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1",
        ]
    deploy:
-      restart_policy:
-        condition: on-failure
+      update_config:
+        failure_action: rollback
+        order: start-first
      labels:
        - "traefik.enable=true"
        - "traefik.docker.network=proxy"
@ -195,27 +216,49 @@ services:

  sidekiq:
    image: *image
+    secrets: *secrets
    command: bundle exec sidekiq
+    configs: *configs
+    entrypoint: *entrypoint
    deploy:
-      restart_policy:
-        condition: on-failure
+      update_config:
+        failure_action: rollback
+        order: start-first
    networks: *bothNetworks
    volumes: *appVolume
    environment: *env

-# secrets:
-#   secret_key_base:
-#     name: ${STACK_NAME}_secret_key_base_${SECRET_DB_PASSWORD_VERSION}
-#     external: true
-#   otp_secret:
-#     name: ${STACK_NAME}_otp_secret_${SECRET_DB_ROOT_PASSWORD_VERSION}
-#     external: true
+secrets:
+  secret_key_base:
+    name: ${STACK_NAME}_secret_key_base_${SECRET_KEY_BASE_VERSION}
+    external: true
+  otp_secret:
+    name: ${STACK_NAME}_otp_secret_${SECRET_OTP_SECRET_VERSION}
+    external: true
+  vapid_private_key:
+    name: ${STACK_NAME}_vapid_private_key_${SECRET_VAPID_PRIVATE_KEY_VERSION}
+    external: true
+  db_password:
+    name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}
+    external: true
+  smtp_password:
+    name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
+    external: true
+
 volumes:
  app:
  redis:
  postgres:
+  es:
+
 networks:
  proxy:
    external: true
  internal_network:
    internal: true
+
+configs:
+  entrypoint_sh:
+    name: ${STACK_NAME}_entrypoint_conf_${ENTRYPOINT_CONF_VERSION}
+    file: entrypoint.sh.tmpl
+    template_driver: golang
--- a/entrypoint.sh.tmpl
+++ b/entrypoint.sh.tmpl
@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -eu
+
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  export "$var"="$val"
+  unset "$fileVar"
+}
+
+file_env "DB_PASS"
+file_env "OTP_SECRET"
+file_env "SECRET_KEY_BASE"
+file_env "SMTP_PASSWORD"
+file_env "VAPID_PRIVATE_KEY"
+
+/usr/bin/tini -- "$@"