feat: make v4 release & add notes

See b3dfa11097

.drone.yml

@@ -0,0 +1,49 @@
+---
+kind: pipeline
+name: deploy to swarm-test.autonomic.zone
+steps:
+  - name: deployment
+    image: git.coopcloud.tech/coop-cloud/stack-ssh-deploy:latest
+    settings:
+      host: swarm-test.autonomic.zone
+      stack: matrix-synapse
+      generate_secrets: true
+      purge: true
+      deploy_key:
+        from_secret: drone_ssh_swarm_test
+      networks:
+        - proxy
+    environment:
+      DOMAIN: matrix-synapse.swarm-test.autonomic.zone
+      STACK_NAME: matrix-synapse
+      LETS_ENCRYPT_ENV: production
+      DISCORD_BRIDGE_YAML_VERSION: v1
+      ENTRYPOINT_CONF_VERSION: v1
+      HOMESERVER_YAML_VERSION: v17
+      LOG_CONFIG_VERSION: v1
+      SHARED_SECRET_AUTH_VERSION: v1
+      SIGNAL_BRIDGE_YAML_VERSION: v1
+      TELEGRAM_BRIDGE_YAML_VERSION: v1
+      SECRET_DB_PASSWORD_VERSION: v1
+      SECRET_FORM_SECRET_VERSION: v1
+      SECRET_MACAROON_SECRET_KEY_VERSION: v1
+      SECRET_REGISTRATION_SHARED_SECRET_VERSION: v1
+trigger:
+  branch:
+    - main
+---
+kind: pipeline
+name: generate recipe catalogue
+steps:
+  - name: release a new version
+    image: plugins/downstream
+    settings:
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - coop-cloud/auto-recipes-catalogue-json
+
+trigger:
+  event: tag

.env.sample

@@ -1,7 +1,10 @@
 TYPE=matrix-synapse
-DOMAIN=matrix.example.com
+DOMAIN=matrix-synapse.example.com
+#TIMEOUT=300
+#ENABLE_AUTO_UPDATE=true
 LETS_ENCRYPT_ENV=production
 COMPOSE_FILE="compose.yml"
+# POST_DEPLOY_CMDS="db set_admin"
 
 ## Admin details
 
@@ -11,8 +14,8 @@ SYNAPSE_ADMIN_EMAIL=admin@example.com
 
 SECRET_DB_PASSWORD_VERSION=v1
 SECRET_FORM_SECRET_VERSION=v1
-SECRET_MACAROON_SECRET_KEY_VERSION=v1
-SECRET_REGISTRATION_SHARED_SECRET_VERSION=v1
+SECRET_MACAROON_VERSION=v1
+SECRET_REGISTRATION_VERSION=v1
 
 ## Federation
 
@@ -48,6 +51,9 @@ ENCRYPTED_BY_DEFAULT=all
 #ENABLE_ALLOWLIST=1
 #FEDERATION_ALLOWLIST="[]"
 
+# Set these to keyservers you trust - usually the same as your federation allowlist
+#TRUSTED_KEYSERVERS="trusted_key_servers:\n  - server_name: 'example.com'\n  - server_name: 'example2.com'"
+
 ## Retention
 
 ALLOWED_LIFETIME_MAX=4w
@@ -72,6 +78,16 @@ RETENTION_MAX_LIFETIME=4w
 
 ## TURN
 
+#COMPOSE_FILE="$COMPOSE_FILE:compose.keycloak3.yml"
+#KEYCLOAK3_ENABLED=1
+#KEYCLOAK3_ID=keycloak3
+#KEYCLOAK3_NAME=
+#KEYCLOAK3_URL=
+#KEYCLOAK3_CLIENT_ID=
+#KEYCLOAK3_CLIENT_DOMAIN=
+#KEYCLOAK3_ALLOW_EXISTING_USERS=false
+#SECRET_KEYCLOAK3_CLIENT_SECRET_VERSION=v1
+
 #COMPOSE_FILE="$COMPOSE_FILE:compose.turn.yml"
 #TURN_ENABLED=1
 #TURN_URIS="[\"turns:coturn.foo.zone?transport=udp\", \"turns:coturn.foo.zone?transport=tcp\"]"
@@ -106,6 +122,7 @@ RETENTION_MAX_LIFETIME=4w
 #ENABLE_ENCRYPTION=true
 #TELEGRAM_APP_ID=
 #TELEGRAM_BRIDGE_PERMISSIONS="{ \"*\": \"relaybot\" }"
+#TELEGRAM_SYNC_CHANNEL_MEMBERS=true
 #SECRET_TELEGRAM_DB_PASSWORD_VERSION=v1
 #SECRET_TELEGRAM_API_HASH_VERSION=v1
 #SECRET_TELEGRAM_BOT_TOKEN_VERSION=v1

README.md

@@ -18,73 +18,32 @@
 1. Set up Docker Swarm and [`abra`](https://docs.coopcloud.tech/abra/)
 2. Deploy [`coop-cloud/traefik`](https://git.coopcloud.tech/coop-cloud/traefik)
 3. `abra app new matrix-synapse --secrets` (optionally with `--pass` if you'd like to save secrets in `pass`)
-4. `abra app YOURAPPDOMAIN config` - be sure to change `$DOMAIN` to something that resolves to your Docker swarm box
-5. `abra app YOURAPPDOMAIN deploy`
-6. Create an initial user: `abra app YOURAPPDOMAIN run app register_new_matrix_user -c /data/homeserver.yaml http://localhost:8008`
+4. `abra app config YOURAPPDOMAIN` - be sure to change `$DOMAIN` to something that resolves to your Docker swarm box
+5. `abra app deploy YOURAPPDOMAIN`
+6. Create an initial user: `abra app run YOURAPPDOMAIN app register_new_matrix_user -c /data/homeserver.yaml http://localhost:8008`
 
 ## Tips & Tricks
 
+### Set Admin User
+
+`abra app cmd YOURAPPDOMAIN db set_admin <adminuser>`
+
 ### Disabling federation
 
-> We're not sure this does it exactly and there is still a discussion running
-> upstream about whether this is the right way to do it & whether it could be
-> more convenient. We welcome issues / change sets to close up more federation
-> functionality.
-
-- use `DISABLE_FEDERATION=1` to turn off federation listeners
-- don't use [`compose.matrix.yml`](https://git.coopcloud.tech/coop-cloud/traefik/src/branch/master/compose.matrix.yml) in your traefik config to keep the federation ports closed
+- Use `DISABLE_FEDERATION=1` to turn off federation listeners
+- Don't use [`compose.matrix.yml`](https://git.coopcloud.tech/coop-cloud/traefik/src/branch/master/compose.matrix.yml) in your traefik config to keep the federation ports closed
 
 ### Enabling federation
 
-See [`#27`](https://git.coopcloud.tech/coop-cloud/matrix-synapse/pulls/27) for more.
-
-Depending on your setup, using `SERVE_SERVER_WELLKNOWN=true` might work to start federating.
-
-Make sure you don't leave `DISABLE_FEDERATION=1` set!
-
-### Seeing what changed in `homeserver.yaml` between versions
-
-Change the version range to suit your needs.
-
-```
-git clone https://github.com/matrix-org/synapse
-cd synapse/docs
-git log --follow -p v1.48.0..v1.51.0 sample_config.yaml
-```
-
-### Generating a new `homeserver.yaml`
-
-The default is also available to see [here](https://matrix-org.github.io/synapse/latest/usage/configuration/homeserver_sample_config.html).
-
-```
-docker run -it \
-  --entrypoint="" \
-  -e SYNAPSE_SERVER_NAME=foo.com \
-  -e SYNAPSE_REPORT_STATS=no \
-  matrixdotorg/synapse:v1.48.0 \
-  sh -c '/start.py generate; cat /data/homeserver.yaml' > homeserver.yaml.tmpl`
-```
-
-### Generating a new `<server>.log.config`
-
-```
-docker run -it \
-  --entrypoint="" \
-  -e SYNAPSE_SERVER_NAME=foo.com \
-  -e SYNAPSE_REPORT_STATS=no \
-  matrixdotorg/synapse:v1.48.0 \
-  sh -c '/start.py generate; cat /data/foo.com.log.config' > log.config
-```
+See [`#27`](https://git.coopcloud.tech/coop-cloud/matrix-synapse/pulls/27) for more. Depending on your setup, using `SERVE_SERVER_WELLKNOWN=true` might work to start federating. Make sure you don't leave `DISABLE_FEDERATION=1` set!
 
 ### Getting client discovery on a custom domain
 
-You'll need to deploy something like [this](https://git.autonomic.zone/ruangrupa/well-known-uris).
-
-This could be implemented in this recipe but we haven't merged it in yet. Change sets are welcome.
+You'll need to deploy something like [this](https://git.autonomic.zone/ruangrupa/well-known-uris). This could be implemented in this recipe but we haven't merged it in yet. Change sets are welcome.
 
 ### Telegram bridging
 
-> WIP
+> WIP docs
 
 Setting it up is a bit of a chicken/egg & chasing cats moment.
 
@@ -123,7 +82,7 @@ Some helpful documentation:
 
 ### Discord bridging
 
-> WIP
+> WIP docs
 
 Just as messy as the Telegram bridging above! Rough guide:
 
@@ -142,9 +101,9 @@ Some helpful documentation:
 
 ### Signal bridging
 
-> WIP
+> WIP docs
 
-OK, it's also awful to set this up. Do you see a pattern emerging? :)
+OK, it's also awful to set this up. Do you see a pattern emerging :)
 
 - fake that you have the required tokens:
   - `abra app secret insert example.com signal_hs_token v1 foo`

abra.sh

@@ -1,7 +1,16 @@
 export DISCORD_BRIDGE_YAML_VERSION=v2
 export ENTRYPOINT_CONF_VERSION=v1
-export HOMESERVER_YAML_VERSION=v17
+export HOMESERVER_YAML_VERSION=v23
 export LOG_CONFIG_VERSION=v2
 export SHARED_SECRET_AUTH_VERSION=v1
 export SIGNAL_BRIDGE_YAML_VERSION=v4
-export TELEGRAM_BRIDGE_YAML_VERSION=v5
+export TELEGRAM_BRIDGE_YAML_VERSION=v6
+
+set_admin () {
+    admin=akadmin
+    if [ -n "$1" ]
+    then
+        admin=$1 
+    fi
+    psql -U synapse -c "UPDATE users SET admin = 1 WHERE name = '@$admin:$DOMAIN'";
+}

compose.keycloak2.yml

@@ -0,0 +1,19 @@
+---
+version: "3.8"
+
+services:
+  app:
+    secrets:
+      - keycloak2_client_secret
+    environment:
+      - KEYCLOAK2_ALLOW_EXISTING_USERS
+      - KEYCLOAK2_CLIENT_ID
+      - KEYCLOAK2_ENABLED
+      - KEYCLOAK2_ID
+      - KEYCLOAK2_NAME
+      - KEYCLOAK2_URL
+
+secrets:
+  keycloak2_client_secret:
+    external: true
+    name: ${STACK_NAME}_keycloak2_client_secret_${SECRET_KEYCLOAK2_CLIENT_SECRET_VERSION}

compose.keycloak3.yml

@@ -0,0 +1,19 @@
+---
+version: "3.8"
+
+services:
+  app:
+    secrets:
+      - keycloak3_client_secret
+    environment:
+      - KEYCLOAK3_ALLOW_EXISTING_USERS
+      - KEYCLOAK3_CLIENT_ID
+      - KEYCLOAK3_ENABLED
+      - KEYCLOAK3_ID
+      - KEYCLOAK3_NAME
+      - KEYCLOAK3_URL
+
+secrets:
+  keycloak3_client_secret:
+    external: true
+    name: ${STACK_NAME}_keycloak3_client_secret_${SECRET_KEYCLOAK3_CLIENT_SECRET_VERSION}

compose.shared_secret_auth.yml

@@ -9,7 +9,7 @@ services:
       - shared_secret_auth
     configs:
       - source: shared_secret_auth
-        target: /usr/local/lib/python3.9/site-packages/shared_secret_authenticator.py
+        target: /usr/local/lib/python3.11/site-packages/shared_secret_authenticator.py
 
 configs:
   shared_secret_auth:

compose.signal.yml

@@ -10,14 +10,14 @@ services:
       - signal-data:/signal-data
 
   signald:
-    image: docker.io/signald/signald:0.23.0-non-root
+    image: docker.io/signald/signald:0.23.2-non-root
     networks:
       - internal
     volumes:
       - signald-data:/signald
 
   signalbridge:
-    image: dock.mau.dev/mautrix/signal:v0.4.2
+    image: dock.mau.dev/mautrix/signal:v0.4.3
     depends_on:
       - signaldb
     configs:

compose.telegram.yml

@@ -10,7 +10,7 @@ services:
       - telegram-data:/telegram-data
 
   telegrambridge:
-    image: dock.mau.dev/mautrix/telegram:v0.12.2
+    image: dock.mau.dev/mautrix/telegram:v0.14.2
     depends_on:
       - telegramdb
     configs:
@@ -25,6 +25,7 @@ services:
       - HOMESERVER_URL
       - TELEGRAM_APP_ID
       - TELEGRAM_BRIDGE_PERMISSIONS
+      - TELEGRAM_SYNC_CHANNEL_MEMBERS
       - VERIFY_SSL
     secrets:
       - telegram_api_hash

compose.yml

@@ -3,7 +3,7 @@ version: "3.8"
 
 services:
   app:
-    image: "matrixdotorg/synapse:v1.74.0"
+    image: "matrixdotorg/synapse:v1.93.0"
     volumes:
       - "data:/data"
     depends_on:
@@ -61,7 +61,14 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=3.0.0+v1.74.0"
+        - "coop-cloud.${STACK_NAME}.version=4.0.0+v1.93.0"
+        - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8008/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 1m
 
   db:
     image: postgres:13-alpine
@@ -74,10 +81,15 @@ services:
       - POSTGRES_INITDB_ARGS="-E \"UTF8\""
       - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
       - POSTGRES_USER=synapse
+      - DOMAIN
     networks:
       - internal
     healthcheck:
-      test: ["CMD", "pg_isready", "-U", "$POSTGRES_USER" ]
+      test: ["CMD", "pg_isready", "-U", "synapse"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 1m
     volumes:
       - postgres:/var/lib/postgresql/data
     deploy:
@@ -102,7 +114,7 @@ configs:
     file: entrypoint.sh.tmpl
     template_driver: golang
   homeserver_yaml:
-    name: ${STACK_NAME}_homserver_yaml_${HOMESERVER_YAML_VERSION}
+    name: ${STACK_NAME}_homeserver_yaml_${HOMESERVER_YAML_VERSION}
     file: homeserver.yaml.tmpl
     template_driver: golang
   log_config:
@@ -116,10 +128,10 @@ secrets:
     name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}
   registration_shared_secret:
     external: true
-    name: ${STACK_NAME}_db_password_${SECRET_REGISTRATION_SHARED_SECRET_VERSION}
+    name: ${STACK_NAME}_registration_${SECRET_REGISTRATION_VERSION}
   macaroon_secret_key:
     external: true
-    name: ${STACK_NAME}_db_password_${SECRET_MACAROON_SECRET_KEY_VERSION}
+    name: ${STACK_NAME}_macaroon_${SECRET_MACAROON_VERSION}
   form_secret:
     external: true
-    name: ${STACK_NAME}_db_password_${SECRET_FORM_SECRET_VERSION}
+    name: ${STACK_NAME}_form_secret_${SECRET_FORM_SECRET_VERSION}

homeserver.yaml.tmpl

@@ -38,7 +38,7 @@ modules:
 # lowercase and may contain an explicit port.
 # Examples: matrix.org, localhost:8080
 #
-server_name: {{ env "DOMAIN" }}
+server_name: {{ or (env "SERVER_NAME") (env "DOMAIN") }}
 
 # The public-facing base URL that clients use to access this Homeserver (not
 # including _matrix/...). This is the same URL a user might enter into the
@@ -427,8 +427,12 @@ signing_key_path: "/data/{{ env "DOMAIN" }}.signing.key"
 #      "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr"
 #  - server_name: "my_other_trusted_server.example.com"
 #
+{{ if eq (env "ENABLE_ALLOWLIST") "1" }}
+{{ env "TRUSTED_KEYSERVERS" }}
+{{ else }}
 trusted_key_servers:
   - server_name: "matrix.org"
+{{ end }}
 
 ## Single sign-on integration ##
 
@@ -604,6 +608,34 @@ oidc_providers:
         display_name_template: "{{ "{{ user.name }}" }}"
   {{ end }}
 
+  {{ if eq (env "KEYCLOAK2_ENABLED") "1" }}
+  - idp_id: {{ env "KEYCLOAK2_ID" }}
+    idp_name: {{ env "KEYCLOAK2_NAME" }}
+    issuer: "{{ env "KEYCLOAK2_URL" }}"
+    client_id: "{{ env "KEYCLOAK2_CLIENT_ID" }}"
+    client_secret: "{{ secret "keycloak2_client_secret" }}"
+    scopes: ["openid", "profile"]
+    allow_existing_users: {{ env "KEYCLOAK2_ALLOW_EXISTING_USERS" }}
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ "{{ user.preferred_username }}" }}"
+        display_name_template: "{{ "{{ user.name }}" }}"
+  {{ end }}
+
+  {{ if eq (env "KEYCLOAK3_ENABLED") "1" }}
+  - idp_id: {{ env "KEYCLOAK3_ID" }}
+    idp_name: {{ env "KEYCLOAK3_NAME" }}
+    issuer: "{{ env "KEYCLOAK3_URL" }}"
+    client_id: "{{ env "KEYCLOAK3_CLIENT_ID" }}"
+    client_secret: "{{ secret "keycloak3_client_secret" }}"
+    scopes: ["openid", "profile"]
+    allow_existing_users: {{ env "KEYCLOAK3_ALLOW_EXISTING_USERS" }}
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ "{{ user.preferred_username }}" }}"
+        display_name_template: "{{ "{{ user.name }}" }}"
+  {{ end }}
+
 # Additional settings to use with single-sign on systems such as OpenID Connect,
 # SAML2 and CAS.
 #
@@ -659,7 +691,7 @@ email:
   # authentication is attempted.
   #
   smtp_user: {{ env "SMTP_USER" }}
-  smtp_pass: {{ secret "smtp_password" }}
+  smtp_pass: "{{ secret "smtp_password" }}"
 
   # Uncomment the following to require TLS transport security for SMTP.
   # By default, Synapse will connect over plain text, and will then switch to

release/4.0.0+v1.93.0

@@ -0,0 +1,8 @@
+We had to rename some secrets: https://git.coopcloud.tech/coop-cloud/matrix-synapse/issues/35
+
+Copy the secrets:
+
+* `registration_shared_secret` to `registration`
+* `macaroon_secret_key` to `macaroon`
+
+Regeneration of these secrets should also work.

telegram_bridge.yaml.tmpl

@@ -151,7 +151,7 @@ bridge:
     # Whether or not to sync the member list in channels.
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
-    sync_channel_members: true
+    sync_channel_members: {{ env "TELEGRAM_SYNC_CHANNEL_MEMBERS" }}
     # Whether or not to skip deleted members when syncing members.
     skip_deleted_members: true
     # Whether or not to automatically synchronize contacts and chats of Matrix users logged into