wip: experimental fetchmail support

..to get DKIM signing working for mailman

.drone.yml

@@ -0,0 +1,16 @@
+
+kind: pipeline
+name: generate recipe catalogue
+steps:
+  - name: release a new version
+    image: plugins/downstream
+    settings:
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - coop-cloud/auto-recipes-catalogue-json
+
+trigger:
+  event: tag

.env.sample

@@ -1,67 +1,81 @@
+# See https://mailu.io/2.0/configuration.html for explanation of (most of) these
+# settings. "Quoted text" in this file comes from that page.
+
+###############################################################################
+# BOILERPLATE SETTINGS (shouldn't need to change these)                       #
+###############################################################################
 TYPE=mailu
-
-# Main mail domain, NOT main web domain (if they are different)
-DOMAIN=example.com
 LETS_ENCRYPT_ENV=production
+COMPOSE_FILE="compose.yml"
 
-# Custom settings used by certdumper_post.sh and Traefik
-WEB_DOMAIN=example.com
-ACME_JSON=${LETS_ENCRYPT_ENV}-acme.json
+###############################################################################
+# REQUIRED SETTINGS (always need to change these)                             #
+###############################################################################
 
-# Mailu settings
-# https://mailu.io
+# Main web domain, NOT main mail domain (if they are different)
+DOMAIN=mailu.example.com
 
-TLS_CERT_FILENAME=$WEB_DOMAIN/certificate.crt
-TLS_KEYPAIR_FILENAME=$WEB_DOMAIN/privatekey.key
+# "This email domain is used for bounce emails, for generating the postmaster
+# email and other technical addresses."
+MAIL_DOMAIN=mailu.example.com
 
-REDIS_ADDRESS=db
+# Hostnames for this server, separated with commas
+# "The first declared hostname is the main hostname and will be exposed over
+# SMTP, IMAP, etc."
+HOSTNAMES=$DOMAIN
 
-# Set to a randomly generated 16 bytes string
-SECRET_KEY=XXXXXXXXXXXXXXXX
+# Run `DOCKER_CONTEXT=<yourserver> docker stack ls | grep traefik | cut -f 1 -d " "` to get that one
+TRAEFIK_STACK_NAME=traefik_example_com
 
-# Subnet of the docker network. This should not conflict with any networks to which your system is connected. (Internal and external!)
-SUBNET=192.168.203.0/24
+###############################################################################
+# OPTIONAL SETTINGS                                                           #
+###############################################################################
 
-# Hostnames for this server, separated with comas
-HOSTNAMES=$WEB_DOMAIN
+# Name of the instance, displayed in the web UI
+SITENAME=mymail
+
+# Linked Website URL
+WEBSITE=https://$DOMAIN
 
 # Postmaster local part (will append the main mail domain)
-POSTMASTER=admin
+POSTMASTER=c
 
-# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
-TLS_FLAVOR=mail
+# Authentication rate limit per IP (per /24 on ipv4 and /56 on ipv6)
+AUTH_RATELIMIT_IP=60/hour
 
-# Authentication rate limit (per source IP address)
-AUTH_RATELIMIT=10/minute
+# Opt-in to statistics, change to "False" to opt in
+DISABLE_STATISTICS=True
 
-# Opt-out of statistics, replace with "True" to opt out
-DISABLE_STATISTICS=False
+# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
+LOG_LEVEL=WARNING
 
-###################################
-# Optional features
-###################################
+# Timezone for the Mailu containers. See this link for all possible values https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+TZ=Etc/UTC
 
 # Expose the admin interface (value: true, false)
 ADMIN=true
 
-# Choose which webmail to run if any (values: roundcube, rainloop, none)
-WEBMAIL=rainloop
+# Choose which webmail to run if any (values: snappymail, roundcube, none)
+WEBMAIL=snappymail
 
-# Dav server implementation (value: radicale, none)
-WEBDAV=none
+# API settings
+# -------------
 
-# Antivirus solution (value: clamav, none)
-ANTIVIRUS=none
+# Authentication token for API requests
+API=false
+#API_TOKEN=
 
-###################################
 # Mail settings
-###################################
+# -------------
 
 # Message size limit in bytes
 # Default: accept messages up to 50MB
 # Max attachment size will be 33% smaller
 MESSAGE_SIZE_LIMIT=50000000
 
+# Message rate limit (per user)
+MESSAGE_RATELIMIT=200/day
+
 # Networks granted relay permissions
 # Use this with care, all hosts in this networks will be able to send mail without authentication!
 RELAYNETS=
@@ -69,10 +83,15 @@ RELAYNETS=
 # Will relay all outgoing mails if configured
 RELAYHOST=
 
+# Enable fetchmail
+FETCHMAIL_ENABLED=False
+#COMPOSE_FILE="$COMPOSE_FILE:compose.fetchmail.yml"
+#FETCHMAIL_ENABLED=True
+
 # Fetchmail delay
 FETCHMAIL_DELAY=600
 
-# Recipient delimiter, character used to delimiter localpart from custom address part
+# Recipient delimiter, character used to delimit localpart from custom address part
 RECIPIENT_DELIMITER=+
 
 # DMARC rua and ruf email
@@ -86,17 +105,64 @@ WELCOME_SUBJECT="Welcome to your new email account"
 WELCOME_BODY="Welcome to your new email account, if you can read this, then it is configured properly!"
 
 # Maildir Compression
-# choose compression-method, default: none (value: bz2, gz)
+# choose compression-method, default: none (value: gz, bz2, lz4, zstd)
 COMPRESSION=
 # change compression-level, default: 6 (value: 1-9)
 COMPRESSION_LEVEL=
 
 # IMAP full-text search is enabled by default. Set the following variable to off in order to disable the feature.
-# FULL_TEXT_SEARCH=off
+#FULL_TEXT_SEARCH=off
+
+# Co-op Cloud settings
+# -------------
+
+# Mailman settings
+# NOTE(3wc): remember to also set RELAYNETS
+#COMPOSE_FILE="$COMPOSE_FILE:compose.mailman.yml"
+#MAILMAN_POSTFIX_OVERRIDES=1
+#MAILMAN_CORE_VOLUME=lists_example_com_mailman-core
+#MAILMAN_CORE_NETWORK=lists_example_com_internal
+
+# NOTE(3wc): think this is no longer needed, see https://github.com/Mailu/Mailu/pull/1904
+# SASL account -> MAIL FROM mappings for more liberal MAIL FROM relaying
+# return-path: https://www.rubydoc.info/gems/actionmailer-rack-upgrade-2/2.3.15/ActionMailer/Base
+# logins and MAIL FROM ownership: http://www.postfix.com/postconf.5.html#smtpd_relay_restrictions
+# there is an open ticket with a further discussion also https://github.com/Mailu/Mailu/issues/1096
+#COMPOSE_FILE="$COMPOSE_FILE:compose.senderlogins.yml"
+#SENDER_LOGINS_POSTFIX_OVERRIDES=1
+
+SECRET_SECRET_KEY_VERSION=v1
+
+###############################################################################
+# INFINITE NERD DEPTH SETTINGS (rarely needed)                                #
+###############################################################################
+
+# Traefik certificates filename, used by certdumper
+ACME_JSON=${LETS_ENCRYPT_ENV}-acme.json
+
+TLS_CERT_FILENAME=$DOMAIN/certificate.crt
+TLS_KEYPAIR_FILENAME=$DOMAIN/privatekey.key
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+# NOTE(3wc): changing this to "letsencrypt" might (but probably won't) allow
+# this recipe to be deployed standalone without Traefik. "notls" might (but
+# might not) disable encryption for everything except the web interfaces.
+TLS_FLAVOR=mail
+
+# Optional features
+# -------------
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=none
+
+# Antivirus solution (value: clamav, none)
+ANTIVIRUS=none
+
+# Scan Macros solution (value: true, false)
+SCAN_MACROS=true
 
-###################################
 # Web settings
-###################################
+# -------------
 
 # Path to redirect / to
 WEBROOT_REDIRECT=/webmail
@@ -107,28 +173,17 @@ WEB_ADMIN=/admin
 # Path to the webmail if enabled
 WEB_WEBMAIL=/webmail
 
-# Website name
-SITENAME=mymail
-
-# Linked Website URL
-WEBSITE=https://$DOMAIN
-
-###################################
 # Advanced settings
-###################################
+# -------------
 
 # Log driver for front service. Possible values:
 # json-file (default)
 # journald (On systemd platforms, useful for Fail2Ban integration)
 # syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
-# LOG_DRIVER=json-file
+#LOG_DRIVER=json-file
 
-# Docker-compose project name, this will prepended to containers names.
-COMPOSE_PROJECT_NAME=mailu
-
-# Default password scheme used for newly created accounts and changed passwords
-# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT)
-PASSWORD_SCHEME=PBKDF2
+# Number of rounds used by the password hashing scheme
+CREDENTIAL_ROUNDS=12
 
 # Header to take the real ip from
 REAL_IP_HEADER=
@@ -139,27 +194,6 @@ REAL_IP_FROM=
 # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
 REJECT_UNLISTED_RECIPIENT=
 
-# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
-LOG_LEVEL=WARNING
-
-###################################
-# Database settings
-###################################
+# NOTE(3wc): Other values are not supported, as compose.yml does not contain
+# configuration for them. Pull requests welcome!
 DB_FLAVOR=sqlite
-
-###################################
-# Mailman settings
-###################################
-# COMPOSE_FILE="compose.yml:compose.mailman.yml"
-# MAILMAN_POSTFIX_OVERRIDES=1
-# MAILMAN_CORE_VOLUME=lists_example_com_mailman-core
-# MAILMAN_CORE_NETWORK=lists_example_com_internal
-
-###################################
-# SASL account -> MAIL FROM mappings for more liberal MAIL FROM relaying
-# return-path: https://www.rubydoc.info/gems/actionmailer-rack-upgrade-2/2.3.15/ActionMailer/Base
-# logins and MAIL FROM ownership: http://www.postfix.com/postconf.5.html#smtpd_relay_restrictions
-# there is an open ticket with a further discussion also https://github.com/Mailu/Mailu/issues/1096
-###################################
-# COMPOSE_FILE="compose.yml:compose.senderlogins.yml"
-# SENDER_LOGINS_POSTFIX_OVERRIDES=1

README.md

@@ -22,12 +22,12 @@ host.
 1. Set up Docker Swarm and [`abra`][abra]
 2. Deploy [`coop-cloud/traefik`][compose-traefik]
 3. `abra app new mailu`
-4. `abra app YOURAPPDOMAIN config` - be sure to change `$WEB_DOMAIN` to something that resolves to
+4. `abra app config YOURAPPDOMAIN` - be sure to change `$WEB_DOMAIN` to something that resolves to
    your Docker swarm box
-5. `abra app YOURAPPDOMAIN deploy`
+5. `abra app deploy YOURAPPDOMAIN`
 9. Create initial user:
 ```
-abra app YOURAPPDOMAIN run admin flask mailu admin admin YOURDOMAIN YOURPASSWORD
+abra app run YOURDOMAIN admin flask mailu admin admin YOURDOMAIN YOURPASSWORD
 ```
 
 [Mailu]: https://mailu.io/

abra.sh

@@ -1,3 +1,3 @@
 export CERTDUMPER_POST_VERSION=v1
-export POSTFIX_OVERRIDE_VERSION=v15
+export POSTFIX_OVERRIDE_VERSION=v17
 export SENDER_LOGIN_VERSIONS=v2

compose.fetchmail.yml

@@ -0,0 +1,14 @@
+
+services:
+  fetchmail:
+    image: ghcr.io/mailu/fetchmail:2.0.34
+    environment: *default-env
+    volumes:
+      - "fetchmail:/data"
+    depends_on:
+      - app
+    networks:
+      - default
+
+volumes:
+  fetchmail:

compose.mailman.yml

@@ -10,18 +10,12 @@ services:
       - MAILMAN_POSTFIX_OVERRIDES
     networks:
       - default
-      - shared_mailman_network
     volumes:
       - "shared-mailman-core:/opt/mailman/"
     configs:
       - source: postfix_override
         target: /overrides/postfix.cf
 
-networks:
-  shared_mailman_network:
-    external: true
-    name: ${MAILMAN_CORE_NETWORK}
-
 volumes:
   # https://git.autonomic.zone/coop-cloud/mailman3/src/branch/master/compose.yml
   shared-mailman-core:

compose.yml

@@ -5,7 +5,10 @@ x-environment:
   - FRONT_ADDRESS=${STACK_NAME}_app
   - ADMIN
   - ANTIVIRUS
-  - AUTH_RATELIMIT
+  - API
+  - API_TOKEN
+  - AUTH_RATELIMIT_IP
+  - MESSAGE_RATELIMIT
   - COMPOSE_PROJECT_NAME
   - COMPRESSION
   - COMPRESSION_LEVEL
@@ -15,6 +18,7 @@ x-environment:
   - DMARC_RUF
   - DOCKER_CONTEXT
   - DOMAIN
+  - FETCHMAIL_ENABLED
   - FETCHMAIL_DELAY
   - FULL_TEXT_SEARCH
   - HOSTNAMES
@@ -22,18 +26,18 @@ x-environment:
   - LOG_DRIVER
   - LOG_LEVEL
   - MESSAGE_SIZE_LIMIT
-  - PASSWORD_SCHEME
+  - CREDENTIAL_ROUNDS
   - POSTMASTER
   - REAL_IP_FROM
   - REAL_IP_HEADER
   - RECIPIENT_DELIMITER
-  - REDIS_ADDRESS
+  - REDIS_ADDRESS=db
   - REJECT_UNLISTED_RECIPIENT
   - RELAYHOST
   - RELAYNETS
-  - SECRET_KEY
+  - SECRET_KEY_FILE=/run/secrets/secret_key
   - SITENAME
-  - SUBNET
+  - SUBNET=192.168.203.0/24
   - TLS_CERT_FILENAME
   - TLS_FLAVOR
   - TLS_KEYPAIR_FILENAME
@@ -43,13 +47,15 @@ x-environment:
   - WEBROOT_REDIRECT
   - WEBSITE
   - WEB_WEBMAIL
+  - WEB_API=/api
   - WELCOME
   - WELCOME_BODY
   - WELCOME_SUBJECT
+  - TZ
 
 services:
   app:
-    image: mailu/nginx:1.8
+    image: ghcr.io/mailu/nginx:2.0.34
     logging:
       driver: json-file
     networks:
@@ -80,48 +86,67 @@ services:
         mode: host
     volumes:
       - "certs:/certs"
+    secrets:
+      - secret_key
     deploy:
       labels:
         - "traefik.enable=true"
         - "traefik.docker.network=proxy"
         - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=80"
-        - "traefik.http.routers.${STACK_NAME}.rule=Host(`${WEB_DOMAIN}`)"
+        - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
+        - "coop-cloud.${STACK_NAME}.version=1.1.0+2.0.34"
   
   db:
-    image: redis:alpine
+    image: redis:7.2.3-alpine
     volumes:
       - "redis:/data"
 
   admin:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:1.8
+    image: ghcr.io/mailu/admin:2.0.34
     environment: *default-env
     healthcheck:
       disable: true
+    secrets:
+      - secret_key
     volumes:
       - "dkim:/dkim"
       - "mailu:/data"
+    networks:
+      - default
 
   imap:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:1.8
+    image: ghcr.io/mailu/dovecot:2.0.34
     environment: *default-env
+    secrets:
+      - secret_key
     volumes:
       - "mail:/mail"
     healthcheck:
       disable: true
+    depends_on:
+      - app
+    networks:
+      - default
 
   smtp:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:1.8
+    image: ghcr.io/mailu/postfix:2.0.34
     environment: *default-env
+    secrets:
+      - secret_key
     volumes:
       - "mailqueue:/queue"
     healthcheck:
       disable: true
+    depends_on:
+      - app
 
   antispam:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:1.8
+    image: ghcr.io/mailu/rspamd:2.0.34
     environment: *default-env
+    secrets:
+      - secret_key
     volumes:
       - "rspamd:/var/lib/rspamd"
       - "dkim:/dkim:ro"
@@ -129,46 +154,38 @@ services:
       disable: true
 
   webmail:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:1.8
+    image: ghcr.io/mailu/webmail:2.0.34
     environment: *default-env
+    networks:
+      - default
     volumes:
       - "webmail:/data"
+    secrets:
+      - secret_key
     deploy:
       replicas: 1
     healthcheck:
       disable: true
 
-  #certdumper:
-  #  restart: always
-  #  image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}traefik-certdumper:master
-  #  environment:
-  #    - DOMAIN=$DOMAIN
-  #    # Set TRAEFIK_VERSION to v2 in your .env if you're using Traefik v2
-  #    - TRAEFIK_VERSION=${TRAEFIK_VERSION:-v2}
-  #  volumes:
-  #    - "/docker/traefik/letsencrypt/acme.json:/traefik/acme.json"
-  #    - "/docker/traefik/letsencrypt/certs:/tmp/work"
-  #    - "/docker/mailu/certs:/output"
-  #  labels:
-  #    # Set watchtower label
-  #    - "com.centurylinklabs.watchtower.enable=true"
-
   certdumper:
-    image: ldez/traefik-certs-dumper:v2.7.4
+    image: ldez/traefik-certs-dumper:v2.8.1
     entrypoint: sh -c '
       apk add jq
-      ; while ! [ -e /traefik/production-acme.json ]
-        || ! [ `jq ".production.Certificates | length" /traefik/production-acme.json` != 0 ]; do
+      ; while ! [ -e /traefik/${ACME_JSON} ]
+        || ! [ `jq ".production.Certificates | length" /traefik/${ACME_JSON}` != 0 ]; do
           sleep 1
       ; done
-      && traefik-certs-dumper file --watch --source /traefik/production-acme.json
-        --dest /output --domain-subdir=true --version v2'
+      && traefik-certs-dumper file --watch --source /traefik/${ACME_JSON}
+        --dest /output --domain-subdir=true --version v2
+        --post-hook "sh /usr/bin/certdumper_post.sh"'
     environment:
-      # Make sure this is the same as the main=-domain in traefik.toml
-      - DOMAIN=$WEB_DOMAIN
+      - DOMAIN=$DOMAIN
     volumes:
       # Folder, which contains the acme.json
-      - "traefik_letsencrypt:/traefik"
+      - type: volume
+        read_only: true
+        source: traefik_letsencrypt
+        target: "/traefik"
       # Folder, where cert.pem and key.pem will be written
       - "certs:/output"
     configs:
@@ -176,14 +193,6 @@ services:
         target: /usr/bin/certdumper_post.sh
         mode: 0555
 
-  #certdumper:
-  #  image: humenius/traefik-certs-dumper:latest
-  #  volumes:
-  #   - traefik_letsencrypt:/traefik:ro
-  #   - certs:/output:rw
-  #  environment:
-  #   - DOMAIN=$WEB_DOMAIN
-
 volumes:
   mailu:
   rspamd:
@@ -194,6 +203,7 @@ volumes:
   certs:
   mailqueue:
   traefik_letsencrypt:
+    name: "${TRAEFIK_STACK_NAME}_letsencrypt"
     external: true
 
 networks:
@@ -210,3 +220,8 @@ configs:
   certdumper_post:
     name: ${STACK_NAME}_certdumper_post_${CERTDUMPER_POST_VERSION}
     file: certdumper_post.sh
+
+secrets:
+  secret_key:
+    external: true
+    name: ${STACK_NAME}_secret_key_${SECRET_SECRET_KEY_VERSION}

postfix.cf.tmpl

@@ -10,9 +10,9 @@
 
 unknown_local_recipient_reject_code = 550
 owner_request_special = no
+virtual_mailbox_maps = regexp:/opt/mailman/var/data/postfix_lmtp, \${podop}mailbox
 transport_maps = regexp:/opt/mailman/var/data/postfix_lmtp \${podop}transport
 local_recipient_maps = regexp:/opt/mailman/var/data/postfix_lmtp
-relay_domains = regexp:/opt/mailman/var/data/postfix_domains \${podop}transport
 {{ end }}
 {{ if eq (env "SENDER_LOGINS_POSTFIX_OVERRIDES") "1" }}
 # https://github.com/Mailu/Mailu/issues/1096

release/0.2.0+1.9

@@ -0,0 +1,11 @@
+When upgrading to 1.9, you'll need to update your app(s') configuration(s) for
+new settings names:
+
+- Rename `AUTH_RATELIMIT` to `AUTH_RATELIMIT_IP`
+- Add MESSAGE_RATELIMIT (default `200/day`)
+- Add `TZ` to specify server timezone, e.g. `TZ=Etc/UTC`
+- Remove `PASSWORD_SCHEME`
+- Add `CREDENTIAL_ROUNDS` (default `12`)
+
+If you haven't made these changes already, it's best to bail on this upgrade
+FIRST (i.e. Ctrl+C) and run `abra app config` first.

release/1.0.1+2.0.16

@@ -0,0 +1 @@
+this version introduces a new variable TRAEFIK_STACK_NAME

release/1.1.0+2.0.34

@@ -0,0 +1,18 @@
+# Secret key
+
+The secret key is now stored as a docker secret instead of a variable in the
+env file. You need to insert it by running:
+`abra app secret insert <your mailu app> secret_key v1 <SECRETKEYHERE>`
+and you can remove `SECRET_KEY` from your config file.
+
+# `DOMAIN` is now `MAIL_DOMAIN`
+
+This is important! If your main e-mail domain is something else than the
+domain you are reaching the web interface at, you need to make changes.
+I.e. in our setup we have `autonomic.zone` pointing an A record at some server
+and `mail.autonomic.zone` is a separate server, and MX records are pointing
+to the server at `mail.autonomic.zone`.
+In this case:
+  - `DOMAIN` should be `mail.autonomic.zone` - this is what traefik will use to
+  provision a certificate
+  - `MAIL_DOMAIN` should be `autonomic.zone`