fix: restore Mailu SQLite database in place

Reviewed-on: #23

.drone.yml

@@ -10,7 +10,7 @@ steps:
         from_secret: drone_abra-bot_token
       fork: true
       repositories:
-        - coop-cloud/auto-recipes-catalogue-json
+        - toolshed/auto-recipes-catalogue-json
 
 trigger:
   event: tag

.env.sample

@@ -31,6 +31,9 @@ TRAEFIK_STACK_NAME=traefik_example_com
 # OPTIONAL SETTINGS                                                           #
 ###############################################################################
 
+# Uncomment this to reënable STARTTLS, although beware the health warnings here: https://nostarttls.secvuln.info/
+#PORTS=25,80,443,465,993,995,4190,110,143,587
+
 # Name of the instance, displayed in the web UI
 SITENAME=mymail
 
@@ -85,8 +88,6 @@ RELAYHOST=
 
 # Enable fetchmail
 FETCHMAIL_ENABLED=False
-#COMPOSE_FILE="$COMPOSE_FILE:compose.fetchmail.yml"
-#FETCHMAIL_ENABLED=True
 
 # Fetchmail delay
 FETCHMAIL_DELAY=600
@@ -118,7 +119,7 @@ COMPRESSION_LEVEL=
 
 # Mailman settings
 # NOTE(3wc): remember to also set RELAYNETS
-#COMPOSE_FILE="$COMPOSE_FILE:compose.mailman.yml"
+#COMPOSE_FILE="compose.yml:compose.mailman.yml"
 #MAILMAN_POSTFIX_OVERRIDES=1
 #MAILMAN_CORE_VOLUME=lists_example_com_mailman-core
 #MAILMAN_CORE_NETWORK=lists_example_com_internal
@@ -128,7 +129,7 @@ COMPRESSION_LEVEL=
 # return-path: https://www.rubydoc.info/gems/actionmailer-rack-upgrade-2/2.3.15/ActionMailer/Base
 # logins and MAIL FROM ownership: http://www.postfix.com/postconf.5.html#smtpd_relay_restrictions
 # there is an open ticket with a further discussion also https://github.com/Mailu/Mailu/issues/1096
-#COMPOSE_FILE="$COMPOSE_FILE:compose.senderlogins.yml"
+#COMPOSE_FILE="compose.yml:compose.senderlogins.yml"
 #SENDER_LOGINS_POSTFIX_OVERRIDES=1
 
 SECRET_SECRET_KEY_VERSION=v1
@@ -197,3 +198,5 @@ REJECT_UNLISTED_RECIPIENT=
 # NOTE(3wc): Other values are not supported, as compose.yml does not contain
 # configuration for them. Pull requests welcome!
 DB_FLAVOR=sqlite
+
+ENABLE_BACKUPS=true

README.md

@@ -4,10 +4,10 @@ Coöp Cloud + [Mailu][Mailu] = 🥳
 
 <!-- metadata -->
 * **Category**: Apps
-* **Status**: 1, alpha
+* **Status**: 2, beta
 * **Image**: [`mailu/*`](https://hub.docker.com/r/mailu/), 4, upstream
 * **Healthcheck**: No
-* **Backups**: No
+* **Backups**: Yes
 * **Email**: N/A
 * **Tests**: No
 * **SSO**: No
@@ -25,11 +25,28 @@ host.
 4. `abra app config YOURAPPDOMAIN` - be sure to change `$WEB_DOMAIN` to something that resolves to
    your Docker swarm box
 5. `abra app deploy YOURAPPDOMAIN`
-9. Create initial user:
+6. Create initial user:
 ```
 abra app run YOURDOMAIN admin flask mailu admin admin YOURDOMAIN YOURPASSWORD
 ```
 
+## Upgrading the recipe
+
+**NOTE** This section is only intended for [recipe maintainers][maintainers]
+
+Minor/patch updates usually work OK with `abra recipe upgrade` and friends. 
+
+For major updates:
+
+1. Go to https://setup.mailu.io/ and generate a new config
+2. Download the new config and compare it to `compose.yml`. Pay attention to:
+  - New / removed services
+  - Added/removed config options
+  - Changes to networking
+3. Test upgrading from the previous version to the new version
+4. `abra recipe sync` / `abra recipe release` as normal
+
 [Mailu]: https://mailu.io/
 [abra]: https://git.autonomic.zone/autonomic-cooperative/abra
 [compose-traefik]: https://git.autonomic.zone/coop-cloud/traefik
+[maintainers]: https://docs.coopcloud.tech/maintainers/

abra.sh

@@ -1,3 +1,5 @@
 export CERTDUMPER_POST_VERSION=v1
 export POSTFIX_OVERRIDE_VERSION=v17
 export SENDER_LOGIN_VERSIONS=v2
+export BACKUP_SQLITE_VERSION=v1
+export RESTORE_SQLITE_VERSION=v1

backup_sqlite.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import os
+import sqlite3
+from pathlib import Path
+
+source_path = Path("/data/main.db")
+backup_dir = Path("/data/backup")
+temporary_path = backup_dir / "main.db.tmp"
+backup_path = backup_dir / "main.db"
+
+os.umask(0o077)
+backup_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
+temporary_path.unlink(missing_ok=True)
+
+try:
+    # Read-only mode prevents silently creating an empty source database.
+    source = sqlite3.connect(
+        f"{source_path.as_uri()}?mode=ro",
+        uri=True,
+        timeout=60,
+    )
+    destination = sqlite3.connect(temporary_path)
+    try:
+        source.backup(destination, pages=256, sleep=0.1)
+        result = destination.execute("PRAGMA integrity_check").fetchall()
+        if result != [("ok",)]:
+            raise RuntimeError(f"SQLite integrity check failed: {result}")
+    finally:
+        destination.close()
+        source.close()
+    # Only publish a fully written, verified snapshot.
+    os.replace(temporary_path, backup_path)
+    print(f"SQLite backup ready: {backup_path}", flush=True)
+except Exception:
+    temporary_path.unlink(missing_ok=True)
+    raise

compose.fetchmail.yml

@@ -1,14 +0,0 @@
-
-services:
-  fetchmail:
-    image: ghcr.io/mailu/fetchmail:2.0.34
-    environment: *default-env
-    volumes:
-      - "fetchmail:/data"
-    depends_on:
-      - app
-    networks:
-      - default
-
-volumes:
-  fetchmail:

compose.yml

@@ -18,7 +18,6 @@ x-environment:
   - DMARC_RUF
   - DOCKER_CONTEXT
   - DOMAIN
-  - FETCHMAIL_ENABLED
   - FETCHMAIL_DELAY
   - FULL_TEXT_SEARCH
   - HOSTNAMES
@@ -52,10 +51,11 @@ x-environment:
   - WELCOME_BODY
   - WELCOME_SUBJECT
   - TZ
+  - PORTS
 
 services:
   app:
-    image: ghcr.io/mailu/nginx:2.0.34
+    image: ghcr.io/mailu/nginx:2024.06.67
     logging:
       driver: json-file
     networks:
@@ -96,15 +96,18 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`)"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
-        - "coop-cloud.${STACK_NAME}.version=1.1.0+2.0.34"
+        - "coop-cloud.${STACK_NAME}.version=4.0.1+2024.06.67"
+        - "backupbot.backup=${ENABLE_BACKUPS:-true}"
+        - "backupbot.backup.volumes.traefik_letsencrypt=false"
+        - "backupbot.backup.volumes.mailu.path=backup"
   
   db:
-    image: redis:7.2.3-alpine
+    image: redis:8.10.3-alpine
     volumes:
       - "redis:/data"
 
   admin:
-    image: ghcr.io/mailu/admin:2.0.34
+    image: ghcr.io/mailu/admin:2024.06.67
     environment: *default-env
     healthcheck:
       disable: true
@@ -115,9 +118,20 @@ services:
       - "mailu:/data"
     networks:
       - default
+    configs:
+      - source: backup_sqlite
+        target: /usr/local/bin/backup_sqlite.py
+        mode: 0444
+      - source: restore_sqlite
+        target: /usr/local/bin/restore_sqlite.py
+        mode: 0444
+    deploy:
+      labels:
+        backupbot.backup.pre-hook: "python3 /usr/local/bin/backup_sqlite.py"
+        backupbot.restore.post-hook: "python3 /usr/local/bin/restore_sqlite.py"
 
   imap:
-    image: ghcr.io/mailu/dovecot:2.0.34
+    image: ghcr.io/mailu/dovecot:2024.06.67
     environment: *default-env
     secrets:
       - secret_key
@@ -131,7 +145,7 @@ services:
       - default
 
   smtp:
-    image: ghcr.io/mailu/postfix:2.0.34
+    image: ghcr.io/mailu/postfix:2024.06.67
     environment: *default-env
     secrets:
       - secret_key
@@ -143,7 +157,7 @@ services:
       - app
 
   antispam:
-    image: ghcr.io/mailu/rspamd:2.0.34
+    image: ghcr.io/mailu/rspamd:2024.06.67
     environment: *default-env
     secrets:
       - secret_key
@@ -154,7 +168,8 @@ services:
       disable: true
 
   webmail:
-    image: ghcr.io/mailu/webmail:2.0.34
+    image: ghcr.io/mailu/webmail:2024.06.67
+    # entrypoint: "tail -f /dev/null"
     environment: *default-env
     networks:
       - default
@@ -168,7 +183,7 @@ services:
       disable: true
 
   certdumper:
-    image: ldez/traefik-certs-dumper:v2.8.1
+    image: ldez/traefik-certs-dumper:v2.11.4
     entrypoint: sh -c '
       apk add jq
       ; while ! [ -e /traefik/${ACME_JSON} ]
@@ -220,6 +235,13 @@ configs:
   certdumper_post:
     name: ${STACK_NAME}_certdumper_post_${CERTDUMPER_POST_VERSION}
     file: certdumper_post.sh
+  backup_sqlite:
+    name: ${STACK_NAME}_backup_sqlite_${BACKUP_SQLITE_VERSION}
+    file: backup_sqlite.py
+  restore_sqlite:
+    name: ${STACK_NAME}_restore_sqlite_${RESTORE_SQLITE_VERSION}
+    file: restore_sqlite.py
+
 
 secrets:
   secret_key:

release/3.0.0+2024.06.27

@@ -0,0 +1,3 @@
+ALERTA! 🚨 Mailu has dropped support for STARTTLS by default – see details here: https://mailu.io/2024.06/releases.html#starttls-ports-disabled-by-default
+
+If you are relying on STARTTLS, e.g. SMTP port 587, please copy the example `PORTS` variable from `~/.abra/recipes/mailu/.env.example` BEFORE deploying.

release/3.2.0+2024.06.61

@@ -0,0 +1 @@
+Contains mitigation measures for php2shell

release/3.2.1+2024.06.66

@@ -0,0 +1 @@
+Actual mitigation for php2shell pending further alpine image upgrades

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}

restore_sqlite.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""Restore Mailu's SQLite snapshot without replacing the live database inode."""
+
+import sqlite3
+from pathlib import Path
+
+backup_path = Path("/data/backup/main.db")
+live_path = Path("/data/main.db")
+
+# The source and destination must already exist. A missing snapshot or live
+# database is a restore failure, not a reason to create an empty database.
+with sqlite3.connect(
+    f"{backup_path.as_uri()}?mode=ro", uri=True, timeout=60
+) as source:
+    if source.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
+        raise RuntimeError(f"Invalid SQLite snapshot: {backup_path}")
+    with sqlite3.connect(
+        f"{live_path.as_uri()}?mode=rw", uri=True, timeout=60
+    ) as destination:
+        source.backup(destination, pages=256, sleep=0.1)
+
+print(f"SQLite restore complete: {live_path}", flush=True)