Merge pull request 'fix backup and restore when database was already created' (#5) from p4u1/plausible:fix-backup-restore into main

Reviewed-on: coop-cloud/plausible#5

.drone.yml

@@ -3,10 +3,12 @@ kind: pipeline
 name: deploy to swarm-test.autonomic.zone
 steps:
   - name: deployment
-    image: decentral1se/stack-ssh-deploy:latest
+    image: git.coopcloud.tech/coop-cloud/stack-ssh-deploy:latest
     settings:
       host: swarm-test.autonomic.zone
       stack: plausible
+      networks:
+       - proxy
       purge: true
       deploy_key:
         from_secret: drone_ssh_swarm_test
@@ -33,24 +35,17 @@ trigger:
 
 ---
 kind: pipeline
-name: recipe release
+name: generate recipe catalogue
 steps:
   - name: release a new version
-    image: decentral1se/drone-abra:latest
-    settings:
-      command: recipe plausible release
-      deploy_key:
-        from_secret: abra_bot_deploy_key
-
-  - name: trigger downstream builds
     image: plugins/downstream
     settings:
-      server: https://drone.autonomic.zone
+      server: https://build.coopcloud.tech
       token:
-        from_secret: decentral1se_token
+        from_secret: drone_abra-bot_token
       fork: true
       repositories:
-        - coop-cloud/auto-apps-json
-    depends_on:
-      - release a new version
-rsion
+        - coop-cloud/auto-recipes-catalogue-json
+
+trigger:
+  event: tag

.env.sample

@@ -6,9 +6,6 @@ DOMAIN=plausible.example.com
 #EXTRA_DOMAINS=', `www.plausible.example.com`'
 LETS_ENCRYPT_ENV=production
 
-ADMIN_USER_EMAIL=replace-me
-ADMIN_USER_NAME=replace-me
-ADMIN_USER_PWD=replace-me
 SECRET_KEY_BASE=replace-me
 DISABLE_AUTH=replace-me # true or false
 DISABLE_REGISTRATION=replace-me # true or false

README.md

@@ -1,16 +1,16 @@
 # plausible
 
-TODO
+*Status:* Work in progress
 
 <!-- metadata -->
 * **Category**: Apps
 * **Status**: 1, alpha
 * **Image**: [`plausible/analytics`](https://hub.docker.com/plausible/analytics), 4, upstream
 * **Healthcheck**:
-* **Backups**:
-* **Email**:
+* **Backups**: Yes
+* **Email**: No
 * **Tests**:
-* **SSO**:
+* **SSO**: No
 <!-- endmetadata -->
 
 ## Basic usage
@@ -19,9 +19,9 @@ TODO
 2. Deploy [`coop-cloud/traefik`]
 3. `abra app new ${REPO_NAME} --secrets` (optionally with `--pass` if you'd like
    to save secrets in `pass`)
-4. `abra app YOURAPPDOMAIN config` - be sure to change `$DOMAIN` to something that resolves to
+4. `abra app config YOURAPPDOMAIN` - be sure to change `$DOMAIN` to something that resolves to
    your Docker swarm box
-5. `abra app YOURAPPDOMAIN deploy`
+5. `abra app deploy YOURAPPDOMAIN`
 6. Open the configured domain in your browser to finish set-up
 
 [`abra`]: https://git.coopcloud.tech/coop-cloud/abra

abra.sh

@@ -1,2 +1,4 @@
-export CLICKHOUSE_CONF_VERSION=v1
+export CLICKHOUSE_CONF_VERSION=v2
 export CLICKHOUSE_USER_CONF_VERSION=v2
+export DB_ENTRYPOINT_VERSION=v1
+export CLICKHOUSE_ENTRYPOINT_VERSION=v2

clickhouse-config.xml

@@ -1,4 +1,4 @@
-<yandex>
+<clickhouse>
     <logger>
         <level>warning</level>
         <console>true</console>
@@ -11,5 +11,5 @@
     <trace_log remove="remove"/>
     <metric_log remove="remove"/>
     <asynchronous_metric_log remove="remove"/>
-</yandex>
+</clickhouse>
 

compose.yml

@@ -3,20 +3,21 @@ version: "3.8"
 
 services:
   app:
-    image: plausible/analytics:latest
-    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh db init-admin && /entrypoint.sh run"
+    image: plausible/analytics:v2.0.0
+    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
     depends_on:
       - db
       - events_db
     environment:
       - BASE_URL=https://$DOMAIN
-      - ADMIN_USER_EMAIL
-      - ADMIN_USER_NAME
-      - ADMIN_USER_PWD
       - SECRET_KEY_BASE
       - DATABASE_URL=postgres://plausible:plausible@${STACK_NAME}_db:5432/plausible
       - SMTP_HOST_ADDR
       - MAILER_EMAIL
+      - SMTP_HOST_PORT
+      - SMTP_USER_NAME
+      - SMTP_USER_PWD
+      - SMTP_HOST_SSL_ENABLED
       - DISABLE_REGISTRATION
       - DISABLE_AUTH
     networks:
@@ -31,39 +32,55 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        ## Redirect from EXTRA_DOMAINS to DOMAIN
-        #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
-        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
-        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
-    # healthcheck:
-    #   test: ["CMD", "curl", "-f", "http://localhost"]
-    #   interval: 30s
-    #   timeout: 10s
-    #   retries: 10
-    #   start_period: 1m
-
+        - coop-cloud.${STACK_NAME}.version=3.0.1+v2.0.0
   db:
-    image: postgres:12
+    image: postgres:13.12
+    configs:
+      - source: db_entrypoint
+        target: /docker-entrypoint.sh
+        mode: 0555
+    # Custom docker entrypoint to handle major Postgres version upgrades
     volumes:
       - db-data:/var/lib/postgresql/data
+    entrypoint: /docker-entrypoint.sh
     environment:
       - POSTGRES_USER=plausible
       - POSTGRES_PASSWORD=plausible
       - POSTGRES_DB=plausible
     networks:
       - internal
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.pre-hook: sh -c 'pg_dump -U "$$POSTGRES_USER" -Fc "$$POSTGRES_DB" | gzip > "/postgres.dump.gz"'
+        backupbot.backup.path: "/postgres.dump.gz"
+        backupbot.backup.post-hook: "rm -f /postgres.dump.gz"
+        backupbot.restore: "true"
+        backupbot.restore.post-hook: sh -c 'gzip -d /postgres.dump.gz && pg_restore --clean -U "$$POSTGRES_USER" --dbname="$$PLAUSIBLE_DB" < /postgres.dump && rm -f /postgres.dump'
 
   plausible_events_db:
-    image: yandex/clickhouse-server:21.3.2.5
+    image: clickhouse/clickhouse-server:23.4.2.11-alpine
     volumes:
       - event-data:/var/lib/clickhouse
+    entrypoint: /custom-entrypoint.sh
     configs:
       - source: clickhouse-config
         target: /etc/clickhouse-server/config.d/logging.xml
       - source: clickhouse-user-config
         target: /etc/clickhouse-server/users.d/clickhouse-user-config.xml
+      - source: clickhouse_entrypoint
+        target: /custom-entrypoint.sh
+        mode: 0555
     networks:
       - internal
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.pre-hook: clickhouse-backup create events
+        backupbot.backup.path: "/var/lib/clickhouse/backup/events"
+        backupbot.backup.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
+        backupbot.restore: "true"
+        backupbot.restore.post-hook: clickhouse-backup restore --rm events && rm -rf /var/lib/clickhouse/backup/events"
 
 volumes:
   db-data:
@@ -81,3 +98,10 @@ configs:
   clickhouse-user-config:
     name: ${STACK_NAME}_clickhouse_user_config_${CLICKHOUSE_USER_CONF_VERSION}
     file: clickhouse-user-config.xml
+  db_entrypoint:
+    name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
+    file: entrypoint.postgres.sh.tmpl
+    template_driver: golang
+  clickhouse_entrypoint:
+    name: ${STACK_NAME}_clickhouse_entrypoint_${CLICKHOUSE_ENTRYPOINT_VERSION}
+    file: entrypoint.clickhouse.sh

compose.yml~

@@ -0,0 +1,102 @@
+---
+version: "3.8"
+
+services:
+  app:
+    image: plausible/analytics:v2.0.0
+    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
+    depends_on:
+      - db
+      - events_db
+    environment:
+      - BASE_URL=https://$DOMAIN
+      - SECRET_KEY_BASE
+      - DATABASE_URL=postgres://plausible:plausible@${STACK_NAME}_db:5432/plausible
+      - SMTP_HOST_ADDR
+      - MAILER_EMAIL
+      - DISABLE_REGISTRATION
+      - DISABLE_AUTH
+    networks:
+      - proxy
+      - internal
+    deploy:
+      restart_policy:
+        condition: on-failure
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=8000"
+        - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
+        - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        - coop-cloud.${STACK_NAME}.version=3.0.1+v2.0.0
+  db:
+    image: postgres:13.12
+    configs:
+      - source: db_entrypoint
+        target: /docker-entrypoint.sh
+        mode: 0555
+    # Custom docker entrypoint to handle major Postgres version upgrades
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    entrypoint: /docker-entrypoint.sh
+    environment:
+      - POSTGRES_USER=plausible
+      - POSTGRES_PASSWORD=plausible
+      - POSTGRES_DB=plausible
+    networks:
+      - internal
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.pre-hook: sh -c 'pg_dump -U "$$POSTGRES_USER" -Fc "$$POSTGRES_DB" | gzip > "/postgres.dump.gz"'
+        backupbot.backup.path: "/postgres.dump.gz"
+        backupbot.backup.post-hook: "rm -f /postgres.dump.gz"
+        backupbot.restore.post-hook: sh -c 'gzip -d /postgres.dump.gz && pg_restore -U "$$POSTGRES_USER" --dbname="$$PLAUSIBLE_DB" < /postgres.dump && rm /postgres.dump'
+
+  plausible_events_db:
+    image: clickhouse/clickhouse-server:23.4.2.11-alpine
+    volumes:
+      - event-data:/var/lib/clickhouse
+    entrypoint: /custom-entrypoint.sh
+    configs:
+      - source: clickhouse-config
+        target: /etc/clickhouse-server/config.d/logging.xml
+      - source: clickhouse-user-config
+        target: /etc/clickhouse-server/users.d/clickhouse-user-config.xml
+      - source: clickhouse_entrypoint
+        target: /custom-entrypoint.sh
+        mode: 0555
+    networks:
+      - internal
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.pre-hook: clickhouse-backup create events
+        backupbot.backup.path: "/var/lib/clickhouse/backup/events"
+        backupbot.backup.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
+        backupbot.restore.post-hook: clickhouse-backup restore events
+        backupbot.resoter.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
+
+volumes:
+  db-data:
+  event-data:
+
+networks:
+  proxy:
+    external: true
+  internal:
+
+configs:
+  clickhouse-config:
+    name: ${STACK_NAME}_clickhouse_config_${CLICKHOUSE_CONF_VERSION}
+    file: clickhouse-config.xml
+  clickhouse-user-config:
+    name: ${STACK_NAME}_clickhouse_user_config_${CLICKHOUSE_USER_CONF_VERSION}
+    file: clickhouse-user-config.xml
+  db_entrypoint:
+    name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
+    file: entrypoint.postgres.sh.tmpl
+    template_driver: golang
+  clickhouse_entrypoint:
+    name: ${STACK_NAME}_clickhouse_entrypoint_${CLICKHOUSE_ENTRYPOINT_VERSION}
+    file: entrypoint.clickhouse.sh

entrypoint.clickhouse.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+CLICKHOUSE_BACKUP_VERSION=2.4.2
+
+ARCH=$(uname -m)
+if [[ $ARCH =~ "aarch64" ]]; then
+  ARCH="arm64"
+elif [[ $ARCH =~ "armv5l" ]]; then
+  ARCH="armv5"
+elif [[ $ARCH =~ "armv6l" ]]; then
+  ARCH="armv6"
+elif [[ $ARCH =~ "armv7l" ]]; then
+  ARCH="armv7"
+elif [[ $ARCH =~ "x86_64" ]]; then
+  ARCH="amd64"
+fi
+
+wget \
+    --quiet \
+    --continue \
+    --no-clobber \
+    --output-document=/tmp/clickhouse-backup.tar.gz \
+    "https://github.com/AlexAkulov/clickhouse-backup/releases/download/v${CLICKHOUSE_BACKUP_VERSION}/clickhouse-backup-linux-${ARCH}.tar.gz" 2>/dev/null
+
+tar -xf /tmp/clickhouse-backup.tar.gz --directory=/usr/local/bin --strip-components=3
+
+/entrypoint.sh

entrypoint.postgres.sh.tmpl

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+MIGRATION_MARKER=$PGDATA/migration_in_progress
+OLDDATA=$PGDATA/old_data
+NEWDATA=$PGDATA/new_data
+
+if [ -e $MIGRATION_MARKER ]; then
+  echo "FATAL: migration was started but did not complete in a previous run. manual recovery necessary"
+  exit 1
+fi
+
+if [ -f $PGDATA/PG_VERSION ]; then
+  DATA_VERSION=$(cat $PGDATA/PG_VERSION)
+
+  if [ -n "$DATA_VERSION" -a "$PG_MAJOR" != "$DATA_VERSION" ]; then
+    echo "postgres data version $DATA_VERSION found, but need $PG_MAJOR. Starting migration"
+    echo "Installing postgres $DATA_VERSION"
+    sed -i "s/$/ $DATA_VERSION/" /etc/apt/sources.list.d/pgdg.list
+    apt-get update && apt-get install -y --no-install-recommends \
+      postgresql-$DATA_VERSION \
+      && rm -rf /var/lib/apt/lists/*
+    echo "shuffling around"
+    gosu postgres mkdir $OLDDATA $NEWDATA
+    chmod 700 $OLDDATA $NEWDATA
+    mv $PGDATA/* $OLDDATA/ || true
+    touch $MIGRATION_MARKER
+    echo "running initdb"
+    # abuse entrypoint script for initdb by making server error out
+    gosu postgres bash -c "export PGDATA=$NEWDATA ; /usr/local/bin/docker-entrypoint.sh --invalid-arg || true"
+    echo "running pg_upgrade"
+    cd /tmp
+    gosu postgres pg_upgrade --link -b /usr/lib/postgresql/$DATA_VERSION/bin -d $OLDDATA -D $NEWDATA -U $POSTGRES_USER
+    cp $OLDDATA/pg_hba.conf $NEWDATA/
+    mv $NEWDATA/* $PGDATA
+    rm -rf $OLDDATA
+    rmdir $NEWDATA
+    rm $MIGRATION_MARKER
+    echo "migration complete"
+  fi
+fi
+
+/usr/local/bin/docker-entrypoint.sh postgres

release/2.0.0+v1.5.1

@@ -0,0 +1,7 @@
+If you're upgrading from a pre-release version, there will be a major Postgresql
+version upgrade -- this should happen automatically, but **please take a
+backup**, at least of the `<stack_name>_db` volume, if not all the data volumes,
+before the upgrade. 
+
+If you haven't taken a backup already, it's probably safest to bail using
+Ctrl+C, take the backup, and re-run your `upgrade` / `deploy` command.

release/3.0.0+v2.0.0

@@ -0,0 +1,8 @@
+⚠ WARNING! ⚠
+
+This major version upgrade of Plausible requires running a manual data migration
+-- otherwise you'll see all historical data disappear (don't worry, it's
+"probably" still there). 
+
+Take a manual docker volume backup, then see here, and strap in: 
+https://github.com/plausible/analytics/discussions/3132