Changes to clickhouse logger

Reviewed-on: javielico/plausible#1

.env.sample

@@ -6,6 +6,9 @@ DOMAIN=plausible.example.com
 #EXTRA_DOMAINS=', `www.plausible.example.com`'
 LETS_ENCRYPT_ENV=production
 
+ADMIN_USER_EMAIL=replace-me
+ADMIN_USER_NAME=replace-me
+ADMIN_USER_PWD=replace-me
 SECRET_KEY_BASE=replace-me
 DISABLE_AUTH=replace-me # true or false
 DISABLE_REGISTRATION=replace-me # true or false

README.md

@@ -7,7 +7,7 @@
 * **Status**: 1, alpha
 * **Image**: [`plausible/analytics`](https://hub.docker.com/plausible/analytics), 4, upstream
 * **Healthcheck**:
-* **Backups**: Yes
+* **Backups**: No
 * **Email**: No
 * **Tests**:
 * **SSO**: No

abra.sh

@@ -1,4 +1,2 @@
-export CLICKHOUSE_CONF_VERSION=v2
+export CLICKHOUSE_CONF_VERSION=v1
 export CLICKHOUSE_USER_CONF_VERSION=v2
-export DB_ENTRYPOINT_VERSION=v1
-export CLICKHOUSE_ENTRYPOINT_VERSION=v2

compose.yml

@@ -3,21 +3,20 @@ version: "3.8"
 
 services:
   app:
-    image: plausible/analytics:v2.0.0
-    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
+    image: plausible/analytics:v1.5.1
+    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh db init-admin && /entrypoint.sh run"
     depends_on:
       - db
       - events_db
     environment:
       - BASE_URL=https://$DOMAIN
+      - ADMIN_USER_EMAIL
+      - ADMIN_USER_NAME
+      - ADMIN_USER_PWD
       - SECRET_KEY_BASE
       - DATABASE_URL=postgres://plausible:plausible@${STACK_NAME}_db:5432/plausible
       - SMTP_HOST_ADDR
       - MAILER_EMAIL
-      - SMTP_HOST_PORT
-      - SMTP_USER_NAME
-      - SMTP_USER_PWD
-      - SMTP_HOST_SSL_ENABLED
       - DISABLE_REGISTRATION
       - DISABLE_AUTH
     networks:
@@ -32,55 +31,29 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - coop-cloud.${STACK_NAME}.version=3.0.1+v2.0.0
+        - coop-cloud.${STACK_NAME}.version=1.1+1.5.1
   db:
-    image: postgres:13.12
-    configs:
-      - source: db_entrypoint
-        target: /docker-entrypoint.sh
-        mode: 0555
-    # Custom docker entrypoint to handle major Postgres version upgrades
+    image: postgres:13.11-alpine
     volumes:
       - db-data:/var/lib/postgresql/data
-    entrypoint: /docker-entrypoint.sh
     environment:
       - POSTGRES_USER=plausible
       - POSTGRES_PASSWORD=plausible
       - POSTGRES_DB=plausible
     networks:
       - internal
-    deploy:
-      labels:
-        backupbot.backup: "true"
-        backupbot.backup.pre-hook: sh -c 'pg_dump -U "$$POSTGRES_USER" -Fc "$$POSTGRES_DB" | gzip > "/postgres.dump.gz"'
-        backupbot.backup.path: "/postgres.dump.gz"
-        backupbot.backup.post-hook: "rm -f /postgres.dump.gz"
-        backupbot.restore: "true"
-        backupbot.restore.post-hook: sh -c 'gzip -d /postgres.dump.gz && pg_restore --clean -U "$$POSTGRES_USER" --dbname="$$PLAUSIBLE_DB" < /postgres.dump && rm -f /postgres.dump'
 
   plausible_events_db:
     image: clickhouse/clickhouse-server:23.4.2.11-alpine
     volumes:
       - event-data:/var/lib/clickhouse
-    entrypoint: /custom-entrypoint.sh
     configs:
       - source: clickhouse-config
         target: /etc/clickhouse-server/config.d/logging.xml
       - source: clickhouse-user-config
         target: /etc/clickhouse-server/users.d/clickhouse-user-config.xml
-      - source: clickhouse_entrypoint
-        target: /custom-entrypoint.sh
-        mode: 0555
     networks:
       - internal
-    deploy:
-      labels:
-        backupbot.backup: "true"
-        backupbot.backup.pre-hook: clickhouse-backup create events
-        backupbot.backup.path: "/var/lib/clickhouse/backup/events"
-        backupbot.backup.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
-        backupbot.restore: "true"
-        backupbot.restore.post-hook: clickhouse-backup restore --rm events && rm -rf /var/lib/clickhouse/backup/events"
 
 volumes:
   db-data:
@@ -98,10 +71,3 @@ configs:
   clickhouse-user-config:
     name: ${STACK_NAME}_clickhouse_user_config_${CLICKHOUSE_USER_CONF_VERSION}
     file: clickhouse-user-config.xml
-  db_entrypoint:
-    name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
-    file: entrypoint.postgres.sh.tmpl
-    template_driver: golang
-  clickhouse_entrypoint:
-    name: ${STACK_NAME}_clickhouse_entrypoint_${CLICKHOUSE_ENTRYPOINT_VERSION}
-    file: entrypoint.clickhouse.sh

compose.yml~

@@ -1,102 +0,0 @@
----
-version: "3.8"
-
-services:
-  app:
-    image: plausible/analytics:v2.0.0
-    command: sh -c "sleep 10 && /entrypoint.sh db createdb && /entrypoint.sh db migrate && /entrypoint.sh run"
-    depends_on:
-      - db
-      - events_db
-    environment:
-      - BASE_URL=https://$DOMAIN
-      - SECRET_KEY_BASE
-      - DATABASE_URL=postgres://plausible:plausible@${STACK_NAME}_db:5432/plausible
-      - SMTP_HOST_ADDR
-      - MAILER_EMAIL
-      - DISABLE_REGISTRATION
-      - DISABLE_AUTH
-    networks:
-      - proxy
-      - internal
-    deploy:
-      restart_policy:
-        condition: on-failure
-      labels:
-        - "traefik.enable=true"
-        - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=8000"
-        - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
-        - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
-        - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - coop-cloud.${STACK_NAME}.version=3.0.1+v2.0.0
-  db:
-    image: postgres:13.12
-    configs:
-      - source: db_entrypoint
-        target: /docker-entrypoint.sh
-        mode: 0555
-    # Custom docker entrypoint to handle major Postgres version upgrades
-    volumes:
-      - db-data:/var/lib/postgresql/data
-    entrypoint: /docker-entrypoint.sh
-    environment:
-      - POSTGRES_USER=plausible
-      - POSTGRES_PASSWORD=plausible
-      - POSTGRES_DB=plausible
-    networks:
-      - internal
-    deploy:
-      labels:
-        backupbot.backup: "true"
-        backupbot.backup.pre-hook: sh -c 'pg_dump -U "$$POSTGRES_USER" -Fc "$$POSTGRES_DB" | gzip > "/postgres.dump.gz"'
-        backupbot.backup.path: "/postgres.dump.gz"
-        backupbot.backup.post-hook: "rm -f /postgres.dump.gz"
-        backupbot.restore.post-hook: sh -c 'gzip -d /postgres.dump.gz && pg_restore -U "$$POSTGRES_USER" --dbname="$$PLAUSIBLE_DB" < /postgres.dump && rm /postgres.dump'
-
-  plausible_events_db:
-    image: clickhouse/clickhouse-server:23.4.2.11-alpine
-    volumes:
-      - event-data:/var/lib/clickhouse
-    entrypoint: /custom-entrypoint.sh
-    configs:
-      - source: clickhouse-config
-        target: /etc/clickhouse-server/config.d/logging.xml
-      - source: clickhouse-user-config
-        target: /etc/clickhouse-server/users.d/clickhouse-user-config.xml
-      - source: clickhouse_entrypoint
-        target: /custom-entrypoint.sh
-        mode: 0555
-    networks:
-      - internal
-    deploy:
-      labels:
-        backupbot.backup: "true"
-        backupbot.backup.pre-hook: clickhouse-backup create events
-        backupbot.backup.path: "/var/lib/clickhouse/backup/events"
-        backupbot.backup.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
-        backupbot.restore.post-hook: clickhouse-backup restore events
-        backupbot.resoter.post-hook: "rm -rf /var/lib/clickhouse/backup/events"
-
-volumes:
-  db-data:
-  event-data:
-
-networks:
-  proxy:
-    external: true
-  internal:
-
-configs:
-  clickhouse-config:
-    name: ${STACK_NAME}_clickhouse_config_${CLICKHOUSE_CONF_VERSION}
-    file: clickhouse-config.xml
-  clickhouse-user-config:
-    name: ${STACK_NAME}_clickhouse_user_config_${CLICKHOUSE_USER_CONF_VERSION}
-    file: clickhouse-user-config.xml
-  db_entrypoint:
-    name: ${STACK_NAME}_db_entrypoint_${DB_ENTRYPOINT_VERSION}
-    file: entrypoint.postgres.sh.tmpl
-    template_driver: golang
-  clickhouse_entrypoint:
-    name: ${STACK_NAME}_clickhouse_entrypoint_${CLICKHOUSE_ENTRYPOINT_VERSION}
-    file: entrypoint.clickhouse.sh

entrypoint.clickhouse.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-CLICKHOUSE_BACKUP_VERSION=2.4.2
-
-ARCH=$(uname -m)
-if [[ $ARCH =~ "aarch64" ]]; then
-  ARCH="arm64"
-elif [[ $ARCH =~ "armv5l" ]]; then
-  ARCH="armv5"
-elif [[ $ARCH =~ "armv6l" ]]; then
-  ARCH="armv6"
-elif [[ $ARCH =~ "armv7l" ]]; then
-  ARCH="armv7"
-elif [[ $ARCH =~ "x86_64" ]]; then
-  ARCH="amd64"
-fi
-
-wget \
-    --quiet \
-    --continue \
-    --no-clobber \
-    --output-document=/tmp/clickhouse-backup.tar.gz \
-    "https://github.com/AlexAkulov/clickhouse-backup/releases/download/v${CLICKHOUSE_BACKUP_VERSION}/clickhouse-backup-linux-${ARCH}.tar.gz" 2>/dev/null
-
-tar -xf /tmp/clickhouse-backup.tar.gz --directory=/usr/local/bin --strip-components=3
-
-/entrypoint.sh

entrypoint.postgres.sh.tmpl

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -e
-
-MIGRATION_MARKER=$PGDATA/migration_in_progress
-OLDDATA=$PGDATA/old_data
-NEWDATA=$PGDATA/new_data
-
-if [ -e $MIGRATION_MARKER ]; then
-  echo "FATAL: migration was started but did not complete in a previous run. manual recovery necessary"
-  exit 1
-fi
-
-if [ -f $PGDATA/PG_VERSION ]; then
-  DATA_VERSION=$(cat $PGDATA/PG_VERSION)
-
-  if [ -n "$DATA_VERSION" -a "$PG_MAJOR" != "$DATA_VERSION" ]; then
-    echo "postgres data version $DATA_VERSION found, but need $PG_MAJOR. Starting migration"
-    echo "Installing postgres $DATA_VERSION"
-    sed -i "s/$/ $DATA_VERSION/" /etc/apt/sources.list.d/pgdg.list
-    apt-get update && apt-get install -y --no-install-recommends \
-      postgresql-$DATA_VERSION \
-      && rm -rf /var/lib/apt/lists/*
-    echo "shuffling around"
-    gosu postgres mkdir $OLDDATA $NEWDATA
-    chmod 700 $OLDDATA $NEWDATA
-    mv $PGDATA/* $OLDDATA/ || true
-    touch $MIGRATION_MARKER
-    echo "running initdb"
-    # abuse entrypoint script for initdb by making server error out
-    gosu postgres bash -c "export PGDATA=$NEWDATA ; /usr/local/bin/docker-entrypoint.sh --invalid-arg || true"
-    echo "running pg_upgrade"
-    cd /tmp
-    gosu postgres pg_upgrade --link -b /usr/lib/postgresql/$DATA_VERSION/bin -d $OLDDATA -D $NEWDATA -U $POSTGRES_USER
-    cp $OLDDATA/pg_hba.conf $NEWDATA/
-    mv $NEWDATA/* $PGDATA
-    rm -rf $OLDDATA
-    rmdir $NEWDATA
-    rm $MIGRATION_MARKER
-    echo "migration complete"
-  fi
-fi
-
-/usr/local/bin/docker-entrypoint.sh postgres

release/2.0.0+v1.5.1

@@ -1,7 +0,0 @@
-If you're upgrading from a pre-release version, there will be a major Postgresql
-version upgrade -- this should happen automatically, but **please take a
-backup**, at least of the `<stack_name>_db` volume, if not all the data volumes,
-before the upgrade. 
-
-If you haven't taken a backup already, it's probably safest to bail using
-Ctrl+C, take the backup, and re-run your `upgrade` / `deploy` command.

release/3.0.0+v2.0.0

@@ -1,8 +0,0 @@
-⚠ WARNING! ⚠
-
-This major version upgrade of Plausible requires running a manual data migration
--- otherwise you'll see all historical data disappear (don't worry, it's
-"probably" still there). 
-
-Take a manual docker volume backup, then see here, and strap in: 
-https://github.com/plausible/analytics/discussions/3132