remove unused env var

Reviewed-on: coop-cloud/bonfire#12

.env.sample

@@ -6,8 +6,15 @@ APP_VERSION=latest
 
 # choose what extra services you want to run
 COMPOSE_FILE="compose.yml:compose.meilisearch.yml"
-# Enable this one for Postgres tuning (recommended, but only once your instance already is deployed and working):
-# COMPOSE_FILE="compose.yml:postgres/compose.postgres.tune.yml:compose.meilisearch.yml"
+
+# To use Sonic instead of Meilisearch as the search backend:
+# COMPOSE_FILE="compose.yml:compose.sonic.yml"
+
+# Enable `compose.mail.yml` to set secrets for sending emails (remember to also include your chosen search backend's compose file):
+# COMPOSE_FILE="compose.yml:compose.mail.yml"
+
+# Enable `compose.postgres.tune.yml` for Postgres tuning (can only be enabled *after* your instance already is deployed and working)  (and remember to also include your chosen search backend and mailer's compose files)
+# COMPOSE_FILE="compose.yml:compose.postgres.tune.yml"
 
 APP_VERSION_FLAVOUR=${APP_VERSION}-${FLAVOUR}
 # Different flavours/forks or architectures may require different builds of bonfire:
@@ -29,6 +36,10 @@ DB_DOCKER_IMAGE=ghcr.io/baosystems/postgis:${DB_DOCKER_VERSION}
 # TODO: maybe to use for Upgrading to a new Postgres version? (NOTE: does not work with postgis data)
 # DB_DOCKER_IMAGE=pgautoupgrade/pgautoupgrade:16-alpine
 
+## BASIC_AUTH
+## Use 'echo $(htpasswd -nB username)' to generate the secret and store it in secret 'usersfile', multiple user/hashedpassword combinations can be added as comma separated list
+#COMPOSE_FILE="$COMPOSE_FILE:compose.basicauth.yml"
+#SECRET_USERSFILE_VERSION=v1
 
 # enter your instance's domain name
 DOMAIN=bonfire.example.com
@@ -39,8 +50,6 @@ DOMAIN=bonfire.example.com
 # enable abra backups
 ENABLE_BACKUPS=true
 
-# what service to use for sending out emails (eg. smtp, mailgun, none) NOTE: you should also set the corresponding keys in secrets.env
-MAIL_BACKEND=none
 
 # uncomment in order to NOT automatically change the database schema when you upgrade the app
 # DISABLE_DB_AUTOMIGRATION=true
@@ -64,10 +73,16 @@ LOG_LEVEL=info
 # and do not check your env file into any public git repo 
 # change ALL the values:
 
+# what service to use for sending out emails (eg. smtp, mailgun, none) NOTE: you should also set the corresponding keys below for the relevant service you choose, and uncomment the COMPOSE_FILE line for the relevant service if needed
+MAIL_BACKEND=none
+
+#COMPOSE_FILE="$COMPOSE_FILE:compose.mail.yml"
+#SECRET_MAIL_PASSWORD_VERSION=v1
 
 # signup to an email service and edit with relevant info, see: https://docs.bonfirenetworks.org/Bonfire.Mailer.html
 # MAIL_DOMAIN=mgo.example.com
 # MAIL_KEY=xyz
+# NOTE: please add `compose.mail.yml` to COMPOSE_FILE above and use the abra secret instead of env for secrets ^
 # MAIL_FROM=admin@example.com
 # MAIL_PROJECT_ID=
 # MAIL_PRIVATE_KEY=
@@ -76,7 +91,6 @@ LOG_LEVEL=info
 # MAIL_SESSION_TOKEN=
 # MAIL_SERVER=
 # MAIL_USER=
-# MAIL_PASSWORD=
 # MAIL_PORT=
 # MAIL_SSL=true
 # MAIL_TLS=if_available
@@ -88,6 +102,7 @@ LOG_LEVEL=info
 # UPLOADS_S3_BUCKET=
 # UPLOADS_S3_ACCESS_KEY_ID=
 # UPLOADS_S3_SECRET_ACCESS_KEY=
+# TODO: please use the abra secret instead of env for secrets ^
 # UPLOADS_S3_REGION=fr-par
 # UPLOADS_S3_HOST=s3.fr-par.scw.cloud
 # UPLOADS_S3_SCHEME=https://
@@ -96,6 +111,12 @@ LOG_LEVEL=info
 # AWS_ROLE_ARN=
 # AWS_WEB_IDENTITY_TOKEN_FILE=
 
+# GHOST Integration
+#GHOST_URL=https://example.ghost.io
+#GHOST_CONTENT_API_KEY=
+#GHOST_ADMIN_API_KEY=
+#IFRAME_ALLOWED_ORIGINS=https://example.com
+
 # Enable using Bonfire as an SSO provider for external apps to sign in with?
 # ENABLE_SSO_PROVIDER=false
 OAUTH_ISSUER=https://${DOMAIN}
@@ -138,6 +159,7 @@ OAUTH_ISSUER=https://${DOMAIN}
 WITH_LV_NATIVE=0
 WITH_IMAGE_VIX=1
 WITH_AI=0
+LIVE_DASHBOARD_LOGGER=false
 
 # error reporting:
 # SENTRY_DSN=
@@ -146,9 +168,10 @@ WITH_AI=0
 # these secrets will be autogenerated/managed by abra and docker"
 SECRET_POSTGRES_PASSWORD_VERSION=v1
 SECRET_MEILI_MASTER_KEY_VERSION=v1
+SECRET_SONIC_PASSWORD_VERSION=v1
 SECRET_SEEDS_PW_VERSION=v1
 SECRET_LIVEBOOK_PASSWORD_VERSION=v1
-
+SECRET_MAIL_KEY_VERSION=v1
 SECRET_SECRET_KEY_BASE_VERSION=v1 # length=128
 SECRET_SIGNING_SALT_VERSION=v1 # length=128
 SECRET_ENCRYPTION_SALT_VERSION=v1 # length=128

README.md

@@ -25,6 +25,12 @@ A [coop-cloud](https://coopcloud.tech) recipe for deploying [Bonfire](https://bo
 ## Upgrades 
 `abra app deploy --force your-server.domain.name`
 
+NOTE: we recommend switching to the new `sonic` search backend (instead of the deprecated `meilisearch`):
+1. comment the `COMPOSE_FILE` line that contains `compose.meilisearch.yml` in the `.env` file for your bonfire instancem and replace with a line like `COMPOSE_FILE="compose.yml:compose.sonic.yml"`
+2. add `SONIC_PASSWORD=a-super-secret-password` to the same file (make sure to change the password after pasting!)
+3. redeploy with `abra app deploy --force your-server.domain.name`
+
+
 [`abra`]: https://docs.coopcloud.tech/abra/
 [`coop-cloud/traefik`]: https://git.coopcloud.tech/coop-cloud/traefik
 

abra.sh

@@ -1,32 +1,5 @@
-export APP_ENTRYPOINT_VERSION=v1
-export PG_BACKUP_VERSION=v4
+export APP_ENTRYPOINT_VERSION=v3
+export PG_BACKUP_VERSION=v6
 export MEILI_BACKUP_VERSION=v4
-export NGINX_CONF_VERSION=eaaa
-export VARNISH_CONFIG_VERSION=baac
-
-# Bonfire.Common.Cache.HTTPPurge.bust_http_urls("/post_comments/01KHRMPNKXH659SPB55EY7R1K9")
-
-# varnishlog -g raw
-
-# defmodule Bonfire.UI.Common.Cache.HTTPPurge.Varnish do
-#   @moduledoc """
-#   HTTP cache purge adapter for Varnish.
-
-#   Configured via `VARNISH_URL` env var (default: `http://localhost:80`),
-#   which is resolved at startup in `Bonfire.Common.RuntimeConfig`.
-
-#   Tag-based purging (`bust_tags/1`) requires the `xkey` vmod and is not
-#   implemented here — extend if your Varnish setup supports it.
-#   """
-
-#   @behaviour Bonfire.Common.Cache.HTTPPurge
-#   use Untangle
-
-#   def bust_urls(urls) when is_list(urls) do
-#     Enum.each(urls, &purge_one/1)
-#     :ok
-#   end
-
-#   def bust_tags(_tags), do: :ok
-
-#   defp purge_one(path) do
+export SONIC_CFG_VERSION=v1
+export SONIC_ENTRYPOINT_VERSION=v1

compose.basicauth.yml

@@ -0,0 +1,14 @@
+version: "3.8"
+services:
+  app:
+    secrets:
+      - usersfile
+    deploy:
+      labels:
+        - "traefik.http.middlewares.${STACK_NAME}_basicauth.basicauth.usersFile=/run/secrets/usersfile"
+        - "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}_basicauth"
+
+secrets:
+  usersfile:
+    name: ${STACK_NAME}_usersfile_${SECRET_USERSFILE_VERSION}
+    external: true

compose.mail.yml

@@ -0,0 +1,15 @@
+version: "3.8"
+
+services:
+  app:
+    secrets:
+      - mail_password
+      - mail_key
+
+secrets:
+  mail_password:
+    external: true
+    name: ${STACK_NAME}_mail_password_${SECRET_MAIL_PASSWORD_VERSION:-v1}
+  mail_key:
+    external: true
+    name: ${STACK_NAME}_mail_key_${SECRET_MAIL_KEY_VERSION:-v1}

compose.meilisearch.yml

@@ -40,5 +40,5 @@ volumes:
    
 configs:
   meili_backup:
-    name: ${STACK_NAME}_meili_backup_${MEILI_BACKUP_VERSION}
+    name: ${STACK_NAME}_meili_backup_${MEILI_BACKUP_VERSION:-v4}
     file: meili_backup.sh

compose.sonic.yml

@@ -0,0 +1,49 @@
+---
+version: "3.8"
+
+services:
+  app:
+    depends_on:
+      - search
+    environment:
+      - SEARCH_ADAPTER=sonic
+      - SONIC_HOST=${STACK_NAME}_search
+      - SONIC_PORT=1491
+      - SONIC_PASSWORD
+      # - SONIC_PASSWORD_FILE=/run/secrets/sonic_password
+    # secrets:
+    #   - sonic_password
+      
+  search:
+    image: valeriansaliou/sonic:v1.5.1
+    # secrets:
+    #   - sonic_password
+    volumes:
+      - "sonic-data:/var/lib/sonic/store"
+    networks:
+      - internal
+    # NOTE: latest versions of Sonic (v1.5.1+) don't have a shell, so we can't have a custom entrypoint script that does pre-startup configuration, so we just need to store the PW in env for now
+    # entrypoint: ["/docker-entrypoint.sh"]
+    configs:
+      - source: sonic_cfg
+        target: /etc/sonic.cfg
+        mode: 0444
+      # - source: sonic_entrypoint
+      #   target: /docker-entrypoint.sh
+      #   mode: 0555
+        
+volumes:
+  sonic-data:
+   
+configs:
+  sonic_cfg:
+    name: ${STACK_NAME}_sonic_cfg_${SONIC_CFG_VERSION:-v1}
+    file: sonic.cfg
+  # sonic_entrypoint:
+  #   name: ${STACK_NAME}_sonic_entrypoint_${SONIC_ENTRYPOINT_VERSION:-v1}
+  #   file: sonic_entrypoint.sh
+
+# secrets:
+#   sonic_password:
+#     external: true
+#     name: ${STACK_NAME}_sonic_password_${SECRET_SONIC_PASSWORD_VERSION:-v1}

compose.yml

@@ -2,35 +2,6 @@
 version: "3.8"
 
 services:
-  varnish:
-    image: varnish:stable
-    environment:
-      - STACK_NAME
-      - VARNISH_SIZE=2G
-    networks:
-    - proxy
-    - internal
-    command: "-p default_keep=300 -p thread_pool_min=50 -p thread_pool_max=1000 -p thread_pools=2 -p listen_depth=2048 -p http_resp_hdr_len=65536"
-    configs:
-     - source: varnish_config
-       target: /etc/varnish/default.vcl
-
-    ulimits:
-      nofile:
-        soft: 65536
-        hard: 65536
-
-    deploy:
-      restart_policy:
-        condition: on-failure
-      labels:
-       - "backupbot.backup=false"
-       - "traefik.enable=true"
-       - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=80"
-       - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
-       - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
-       - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-
   app:
     image: ${APP_DOCKER_IMAGE}
     logging:
@@ -46,8 +17,6 @@ services:
       - POSTGRES_DB=bonfire_db
       - PUBLIC_PORT=443
       - MIX_ENV=prod
-      # - VARNISH_URL=http://${STACK_NAME}_varnish:80
-      - VARNISH_URL=https://${DOMAIN}
       
       - HOSTNAME
       - INSTANCE_DESCRIPTION
@@ -64,6 +33,7 @@ services:
       - WITH_LV_NATIVE
       - WITH_IMAGE_VIX
       - WITH_AI
+      - LIVE_DASHBOARD_LOGGER
       
       - DB_SLOW_QUERY_MS
       - DB_STATEMENT_TIMEOUT
@@ -77,7 +47,6 @@ services:
       - MAIL_PRIVATE_KEY
       - MAIL_BASE_URI
       - MAIL_REGION
-      - MAIL_SESSION_TOKEN
       - MAIL_SERVER
       - MAIL_USER
       - MAIL_PASSWORD
@@ -140,6 +109,11 @@ services:
       - ORCID_CLIENT_ID
       - ORCID_CLIENT_SECRET
 
+      - GHOST_URL
+      - GHOST_CONTENT_API_KEY
+      - GHOST_ADMIN_API_KEY
+      - IFRAME_ALLOWED_ORIGINS
+      
     secrets:
       - postgres_password
       - secret_key_base
@@ -152,6 +126,7 @@ services:
       - upload-data:/opt/app/data/uploads
       # - backup-data:/opt/app/data/backup
     networks:
+      - proxy
       - internal
     entrypoint: ["/docker-entrypoint.sh", "./bin/bonfire", "start"]
     configs:
@@ -165,6 +140,16 @@ services:
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
         #- backupbot.backup.volumes.upload-data: "true"
         #- backupbot.backup.volumes.upload-data.path: "/opt/app/data/uploads"
+        - "traefik.enable=true"
+        - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=4000"
+        - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
+        - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
+        - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
+        #- "traefik.http.routers.${STACK_NAME}.middlewares=error-pages-middleware"
+        #- "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=80"
+        #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect"
+        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true"
+        #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}"
     # healthcheck:
     #   test: ["CMD", "curl", "-f", "http://localhost"]
     #   interval: 30s
@@ -201,7 +186,8 @@ services:
     # -c statement_timeout=1800000
     # -c pg_stat_statements.track=all
     #entrypoint: ['tail', '-f', '/dev/null'] # uncomment when the Postgres DB is corrupted and won't start
-    labels:
+    deploy:
+      labels:
         backupbot.backup: ${ENABLE_BACKUPS:-true}
         # backupbot.backup.volumes.db-data: false
         backupbot.backup.volumes.db-data.path: "backup.sql"
@@ -224,40 +210,32 @@ networks:
 
 configs:
   app_entrypoint:
-    name: ${STACK_NAME}_app_entrypoint_${APP_ENTRYPOINT_VERSION}
+    name: ${STACK_NAME}_app_entrypoint_${APP_ENTRYPOINT_VERSION:-v3}
     file: entrypoint.sh.tmpl
     template_driver: golang
   pg_backup:
-    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION}
+    name: ${STACK_NAME}_pg_backup_${PG_BACKUP_VERSION:-v4}
     file: pg_backup.sh
-  # nginx_conf:
-  #   name: ${STACK_NAME}_nginx_conf_${NGINX_CONF_VERSION}
-  #   file: nginx.conf.tmpl
-  #   template_driver: golang
-  varnish_config:
-    name: ${STACK_NAME}_varnish_config_${VARNISH_CONFIG_VERSION}
-    file: default.vcl.tmpl
-    template_driver: golang
 
 secrets:
   postgres_password:
     external: true
-    name: ${STACK_NAME}_postgres_password_${SECRET_POSTGRES_PASSWORD_VERSION}
+    name: ${STACK_NAME}_postgres_password_${SECRET_POSTGRES_PASSWORD_VERSION:-v1}
   secret_key_base:
     external: true
-    name: ${STACK_NAME}_secret_key_base_${SECRET_SECRET_KEY_BASE_VERSION}
+    name: ${STACK_NAME}_secret_key_base_${SECRET_SECRET_KEY_BASE_VERSION:-v1}
   signing_salt:
     external: true
-    name: ${STACK_NAME}_signing_salt_${SECRET_SIGNING_SALT_VERSION}
+    name: ${STACK_NAME}_signing_salt_${SECRET_SIGNING_SALT_VERSION:-v1}
   encryption_salt:
     external: true
-    name: ${STACK_NAME}_encryption_salt_${SECRET_ENCRYPTION_SALT_VERSION}
+    name: ${STACK_NAME}_encryption_salt_${SECRET_ENCRYPTION_SALT_VERSION:-v1}
   meili_master_key:
     external: true
-    name: ${STACK_NAME}_meili_master_key_${SECRET_MEILI_MASTER_KEY_VERSION}
+    name: ${STACK_NAME}_meili_master_key_${SECRET_MEILI_MASTER_KEY_VERSION:-v1}
   seeds_pw:
     external: true
-    name: ${STACK_NAME}_seeds_pw_${SECRET_SEEDS_PW_VERSION}
+    name: ${STACK_NAME}_seeds_pw_${SECRET_SEEDS_PW_VERSION:-v1}
   livebook_password:
     external: true
-    name: ${STACK_NAME}_livebook_password_${SECRET_LIVEBOOK_PASSWORD_VERSION}
+    name: ${STACK_NAME}_livebook_password_${SECRET_LIVEBOOK_PASSWORD_VERSION:-v1}

default.vcl.tmpl

@@ -1,23 +0,0 @@
-vcl 4.1;
-
-backend default {
-    .host = "{{ env "STACK_NAME" }}_app";
-    .port = "4000";
-}
-
-sub vcl_recv {
-	if (req.method == "PURGE" || req.method == "purge") {
-        return (purge);
-	}
-
-    if (req.http.upgrade ~ "(?i)websocket") {
-        return (pipe);
-    }
-}
-
-sub vcl_pipe {
-    if (req.http.upgrade) {
-        set bereq.http.upgrade = req.http.upgrade;
-        set bereq.http.connection = req.http.connection;
-    }
-}

entrypoint.sh.tmpl

@@ -9,7 +9,17 @@ export ENCRYPTION_SALT=$(cat /run/secrets/encryption_salt)
 export SEEDS_PW=$(cat /run/secrets/seeds_pw)
 export LIVEBOOK_PASSWORD=$(cat /run/secrets/livebook_password)
 
+# Only read the secret when the MAIL_PASSWORD was not set to remain backwards compatible
+if [ -f /run/secrets/mail_password ] && [ -z "${MAIL_PASSWORD}" ]; then
+    export MAIL_PASSWORD=$(cat /run/secrets/mail_password)
+fi
+
+# Only read the secret when the MAIL_KEY was not set to remain backwards compatible
+if [ -f /run/secrets/mail_key ] && [ -z "${MAIL_KEY}" ]; then
+    export MAIL_KEY=$(cat /run/secrets/mail_key)
+fi
+
 echo "....Secrets have been loaded, now run $@...."
 
 # This will exec the CMD from your Dockerfile
-exec "$@"
+exec "$@"

pg_backup.sh

@@ -6,18 +6,17 @@ BACKUP_PATH="/var/lib/postgresql/data"
 LATEST_BACKUP_FILE="${BACKUP_PATH}/backup.sql"
 
 function backup {
-  FILE_WITH_DATE="${BACKUP_PATH}/backup_$(date +%F).sql"
-  
   if [ -f "$POSTGRES_PASSWORD_FILE" ]; then
     export PGPASSWORD=$(cat "$POSTGRES_PASSWORD_FILE")
   fi
-  
-  echo "Creating backup at ${FILE_WITH_DATE}..."
-  pg_dump -U "${POSTGRES_USER:-postgres}" "${POSTGRES_DB:-postgres}" > "${FILE_WITH_DATE}"
-  
-  echo "Copying to ${LATEST_BACKUP_FILE}..."
-  cp -f "${FILE_WITH_DATE}" "${LATEST_BACKUP_FILE}"
-  
+
+  # Keep a single backup.sql (restic handles versioning); write to a temp file and move
+  # atomically so a failed dump never clobbers the last good backup.
+  echo "Creating backup at ${LATEST_BACKUP_FILE}..."
+  rm -f "${LATEST_BACKUP_FILE}.tmp"
+  pg_dump -U "${POSTGRES_USER:-postgres}" "${POSTGRES_DB:-postgres}" > "${LATEST_BACKUP_FILE}.tmp"
+  mv -f "${LATEST_BACKUP_FILE}.tmp" "${LATEST_BACKUP_FILE}"
+
   echo "Backup done. You will find it at ${LATEST_BACKUP_FILE}"
 }
 

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ]
+}

sonic.cfg

@@ -0,0 +1,43 @@
+# Sonic configuration
+# https://github.com/valeriansaliou/sonic/blob/master/CONFIGURATION.md
+
+[server]
+  log_level = "error"
+
+[channel]
+  inet = "0.0.0.0:1491"
+  tcp_timeout = 300
+
+  [channel.search]
+    query_limit_default = 10
+    query_limit_maximum = 200
+    suggest_limit_default = 5
+    suggest_limit_maximum = 20
+
+[store.kv]
+  path = "/var/lib/sonic/store/kv/"
+  retain_word_objects = 1000
+
+  [store.kv.pool]
+    inactive_after = 1800
+
+  [store.kv.database]
+    flush_after = 900
+    compress = true
+    parallelism = 2
+    max_files = 100
+    max_compactions = 1
+    max_flushes = 1
+    write_buffer = 16384
+    write_ahead_log = true
+
+[store.fst]
+  path = "/var/lib/sonic/store/fst/"
+
+  [store.fst.pool]
+    inactive_after = 300
+
+  [store.fst.graph]
+    consolidate_after = 180
+    max_size = 2048
+    max_words = 250000

sonic_entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+export SONIC_CHANNEL__AUTH_PASSWORD="$(cat /run/secrets/sonic_password)"
+
+exec sonic -c /etc/sonic.cfg