Use livekit_api_secret

- correct URL for community proxy
- allow nginx to recheck if backend was done before
- new karrot beta images

.drone.yml

@@ -0,0 +1,17 @@
+---
+kind: pipeline
+name: generate recipe catalogue
+steps:
+  - name: release a new version
+    image: plugins/downstream
+    settings:
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - coop-cloud/auto-recipes-catalogue-json
+
+trigger:
+  event: tag
+

.env.sample

@@ -1,16 +1,51 @@
 TYPE=karrot
 
 DOMAIN=karrot.example.com
-LETS_ENCRYPT_ENV=production
-
-# account id for maxmind (for GeoIP)
-MAXMIND_ACCOUNT_ID=
-# License key for maxmind 
-MAXMIND_LICENSE_KEY=
-
-# if I set this it fails with:
-# TypeError: SelectorEventLoop required, instead got: <uvloop.Loop running=False closed=False debug=False>
-#LISTEN_CONCURRENCY=4
-
 
 SECRET_DB_PASSWORD_VERSION=v1
+SECRET_SECRET_KEY_VERSION=v1
+SECRET_SMTP_PASSWORD_VERSION=v1
+
+# account id for maxmind (for GeoIP)
+#MAXMIND_ACCOUNT_ID=
+SECRET_MAXMIND_LICENSE_KEY_VERSION=v1
+
+SITE_NAME=karrot dev
+SITE_LOGO=https://user-images.githubusercontent.com/31616/36565633-517373a4-1821-11e8-9948-5bf6887c667e.png
+
+FILE_UPLOAD_MAX_SIZE=10m
+
+# postal,smtp,console
+EMAIL_BACKEND=console
+
+# only set these when using EMAIL_BACKEND=smtp
+# SMTP USER and EMAIL_FROM are usually the same
+#EMAIL_FROM=
+#SMTP_USER=
+#SMTP_HOST=
+#SMTP_USE_SSL=
+#SMTP_USE_TLS=true
+#SMTP_PORT=587
+
+# only set these when using EMAIL_BACKEND=postal
+#POSTAL_API_KEY=
+#POSTAL_API_URL=
+#POSTAL_WEBHOOK_KEY=
+
+# only if you have configured incoming emails
+#EMAIL_REPLY_DOMAIN=
+
+# For web push set this, and the vapid private key secret
+# You need to generate a valid vapid keypair
+#VAPID_PUBLIC_KEY=
+#VAPID_ADMIN_EMAIL=
+#SECRET_VAPID_PRIVATE_KEY_VERSION=v1
+
+# for video calls
+#MEET_LIVEKIT_ENDPOINT=
+#MEET_LIVEKIT_API_KEY=
+#SECRET_LIVEKIT_API_SECRET_VERSION=v1
+
+SITE_URL=https://${DOMAIN}
+LETS_ENCRYPT_ENV=production
+CSRF_TRUSTED_ORIGINS=${SITE_URL}

README.md

@@ -0,0 +1,29 @@
+# karrot
+
+Karrot is a free and open-source tool for grassroots initiatives and groups of people that want to coordinate face-to-face activities on a local, autonomous and voluntary basis.
+
+<!-- metadata -->
+
+* **Category**: Utilities
+* **Status**: 3, stable
+* **Image**: [`karrot-backend`](https://hub.docker.com/r/vlafvlaf/karrot_backend),4,upstream
+* **Healthcheck**: Yes
+* **Backups**: No
+* **Email**: Yes
+* **Tests**: No
+* **SSO**:  No
+
+<!-- endmetadata -->
+
+## Basic usage
+
+1. Set up Docker Swarm and [`abra`]
+2. `abra app new karrot`
+3. `abra app config <karrot app name>`
+4. `abra app deploy <karrot app name>`
+
+## Configuration options
+
+`MAXMIND_ACCOUNT_ID` and `MAXMIND_ACCOUNT_KEY` are API credentials from maxmind.com. You need an account there to get GeoIP data for Karrot.
+
+[`abra`]: https://git.coopcloud.tech/coop-cloud/abra

abra.sh

@@ -1,2 +1,3 @@
-export NGINX_CONFIG_VERSION=v1
-export POSTGRES_ENTRYPOINT_VERSION=v1
+export NGINX_CONFIG_VERSION=v23
+export GEOIP_CONFIG_VERSION=v1
+export ENTRYPOINT_VERSION=v3

compose.yml

@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   web:
-    image: "vlafvlaf/karrot_frontend:0.0.3"
+    image: "ghcr.io/karrot-dev/karrot-docker-images:13.0.0-frontend"
     configs:
       - source: nginx_config
         target: /etc/nginx/conf.d/default.conf
@@ -10,11 +10,13 @@ services:
       - app
     environment:
       - DOMAIN
+      - FILE_UPLOAD_MAX_SIZE
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost/"]
-      interval: 10s
+      interval: 15s
       timeout: 3s
-      retries: 30
+      retries: 2
+      start_period: 15s
     networks:
       - internal
       - proxy
@@ -30,52 +32,89 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=0.0.1+testing"
 
   app:
-    image: "vlafvlaf/karrot_backend:test"
+    image: "ghcr.io/karrot-dev/karrot-docker-images:13.0.0-backend"
     networks:
       - internal
     depends_on:
       - db
       - redis
+    configs:
+      - source: geoip_conf
+        target: /etc/GeoIP.conf
+        mode: 0555
+      - source: entrypoint
+        target: /custom-entrypoint.sh
+        mode: 0555
+    entrypoint: /custom-entrypoint.sh
+    secrets:
+      - db_password
+      - secret_key
+      - maxmind_license_key
+      - smtp_password
+      - vapid_private_key
+      - livekit_api_secret
     volumes:
       - "shiv_data:/root/.shiv"
       - "geoip_data:/var/lib/GeoIP"
       - "app_data:/app/uploads"
     environment:
-      - MAXMIND_ACCOUNT_ID
-      - MAXMIND_LICENSE_KEY
-      - SITE_URL=http://localhost:8000
+      - CSRF_TRUSTED_ORIGINS
+      - DATABASE_CONN_MAX_AGE
+      - DATABASE_HOST=db
+      - DATABASE_NAME=karrot
+      - DATABASE_PASSWORD_FILE=/run/secrets/db_password
+      - DATABASE_PORT=5432
+      - DATABASE_USER=karrot
+      - EMAIL_BACKEND
+      - EMAIL_FROM
+      - EMAIL_REPLY_DOMAIN
+      - FILE_UPLOAD_DIR=/app/uploads
+      - FILE_UPLOAD_MAX_SIZE
+      - FORUM_BANNER_TOPIC_ID
+      - FORUM_DISCUSSIONS_FEED
       - LISTEN_HOST=0.0.0.0
       - LISTEN_SERVER=uvicorn
-      - SECRET_KEY=foobar
-      - DATABASE_HOST=db
-      - DATABASE_PORT=5432
-      - DATABASE_NAME=karrot
-      - DATABASE_USER=karrot
-      - DATABASE_PASSWORD=karrot
+      - MAXMIND_ACCOUNT_ID
+      - MODE=prod
+      - POSTAL_API_KEY
+      - POSTAL_API_URL
+      - POSTAL_WEBHOOK_KEY
+      - PROXY_DISCOURSE_URL
+      - REDIS_DB=0
       - REDIS_HOST=redis
       - REDIS_PORT=6379
-      - REDIS_DB=0
+      - SECRET_KEY_FILE=/run/secrets/secret_key
+      - SITE_LOGO
+      - SITE_NAME
+      - SITE_URL
+      - SMTP_HOST
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
+      - SMTP_PORT
+      - SMTP_USE_SSL
+      - SMTP_USE_TLS
+      - SMTP_USER
+      - VAPID_ADMIN_EMAIL
+      - VAPID_PUBLIC_KEY
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
+      - MEET_LIVEKIT_ENDPOINT
+      - MEET_LIVEKIT_API_KEY
+      - MEET_LIVEKIT_API_SECRET_FILE=/run/secrets/livekit_api_secret
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000/api/"]
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/config/"]
       interval: 10s
       timeout: 3s
-      retries: 90
-    command: >
-      sh -c "
-      echo 'EditionIDs GeoLite2-City GeoLite2-Country' > /etc/GeoIP.conf &&
-      echo AccountID $${MAXMIND_ACCOUNT_ID} >> /etc/GeoIP.conf &&
-      echo LicenseKey $${MAXMIND_LICENSE_KEY} >> /etc/GeoIP.conf &&
-      echo 'Updating geoip data, first time could be slow...' &&
-      geoipupdate &&
-      pip install tzdata &&
-      python karrot-backend.pyz migrate &&
-      python karrot-backend.pyz server
-      "
+      retries: 3 
+      start_period: 45s
+    deploy:
+      labels:
+        - "coop-cloud.${STACK_NAME}.version=0.1.8+13.0.0"
+        - "backupbot.backup=true"
+        - "backupbot.backup.path=/app/uploads"
+
   worker:
-    image: "vlafvlaf/karrot_backend:test"
+    image: "ghcr.io/karrot-dev/karrot-docker-images:13.0.0-backend"
     depends_on:
       # shiv + geoip data gets loaded on the first run of the app
       # so to ensure it's available in the worker too, we need to wait
@@ -83,32 +122,62 @@ services:
     volumes:
       - "shiv_data:/root/.shiv"
       - "geoip_data:/var/lib/GeoIP"
+    configs:
+      - source: entrypoint
+        target: /custom-entrypoint.sh
+        mode: 0555
+    entrypoint: /custom-entrypoint.sh
     networks:
       - internal
+    secrets:
+      - db_password
+      - secret_key
+      - smtp_password
+      - vapid_private_key
+      - livekit_api_secret
     environment:
-      - SITE_URL=http://localhost:8000
+      - DATABASE_CONN_MAX_AGE
+      - DATABASE_HOST=db
+      - DATABASE_NAME=karrot
+      - DATABASE_PASSWORD_FILE=/run/secrets/db_password
+      - DATABASE_PORT=5432
+      - DATABASE_USER=karrot
+      - EMAIL_BACKEND
+      - EMAIL_FROM
+      - EMAIL_REPLY_DOMAIN
+      - IS_WORKER=1
       - LISTEN_HOST=0.0.0.0
       - LISTEN_SERVER=uvicorn
-      - SECRET_KEY=foobar
-      - DATABASE_HOST=db
-      - DATABASE_PORT=5432
-      - DATABASE_NAME=karrot
-      - DATABASE_USER=karrot
-      - DATABASE_PASSWORD=karrot
+      - MODE=prod
+      - POSTAL_API_KEY
+      - POSTAL_API_URL
+      - POSTAL_WEBHOOK_KEY
+      - REDIS_DB=0
       - REDIS_HOST=redis
       - REDIS_PORT=6379
-      - REDIS_DB=0
-    command: 
-      sh -c "
-      pip install tzdata &&
-      python karrot-backend.pyz worker"
-  
+      - SECRET_KEY_FILE=/run/secrets/secret_key
+      - SITE_LOGO
+      - SITE_NAME
+      - SITE_URL
+      - SMTP_HOST
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
+      - SMTP_PORT
+      - SMTP_USE_SSL
+      - SMTP_USE_TLS
+      - SMTP_USER
+      - VAPID_ADMIN_EMAIL
+      - VAPID_PUBLIC_KEY
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
+      - MEET_LIVEKIT_ENDPOINT
+      - MEET_LIVEKIT_API_KEY
+      - MEET_LIVEKIT_API_SECRET_FILE=/run/secrets/livekit_api_secret
+
   redis:
     image: "redis:6-alpine"
     command: ["redis-server", "--appendonly", "yes"]
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
-      interval: 1s
+      interval: 10s
       timeout: 3s
       retries: 30
     volumes:
@@ -120,36 +189,59 @@ services:
     image: "postgres:14-alpine"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U karrot"]
-      interval: 1s
+      interval: 10s
       timeout: 3s
       retries: 30
-    configs:
-      - source: postgres_extensions
-        target: /docker-entrypoint-initdb.d/extensions.sql
-        mode: 0555
+    secrets:
+      - db_password
     volumes:
       - "postgres_data:/var/lib/postgresql/data"
     networks:
       - internal
     environment:
-      - POSTGRES_PASSWORD=karrot
+      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
       - POSTGRES_USER=karrot
       - POSTGRES_DB=karrot
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.pre-hook: "PGPASSWORD=$$(cat $${POSTGRES_PASSWORD_FILE}) pg_dump -U $${POSTGRES_USER} $${POSTGRES_DB} > /var/lib/postgresql/data/postgres-backup.sql"
+        backupbot.backup.post-hook: "rm -rf /var/lib/postgresql/data/postgres-backup.sql"
+        backupbot.backup.path: "/var/lib/postgresql/data/"
  
 
-secrets:
-  db_password:
-    external: true
-    name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}
-
 configs:
   nginx_config:
     name: ${STACK_NAME}_nginx_config_${NGINX_CONFIG_VERSION}
     file: nginx.conf.tmpl
     template_driver: golang
-  postgres_extensions:
-    name: ${STACK_NAME}_postgres_extensions_${POSTGRES_ENTRYPOINT_VERSION}
-    file: pg_extensions.sql
+  geoip_conf:
+    name: ${STACK_NAME}_geoip_conf_${GEOIP_CONFIG_VERSION}
+    file: geoip.conf.tmpl
+    template_driver: golang
+  entrypoint:
+    name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION}
+    file: entrypoint.sh
+
+secrets:
+  db_password:
+    external: true
+    name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}
+  secret_key:
+    external: true
+    name: ${STACK_NAME}_secret_key_${SECRET_SECRET_KEY_VERSION}
+  maxmind_license_key:
+    external: true
+    name: ${STACK_NAME}_maxmind_license_key_${SECRET_MAXMIND_LICENSE_KEY_VERSION}
+  smtp_password:
+    external: true
+    name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
+  vapid_private_key:
+    external: true
+    name: ${STACK_NAME}_vapid_private_key_${SECRET_VAPID_PRIVATE_KEY_VERSION}
+  livekit_api_secret:
+    external: true
+    name: ${STACK_NAME}_livekit_api_secret_${SECRET_LIVEKIT_API_SECRET_VERSION}
 
 volumes:
   shiv_data:
@@ -157,6 +249,7 @@ volumes:
   app_data:
   postgres_data:
   redis_data:
+
 networks:
   proxy:
     external: true

entrypoint.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+
+  local val="$def"
+
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+
+  export "$var"="$val"
+  unset "$fileVar"
+}
+
+file_env DATABASE_PASSWORD
+file_env SECRET_KEY
+file_env SMTP_PASSWORD
+file_env VAPID_PRIVATE_KEY
+
+/docker-entrypoint.sh

geoip.conf.tmpl

@@ -0,0 +1,3 @@
+EditionIDs GeoLite2-City GeoLite2-Country
+AccountID {{ env "MAXMIND_ACCOUNT_ID" }}
+LicenseKey {{ secret "maxmind_license_key" }}

nginx.conf.tmpl

@@ -3,6 +3,8 @@ map $http_upgrade $connection_upgrade {
     '' close;
 }
 
+client_max_body_size {{ env "FILE_UPLOAD_MAX_SIZE" }};
+
 server { 
     listen 80;
     server_name {{ env "DOMAIN" }};
@@ -39,9 +41,9 @@ server {
         alias /app/uploads/;
         expires max;
     }
-    
+
     location /community_proxy/ {
-        proxy_pass https://community.foodsaving.world/;
+        proxy_pass https://community.karrot.world/;
     }
     
     location ~ ^\/(api(\-auth)?|docs|silk)\/ {
@@ -50,7 +52,12 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         # this port is whatever port 80 is mapped to outside the container
         proxy_set_header Host $host:80;
-        proxy_pass http://app:8000;
+
+        # resolver + backend as a variable means can run when backend is not up yet
+        resolver 127.0.0.11 valid=3s;
+        set $backend app:8000;
+        proxy_pass http://$backend$request_uri;
+
         proxy_redirect off;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;