Update to karrot v17.0.0

- includes plugins

.drone.yml

@@ -0,0 +1,17 @@
+---
+kind: pipeline
+name: generate recipe catalogue
+steps:
+  - name: release a new version
+    image: plugins/downstream
+    settings:
+      server: https://build.coopcloud.tech
+      token:
+        from_secret: drone_abra-bot_token
+      fork: true
+      repositories:
+        - toolshed/auto-recipes-catalogue-json
+
+trigger:
+  event: tag
+

.env.sample

@@ -1,16 +1,62 @@
 TYPE=karrot
 
+# For more information about these options
+# see https://docs.karrot.world/self-host/options
+
 DOMAIN=karrot.example.com
-LETS_ENCRYPT_ENV=production
 
-# account id for maxmind (for GeoIP)
-MAXMIND_ACCOUNT_ID=
-# License key for maxmind 
-MAXMIND_LICENSE_KEY=
-
-# if I set this it fails with:
-# TypeError: SelectorEventLoop required, instead got: <uvloop.Loop running=False closed=False debug=False>
-#LISTEN_CONCURRENCY=4
+COMPOSE_FILE="compose.yml"
 
+SITE_NAME=karrot dev
+SITE_LOGO=https://user-images.githubusercontent.com/31616/36565633-517373a4-1821-11e8-9948-5bf6887c667e.png
 
 SECRET_DB_PASSWORD_VERSION=v1
+SECRET_SECRET_KEY_VERSION=v1
+SECRET_SMTP_PASSWORD_VERSION=v1
+
+# account id for maxmind (for GeoIP)
+# uncomment if using maxmind account
+# make sure to add the maxmind_license_key secret too
+#COMPOSE_FILE="$COMPOSE_FILE:compose.geoip.yml"
+#MAXMIND_ACCOUNT_ID=
+SECRET_MAXMIND_LICENSE_KEY_VERSION=v1
+
+FILE_UPLOAD_MAX_SIZE=10m
+
+# postal,smtp,console
+EMAIL_BACKEND=console
+
+# only set these when using EMAIL_BACKEND=smtp
+# SMTP USER and EMAIL_FROM are usually the same
+#EMAIL_FROM=
+#SMTP_USER=
+#SMTP_HOST=
+#SMTP_USE_SSL=
+#SMTP_USE_TLS=true
+#SMTP_PORT=587
+
+# only set these when using EMAIL_BACKEND=postal
+#POSTAL_API_KEY=
+#POSTAL_API_URL=
+#POSTAL_WEBHOOK_KEY=
+
+# only if you have configured incoming emails
+#EMAIL_REPLY_DOMAIN=
+
+# For web push set this, and the vapid private key secret
+# You need to generate a valid vapid keypair
+# You can generate one by running:
+# docker run --rm codeberg.org/karrot/generate-vapid-keypair
+#VAPID_PUBLIC_KEY=
+#VAPID_ADMIN_EMAIL=
+#SECRET_VAPID_PRIVATE_KEY_VERSION=v1
+
+# for video calls
+#MEET_LIVEKIT_ENDPOINT=
+#MEET_LIVEKIT_API_KEY=
+#SECRET_LIVEKIT_API_SECRET_VERSION=v1
+
+# You probably don't need to touch these
+SITE_URL=https://${DOMAIN}
+LETS_ENCRYPT_ENV=production
+CSRF_TRUSTED_ORIGINS=${SITE_URL}

README.md

@@ -0,0 +1,31 @@
+# karrot
+
+Karrot is a free and open-source tool for grassroots initiatives and groups of people that want to coordinate face-to-face activities on a local, autonomous and voluntary basis.
+
+<!-- metadata -->
+
+* **Category**: Utilities
+* **Status**: 3, stable
+* **Image**: [`karrot-frontend`](https://codeberg.org/karrot/-/packages/container/karrot-backend)/[`karrot-frontend`](https://codeberg.org/karrot/-/packages/container/karrot-backend),4,upstream
+* **Healthcheck**: Yes
+* **Backups**: Yes
+* **Email**: Yes
+* **Tests**: No
+* **SSO**:  No
+
+<!-- endmetadata -->
+
+## Basic usage
+
+1. Set up Docker Swarm and [`abra`]
+2. `abra app new karrot`
+3. `abra app config <karrot app name>`
+4. `abra app deploy <karrot app name>`
+
+See [Karrot Self-hosting docs](https://docs.karrot.world/self-host/coop-cloud/getting-started) for more information.
+
+## Configuration options
+
+`MAXMIND_ACCOUNT_ID` and `MAXMIND_ACCOUNT_KEY` are API credentials from maxmind.com. You need an account there to get GeoIP data for Karrot.
+
+[`abra`]: https://git.coopcloud.tech/coop-cloud/abra

abra.sh

@@ -1,2 +1,17 @@
-export NGINX_CONFIG_VERSION=v1
-export POSTGRES_ENTRYPOINT_VERSION=v1
+fix-permissions() {
+  if [ "$(whoami)" != "root" ]; then
+    echo "error: you must be root to fix permissions"
+    echo "Try adding '--user root'"
+    exit 1
+  fi
+  
+  echo "Fixing permissions"
+  
+  echo "Making karrot the owner of uploads"
+  chown -R karrot:karrot /app/uploads
+
+  echo "Making karrot the owner of plugins"
+  chown -R karrot:karrot /app/plugins
+
+  echo "Done"
+}

backend/Dockerfile

@@ -1,13 +0,0 @@
-FROM python:3.7-buster
-
-WORKDIR /app
-
-RUN apt-get update && \
-    apt-get install -y gdal-bin
-
-COPY ./karrot-backend.pyz /app/karrot-backend.pyz
-
-RUN sed -i -e's/ main/ main contrib non-free/g' /etc/apt/sources.list && \
-    apt-get update && \
-    apt-get install -y libmaxminddb0 libmaxminddb-dev geoipupdate
-

compose.geoip.yml

@@ -0,0 +1,30 @@
+version: "3.8"
+
+services:
+  app:
+    volumes:
+      - "geoip_data:/var/lib/GeoIP"
+
+  worker:
+    volumes:
+      - "geoip_data:/var/lib/GeoIP"
+
+  geoip:
+    image: "ghcr.io/maxmind/geoipupdate:v6"
+    volumes:
+      - "geoip_data:/usr/share/GeoIP"
+    secrets:
+      - maxmind_license_key
+    environment:
+      - "GEOIPUPDATE_EDITION_IDS=GeoLite2-City GeoLite2-Country"
+      - "GEOIPUPDATE_ACCOUNT_ID=${MAXMIND_ACCOUNT_ID:-}"
+      - "GEOIPUPDATE_LICENSE_KEY_FILE=/run/secrets/maxmind_license_key"
+      - "GEOIPUPDATE_FREQUENCY=72"
+
+secrets:
+  maxmind_license_key:
+    external: true
+    name: ${STACK_NAME}_maxmind_license_key_${SECRET_MAXMIND_LICENSE_KEY_VERSION}
+
+volumes:
+  geoip_data:

compose.yml

@@ -2,24 +2,27 @@ version: "3.8"
 
 services:
   web:
-    image: "vlafvlaf/karrot_frontend:0.0.3"
-    configs:
-      - source: nginx_config
-        target: /etc/nginx/conf.d/default.conf
+    image: "codeberg.org/karrot/karrot-frontend:v17.0.0"
     depends_on:
       - app
     environment:
       - DOMAIN
+      - FILE_UPLOAD_MAX_SIZE
+      - FILE_UPLOAD_DIR=/app/uploads/
+      - CSP_CONNECT_SRC=${CSP_CONNECT_SRC:-}
+      - LISTEN=80
+      - BACKEND=app:8000
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost/"]
-      interval: 30s
+      interval: 15s
       timeout: 3s
-      retries: 30
+      retries: 2
+      start_period: 15s
     networks:
       - internal
       - proxy
     volumes:
-      - "app_data:/app/uploads"
+      - "app_data:/app/uploads/"
     deploy:
       update_config:
         failure_action: rollback
@@ -30,79 +33,135 @@ services:
         - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})"
         - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure"
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
-        - "coop-cloud.${STACK_NAME}.version=0.0.1+testing"
 
   app:
-    image: "vlafvlaf/karrot_backend:test"
+    image: "codeberg.org/karrot/karrot-backend:v17.0.0"
     networks:
       - internal
     depends_on:
       - db
       - redis
+    secrets:
+      - db_password
+      - secret_key
+      - smtp_password
+      - vapid_private_key
+      - livekit_api_secret
     volumes:
-      - "shiv_data:/root/.shiv"
-      - "geoip_data:/var/lib/GeoIP"
-      - "app_data:/app/uploads"
+      - "app_data:/app/uploads/"
+      - "plugins_data:/app/plugins/"
     environment:
-      - MAXMIND_ACCOUNT_ID
-      - MAXMIND_LICENSE_KEY
-      - SITE_URL=http://localhost:8000
+      - ADMIN_EMAILS
+      - CSRF_TRUSTED_ORIGINS
+      - DATABASE_CONN_MAX_AGE
+      - DATABASE_HOST=db
+      - DATABASE_NAME=karrot
+      - DATABASE_PASSWORD_FILE=/run/secrets/db_password
+      - DATABASE_PORT=5432
+      - DATABASE_USER=karrot
+      - EMAIL_BACKEND
+      - EMAIL_FROM
+      - EMAIL_REPLY_DOMAIN
+      - FILE_UPLOAD_DIR=/app/uploads/
+      - FILE_UPLOAD_USE_ACCEL_REDIRECT=true
+      - FILE_UPLOAD_MAX_SIZE
+      - FORUM_BANNER_TOPIC_ID
+      - FORUM_DISCUSSIONS_FEED
       - LISTEN_HOST=0.0.0.0
       - LISTEN_SERVER=uvicorn
-      - SECRET_KEY=foobar
-      - DATABASE_HOST=db
-      - DATABASE_PORT=5432
-      - DATABASE_NAME=karrot
-      - DATABASE_USER=karrot
-      - DATABASE_PASSWORD=karrot
+      - MODE=prod
+      - POSTAL_API_KEY
+      - POSTAL_API_URL
+      - POSTAL_WEBHOOK_KEY
+      - PROXY_DISCOURSE_URL
+      - PLUGIN_DIR=/app/plugins/
+      - REDIS_DB=0
       - REDIS_HOST=redis
       - REDIS_PORT=6379
-      - REDIS_DB=0
+      - SECRET_KEY_FILE=/run/secrets/secret_key
+      - SITE_LOGO
+      - SITE_NAME
+      - SITE_URL
+      - SMTP_HOST
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
+      - SMTP_PORT
+      - SMTP_USE_SSL
+      - SMTP_USE_TLS
+      - SMTP_USER
+      - VAPID_ADMIN_EMAIL
+      - VAPID_PUBLIC_KEY
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
+      - MEET_LIVEKIT_ENDPOINT
+      - MEET_LIVEKIT_API_KEY
+      - MEET_LIVEKIT_API_SECRET_FILE=/run/secrets/livekit_api_secret
+      - MIGRATE=yes
+    command: server
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8000/api/"]
-      interval: 30s
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/settings/"]
+      interval: 10s
       timeout: 3s
-      retries: 90
-    command: >
-      sh -c "
-      echo 'EditionIDs GeoLite2-City GeoLite2-Country' > /etc/GeoIP.conf &&
-      echo AccountID $${MAXMIND_ACCOUNT_ID} >> /etc/GeoIP.conf &&
-      echo LicenseKey $${MAXMIND_LICENSE_KEY} >> /etc/GeoIP.conf &&
-      echo 'Updating geoip data, first time could be slow...' &&
-      geoipupdate &&
-      pip install tzdata &&
-      python karrot-backend.pyz migrate &&
-      python karrot-backend.pyz server
-      "
+      retries: 3 
+      start_period: 45s
+    deploy:
+      labels:
+        - "coop-cloud.${STACK_NAME}.version=3.0.0+17.0.0"
+        - "backupbot.backup=true"
+        - "backupbot.backup.path=/app/uploads"
+
   worker:
-    image: "vlafvlaf/karrot_backend:test"
+    image: "codeberg.org/karrot/karrot-backend:v17.0.0"
     depends_on:
-      # shiv + geoip data gets loaded on the first run of the app
-      # so to ensure it's available in the worker too, we need to wait
       - app
-    volumes:
-      - "shiv_data:/root/.shiv"
-      - "geoip_data:/var/lib/GeoIP"
     networks:
       - internal
+    secrets:
+      - db_password
+      - secret_key
+      - smtp_password
+      - vapid_private_key
+      - livekit_api_secret
+    volumes:
+      - "app_data:/app/uploads/"
+      - "plugins_data:/app/plugins/"
     environment:
-      - SITE_URL=http://localhost:8000
+      - ADMIN_EMAILS
+      - DATABASE_CONN_MAX_AGE
+      - DATABASE_HOST=db
+      - DATABASE_NAME=karrot
+      - DATABASE_PASSWORD_FILE=/run/secrets/db_password
+      - DATABASE_PORT=5432
+      - DATABASE_USER=karrot
+      - EMAIL_BACKEND
+      - EMAIL_FROM
+      - EMAIL_REPLY_DOMAIN
       - LISTEN_HOST=0.0.0.0
       - LISTEN_SERVER=uvicorn
-      - SECRET_KEY=foobar
-      - DATABASE_HOST=db
-      - DATABASE_PORT=5432
-      - DATABASE_NAME=karrot
-      - DATABASE_USER=karrot
-      - DATABASE_PASSWORD=karrot
+      - MODE=prod
+      - POSTAL_API_KEY
+      - POSTAL_API_URL
+      - POSTAL_WEBHOOK_KEY
+      - PLUGIN_DIR=/app/plugins/
+      - REDIS_DB=0
       - REDIS_HOST=redis
       - REDIS_PORT=6379
-      - REDIS_DB=0
-    command: 
-      sh -c "
-      pip install tzdata &&
-      python karrot-backend.pyz worker"
-  
+      - SECRET_KEY_FILE=/run/secrets/secret_key
+      - SITE_LOGO
+      - SITE_NAME
+      - SITE_URL
+      - SMTP_HOST
+      - SMTP_PASSWORD_FILE=/run/secrets/smtp_password
+      - SMTP_PORT
+      - SMTP_USE_SSL
+      - SMTP_USE_TLS
+      - SMTP_USER
+      - VAPID_ADMIN_EMAIL
+      - VAPID_PUBLIC_KEY
+      - VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
+      - MEET_LIVEKIT_ENDPOINT
+      - MEET_LIVEKIT_API_KEY
+      - MEET_LIVEKIT_API_SECRET_FILE=/run/secrets/livekit_api_secret
+    command: worker
+
   redis:
     image: "redis:6-alpine"
     command: ["redis-server", "--appendonly", "yes"]
@@ -123,40 +182,46 @@ services:
       interval: 10s
       timeout: 3s
       retries: 30
-    configs:
-      - source: postgres_extensions
-        target: /docker-entrypoint-initdb.d/extensions.sql
-        mode: 0555
+    secrets:
+      - db_password
     volumes:
       - "postgres_data:/var/lib/postgresql/data"
     networks:
       - internal
     environment:
-      - POSTGRES_PASSWORD=karrot
+      - POSTGRES_PASSWORD_FILE=/run/secrets/db_password
       - POSTGRES_USER=karrot
       - POSTGRES_DB=karrot
- 
+    deploy:
+      labels:
+        backupbot.backup: "true"
+        backupbot.backup.pre-hook: "PGPASSWORD=$$(cat $${POSTGRES_PASSWORD_FILE}) pg_dump -U $${POSTGRES_USER} $${POSTGRES_DB} > /var/lib/postgresql/data/postgres-backup.sql"
+        backupbot.backup.post-hook: "rm -rf /var/lib/postgresql/data/postgres-backup.sql"
+        backupbot.backup.path: "/var/lib/postgresql/data/"
 
 secrets:
   db_password:
     external: true
     name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION}
-
-configs:
-  nginx_config:
-    name: ${STACK_NAME}_nginx_config_${NGINX_CONFIG_VERSION}
-    file: nginx.conf.tmpl
-    template_driver: golang
-  postgres_extensions:
-    name: ${STACK_NAME}_postgres_extensions_${POSTGRES_ENTRYPOINT_VERSION}
-    file: pg_extensions.sql
+  secret_key:
+    external: true
+    name: ${STACK_NAME}_secret_key_${SECRET_SECRET_KEY_VERSION}
+  smtp_password:
+    external: true
+    name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION}
+  vapid_private_key:
+    external: true
+    name: ${STACK_NAME}_vapid_private_key_${SECRET_VAPID_PRIVATE_KEY_VERSION}
+  livekit_api_secret:
+    external: true
+    name: ${STACK_NAME}_livekit_api_secret_${SECRET_LIVEKIT_API_SECRET_VERSION}
 
 volumes:
-  shiv_data:
-  geoip_data:
   app_data:
+  plugins_data:
   postgres_data:
   redis_data:
+
 networks:
   proxy:
     external: true

frontend/Dockerfile

@@ -1,4 +0,0 @@
-FROM nginx:1.20.1-alpine
-
-RUN curl https://download.karrot.world/karrot-frontend-production.zip -o karrot-frontend.zip && \
-    unzip -o karrot-frontend.zip -d /usr/share/nginx/html

frontend/nginx.conf

@@ -1,64 +0,0 @@
-map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-}
-
-server { 
-    listen 80;
-    server_name web;
-
-    root /usr/share/nginx/html;
-
-    location / {
-        try_files $uri /index.html;
-        if_modified_since off;
-        expires off;
-        etag off;
-        # TODO: csp headers
-    }
-
-    location /css {
-      expires max;
-    }
-
-    location /js {
-      expires max;
-    }
-
-    location /img {
-      expires max;
-    }
-
-    location /fonts {
-      expires max;
-    }
-
-    # /app/uploads
-
-    location /media/ {
-        alias /app/uploads/;
-        expires max;
-    }
-    
-    location /community_proxy/ {
-        proxy_pass https://community.foodsaving.world/;
-    }
-
-    location ^\/(api(\-auth)?|docs|silk)\/ {
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        # this port is whatever port 80 is mapped to outside the container
-        proxy_set_header Host $host:8080;
-        proxy_pass http://app:8000;
-        proxy_redirect off;
-        proxy_http_version 1.1;
-        proxy_set_header Upgrade $http_upgrade;
-        proxy_set_header Connection $connection_upgrade;
-        proxy_set_header Sec-WebSocket-Extensions $http_sec_websocket_extensions;
-        proxy_set_header Sec-WebSocket-Key $http_sec_websocket_key;
-        proxy_set_header Sec-WebSocket-Protocol $http_sec_websocket_protocol;
-        proxy_set_header Sec-WebSocket-Version $http_sec_websocket_version;
-    }
-
-}

nginx.conf.tmpl

@@ -3,6 +3,8 @@ map $http_upgrade $connection_upgrade {
     '' close;
 }
 
+client_max_body_size {{ env "FILE_UPLOAD_MAX_SIZE" }};
+
 server { 
     listen 80;
     server_name {{ env "DOMAIN" }};
@@ -39,9 +41,9 @@ server {
         alias /app/uploads/;
         expires max;
     }
-    
+
     location /community_proxy/ {
-        proxy_pass https://community.foodsaving.world/;
+        proxy_pass https://community.karrot.world/;
     }
     
     location ~ ^\/(api(\-auth)?|docs|silk)\/ {
@@ -50,7 +52,12 @@ server {
         proxy_set_header X-Forwarded-Proto $scheme;
         # this port is whatever port 80 is mapped to outside the container
         proxy_set_header Host $host:80;
-        proxy_pass http://app:8000;
+
+        # resolver + backend as a variable means can run when backend is not up yet
+        resolver 127.0.0.11 valid=3s;
+        set $backend app:8000;
+        proxy_pass http://$backend$request_uri;
+
         proxy_redirect off;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;

release/1.0.0+14.0.1

@@ -0,0 +1,32 @@
+Major upgrade because this switches to new set of docker images with new python version.
+
+Full release info available here: https://codeberg.org/karrot/karrot/releases/tag/v14.0.1
+
+## Fix to uploaded file permissions
+
+We now run the container as non-root user which means the file permissions need updating.
+
+After you deployment you can fix that by running:
+
+```
+abra app cmd --user root <domain> app fix-permissions
+```
+
+(Note: we need `--user root` there, as we need to be `root` in the container to change the permissions)
+
+## geoip changes
+
+Now the geoip update server is run using an additional compose file config, so if you are using geoip with a maxmind account, modify your config to include:
+
+```
+COMPOSE_FILE="compose.yml"
+COMPOSE_FILE="$COMPOSE_FILE:compose.geoip.yml"
+MAXMIND_ACCOUNT_ID=youraccountid
+SECRET_MAXMIND_LICENSE_KEY_VERSION=v1
+```
+
+And ensure you have the `maxmind_license_key` secret set, which you can do with:
+
+```
+abra app secret insert <domain> maxmind_license_key v1 <key>
+```