From d6745ee952b53986d5da74270fa23e03a38f9aad Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Tue, 20 Oct 2026 21:10:28 +0200 Subject: [PATCH] Upgrade to drive v0.25.0, replace MinIO with RustFS - use rustfs instead of minio and provide migration script - Schedule clean_pending_items/purge_deleted_items with celery beat - Give celery workers the S3 access key secret Following the [C4DT AI Policy](https://git.coopcloud.tech/c4dt/ai-policy/src/branch/main/AI-POLICY.md): #LLM_vibe #LLM_model Claude Opus 5.5 - [x] I take full responsibility for this code - [x] I understand every line of this code - [x] I tested this code thoroughly on my own computer / servers - [x] None of the commit messages were created using LLMs - [x] I will participate in any discussion on my own and will not use an LLM for the discussion --- .env.sample | 36 +++++++++++-- README.md | 34 ++++++++++-- abra-entrypoint.sh | 4 +- abra.sh | 16 ++++-- compose.minio-migration.yml | 47 ++++++++++++++++ compose.yml | 105 ++++++++++++++++++------------------ migrate-from-minio.sh | 96 +++++++++++++++++++++++++++++++++ minio-initialize.sh | 34 ------------ nginx.conf.tmpl | 12 ++--- release/next | 25 +++++++++ s3-initialize.sh | 51 ++++++++++++++++++ 11 files changed, 353 insertions(+), 107 deletions(-) create mode 100644 compose.minio-migration.yml create mode 100755 migrate-from-minio.sh delete mode 100644 minio-initialize.sh create mode 100644 release/next create mode 100755 s3-initialize.sh diff --git a/.env.sample b/.env.sample index ca26604..f3ec87e 100644 --- a/.env.sample +++ b/.env.sample @@ -1,7 +1,8 @@ TYPE=lasuite-drive DOMAIN=lasuite-drive.example.com -MINIO_DOMAIN="minio.${DOMAIN}" +# public domain of the RustFS object storage (S3) +RUSTFS_DOMAIN="rustfs.${DOMAIN}" COLLABORA_DOMAIN="collabora.${DOMAIN}" ONLY_OFFICE_DOMAIN="onlyoffice.${DOMAIN}" @@ -20,10 +21,10 @@ SECRET_DJANGO_SK_VERSION=v1 SECRET_OIDC_RPCS_VERSION=v1 # DJANGO_SUPERUSER_PASSWORD SECRET_DJANGO_SP_VERSION=v1 -# MINIO_ROOT_PASSWORD -SECRET_MINIO_RP_VERSION=v1 -# MINIO_ROOT_USER -SECRET_MINIO_RU_VERSION=v1 +# RUSTFS_SECRET_KEY +SECRET_RUSTFS_RP_VERSION=v1 +# RUSTFS_ACCESS_KEY +SECRET_RUSTFS_RU_VERSION=v1 # POSTGRES_PASSWORD SECRET_POSTGRES_P_VERSION=v1 # DJANGO_HOST_EMAIL_PASSWORD @@ -87,6 +88,31 @@ LOGGING_LEVEL_LOGGERS_APP=INFO # Set to false to disable backup-bot labels (default: true) #ENABLE_BACKUPS=true +############################################################################## +# MAINTENANCE SCHEDULING +############################################################################## +# Celery Beat crontab (m h dom mon dow) followed by the command arguments +# Clean up uploads that never finished (default: daily at 0:30) +#CLEAN_PENDING_ITEMS_SCHEDULE="30 0 * * * --hours=48" +# Permanently purge items deleted from the trash (default: daily at 0:45) +#PURGE_DELETED_ITEMS_SCHEDULE="45 0 * * *" + +############################################################################## +# SDK +############################################################################## +# Enable the SDK picker pages (/sdk) and relay API used by other La Suite apps +#SDK_ENABLED=false + +############################################################################## +# MINIO MIGRATION +############################################################################## +# Only needed once when upgrading from recipe versions using MinIO (<= 0.13.x), +# see README "Migrating from MinIO" +#COMPOSE_FILE="compose.yml:compose.minio-migration.yml" +# versions of the old MinIO secrets, keep them from your previous config +#SECRET_MINIO_RP_VERSION=v1 +#SECRET_MINIO_RU_VERSION=v1 + ############################################################################## # WOPI SCHEDULING ############################################################################## diff --git a/README.md b/README.md index be89bcc..c1b3449 100644 --- a/README.md +++ b/README.md @@ -19,25 +19,53 @@ ### Setting Up Domains -This recipe requires four domains. One domain for drive, and one for minio which also needs its own public endpoint (see [https://github.com/suitenumerique/drive/issues/476](https://github.com/suitenumerique/drive/issues/476)), one domain for collabora, and one domain for onlyoffice. For example (`drive.yourdomain.tld`, `minio.drive.yourdomain.tld`, `collabora.drive.yourdomain.tld`, `onlyoffice.drive.yourdomain.tld`). All of these domains need to have an A record pointing the IP address of your server. +This recipe requires four domains. One domain for drive, one for the S3 object storage ([RustFS](https://rustfs.com)) which also needs its own public endpoint (see [https://github.com/suitenumerique/drive/issues/476](https://github.com/suitenumerique/drive/issues/476)), one domain for collabora, and one domain for onlyoffice. For example (`drive.yourdomain.tld`, `rustfs.drive.yourdomain.tld`, `collabora.drive.yourdomain.tld`, `onlyoffice.drive.yourdomain.tld`). All of these domains need to have an A record pointing the IP address of your server. ### Installation Steps * `abra app new lasuite-drive --secrets` * `abra app config ` - - make sure to set MINIO_DOMAIN, COLLABORA_DOMAIN, ONLY_OFFICE_DOMAIN to the domains you set up for each. + - make sure to set RUSTFS_DOMAIN, COLLABORA_DOMAIN, ONLY_OFFICE_DOMAIN to the domains you set up for each. * `abra app deploy ` You should then be able to visit the landing page of your app, but not yet to login. To login, you need to deploy and integrate single sign on (described below in the "Configure Authentication" section). * Migrations run automatically on backend startup. To trigger manually: `abra app cmd backend migrate` -* Minio buckets are created automatically on first deploy. To manually trigger: `abra app cmd minio minio_initialize` +* The S3 bucket is created automatically on backend startup. To manually trigger: `abra app cmd backend s3_initialize` Wopi discovery is supposed to happen automatically, but if collabora/onlyoffice are not connecting, you can try running: * `abra app cmd backend trigger_wopi` # connects only office & collabora (if they stop working, try running this again) +Maintenance tasks (cleaning unfinished uploads, purging the trash) are scheduled by celery beat, see `CLEAN_PENDING_ITEMS_SCHEDULE` and `PURGE_DELETED_ITEMS_SCHEDULE` in `.env.sample`. + +## Migrating from MinIO + +Since upstream drive v0.24, and in this recipe since the version packaging drive v0.25, the S3 object storage is [RustFS](https://rustfs.com) instead of MinIO. RustFS cannot read the MinIO data directory, so the media files have to be copied once. RustFS uses new secrets and a new domain variable. + +1. `abra app config ` and add: + ``` + RUSTFS_DOMAIN="${MINIO_DOMAIN}" + SECRET_RUSTFS_RP_VERSION=v1 + SECRET_RUSTFS_RU_VERSION=v1 + COMPOSE_FILE="compose.yml:compose.minio-migration.yml" + ``` + Keep `MINIO_DOMAIN`, `SECRET_MINIO_RP_VERSION` and `SECRET_MINIO_RU_VERSION`, they are still needed during the migration. The `COMPOSE_FILE` line temporarily runs the old MinIO service on the old `minio` volume, next to RustFS. +2. Generate the RustFS credentials: + ``` + abra app secret generate rustfs_ru v1 + abra app secret generate rustfs_rp v1 + ``` +3. `abra app upgrade ` (or `abra app deploy --force`) +4. `abra app cmd backend migrate_from_minio` + + This copies all objects (latest version only) from MinIO to RustFS. Objects that already exist on RustFS are never overwritten, so if the copy is interrupted, just run it again. Once a run has completed, further runs are refused: they would copy back files that users deleted in the meantime. If you really need to, run `abra app cmd backend migrate_from_minio -- --force`. +5. Check that your files can be opened in drive, then remove the `COMPOSE_FILE` line from the config and redeploy: `abra app deploy --force`. If you like, replace `RUSTFS_DOMAIN="${MINIO_DOMAIN}"` with the actual domain and remove `MINIO_DOMAIN`, `SECRET_MINIO_RP_VERSION` and `SECRET_MINIO_RU_VERSION`. +6. Once you are sure everything is there, you can remove the old volume and secrets on the server: `docker volume rm _minio`, `abra app secret rm minio_ru` and `abra app secret rm minio_rp` + +Until step 4 is done, files uploaded before the upgrade will not be accessible. + ## Configure Authentication lasuite-drive **requires** an OpenID Connect (OIDC) single sign-on provider; deployment has been tested with [Keycloak](https://git.coopcloud.tech/coop-cloud/keycloak), which we recommend, or you could also try [Authentik](https://git.coopcloud.tech/coop-cloud/authentik), both of which are installable using Co-op Cloud. diff --git a/abra-entrypoint.sh b/abra-entrypoint.sh index 90fcc0b..72c8fec 100644 --- a/abra-entrypoint.sh +++ b/abra-entrypoint.sh @@ -3,8 +3,8 @@ set -e [ -f /run/secrets/postgres_p ] && export DB_PASSWORD="$(cat /run/secrets/postgres_p)" [ -f /run/secrets/django_sk ] && export DJANGO_SECRET_KEY="$(cat /run/secrets/django_sk)" -[ -f /run/secrets/minio_rp ] && export AWS_S3_SECRET_ACCESS_KEY="$(cat /run/secrets/minio_rp)" -[ -f /run/secrets/minio_ru ] && export AWS_S3_ACCESS_KEY_ID="$(cat /run/secrets/minio_ru)" +[ -f /run/secrets/rustfs_rp ] && export AWS_S3_SECRET_ACCESS_KEY="$(cat /run/secrets/rustfs_rp)" +[ -f /run/secrets/rustfs_ru ] && export AWS_S3_ACCESS_KEY_ID="$(cat /run/secrets/rustfs_ru)" [ -f /run/secrets/django_sp ] && export DJANGO_SUPERUSER_PASSWORD="$(cat /run/secrets/django_sp)" [ -f /run/secrets/oidc_rpcs ] && export OIDC_RP_CLIENT_SECRET="$(cat /run/secrets/oidc_rpcs)" [ -f /run/secrets/email_pass ] && export DJANGO_EMAIL_HOST_PASSWORD="$(cat /run/secrets/email_pass)" diff --git a/abra.sh b/abra.sh index 72ddd1a..08034ef 100755 --- a/abra.sh +++ b/abra.sh @@ -1,11 +1,12 @@ # Set any config versions here # Docs: https://docs.coopcloud.tech/maintainers/handbook/#manage-configs -export ABRA_ENTRYPOINT_VERSION=v11 -export NGINX_CONF_VERSION=v6 +export ABRA_ENTRYPOINT_VERSION=v12 +export NGINX_CONF_VERSION=v7 export ONLYOFFICE_CONF_VERSION=v2 export PG_BACKUP_VERSION=v4 export MIGRATE_VERSION=v1 -export MINIO_INITIALIZE_VERSION=v2 +export S3_INITIALIZE_VERSION=v1 +export MIGRATE_FROM_MINIO_VERSION=v1 environment() { # this exports all the secrets as environment variables @@ -16,8 +17,13 @@ migrate() { /migrate.sh } -minio_initialize() { - /minio-initialize.sh +s3_initialize() { + /s3-initialize.sh +} + +# only available when deployed with compose.minio-migration.yml +migrate_from_minio() { + /migrate-from-minio.sh "$@" } trigger_wopi() { diff --git a/compose.minio-migration.yml b/compose.minio-migration.yml new file mode 100644 index 0000000..5c25292 --- /dev/null +++ b/compose.minio-migration.yml @@ -0,0 +1,47 @@ +--- +# Temporary overlay to migrate the media files from the old MinIO service +# (recipe versions up to 0.13.x) to RustFS. See README "Migrating from MinIO". +services: + minio: + image: bitnamilegacy/minio:2025.7.23-debian-12-r5 + user: root + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_ru + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_rp + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 20s + retries: 60 + networks: + - backend + volumes: + - minio:/bitnami/minio/data + secrets: + - minio_rp + - minio_ru + + backend: + secrets: + - minio_rp + - minio_ru + configs: + - source: migrate_from_minio + target: /migrate-from-minio.sh + mode: 0555 + +volumes: + minio: + +configs: + migrate_from_minio: + name: ${STACK_NAME}_migrate_from_minio_${MIGRATE_FROM_MINIO_VERSION} + file: migrate-from-minio.sh + +secrets: + minio_rp: + external: true + name: ${STACK_NAME}_minio_rp_${SECRET_MINIO_RP_VERSION} + minio_ru: + external: true + name: ${STACK_NAME}_minio_ru_${SECRET_MINIO_RU_VERSION} diff --git a/compose.yml b/compose.yml index fc32287..0eda949 100644 --- a/compose.yml +++ b/compose.yml @@ -34,10 +34,10 @@ x-common-env: &common-env DRIVE_BASE_URL: "https://${DOMAIN}" # Media STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage - AWS_S3_ENDPOINT_URL: http://minio:9000 - AWS_S3_DOMAIN_REPLACE: https://${MINIO_DOMAIN} - # AWS_S3_ACCESS_KEY_ID supplied via secret (this is same MINIO_ROOT_USER) - # AWS_S3_SECRET_ACCESS_KEY supplied via secret (this is same as MINIO_ROOT_PASSWORD) + AWS_S3_ENDPOINT_URL: http://rustfs:9000 + AWS_S3_DOMAIN_REPLACE: https://${RUSTFS_DOMAIN} + # AWS_S3_ACCESS_KEY_ID supplied via secret (this is same as RUSTFS_ACCESS_KEY) + # AWS_S3_SECRET_ACCESS_KEY supplied via secret (this is same as RUSTFS_SECRET_KEY) MEDIA_BASE_URL: https://${DOMAIN} AWS_STORAGE_BUCKET_NAME: drive-media-storage # OIDC - settings from .env, see .env.sample @@ -63,6 +63,8 @@ x-common-env: &common-env AI_MODEL: llama # Collaboration COLLABORATION_API_URL: https://$DOMAIN/collaboration/api/ + # SDK picker pages (/sdk) and SDK relay API, used by other La Suite apps + SDK_ENABLED: ${SDK_ENABLED:-false} # WOPI WOPI_CLIENTS: "collabora,onlyoffice" @@ -87,9 +89,10 @@ x-postgres-env: &postgres-env DB_PORT: 5432 # DB_PASSWORD supplied via secrets (this is same as POSTGRES_PASSWORD) -x-minio-env: &minio-env - MINIO_ROOT_USER_FILE: /run/secrets/minio_ru - MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_rp +x-rustfs-env: &rustfs-env + RUSTFS_ACCESS_KEY_FILE: /run/secrets/rustfs_ru + RUSTFS_SECRET_KEY_FILE: /run/secrets/rustfs_rp + RUSTFS_CONSOLE_ENABLE: "false" services: @@ -119,7 +122,8 @@ services: image: lasuite/drive-backend:v0.25.0 command: [ "gunicorn", "-c", "/usr/local/etc/gunicorn/drive.py", "drive.wsgi:application" ] entrypoint: > - sh -c "if [ \"$$AUTO_MIGRATIONS\" = \"true\" ]; then /migrate.sh; fi && exec /abra-entrypoint.sh /usr/local/bin/entrypoint \"$$@\"" -- + sh -c "/s3-initialize.sh || echo '⚠ s3-initialize failed, run: abra app cmd backend s3_initialize' ; + if [ \"$$AUTO_MIGRATIONS\" = \"true\" ]; then /migrate.sh; fi && exec /abra-entrypoint.sh /usr/local/bin/entrypoint \"$$@\"" -- environment: <<: [ *common-env, *postgres-env ] AUTO_MIGRATIONS: "${AUTO_MIGRATIONS:-true}" @@ -134,12 +138,15 @@ services: - source: migrate target: /migrate.sh mode: 0555 + - source: s3_initialize + target: /s3-initialize.sh + mode: 0555 secrets: - django_sk - django_sp - oidc_rpcs - - minio_rp - - minio_ru + - rustfs_rp + - rustfs_ru - postgres_p - email_pass @@ -161,7 +168,8 @@ services: - django_sk - django_sp - oidc_rpcs - - minio_rp + - rustfs_rp + - rustfs_ru - postgres_p - email_pass @@ -174,6 +182,9 @@ services: entrypoint: ["/abra-entrypoint.sh", "/usr/local/bin/entrypoint"] environment: <<: [*common-env, *postgres-env] + # Maintenance commands scheduled by celery beat: crontab expression followed by command arguments + CLEAN_PENDING_ITEMS_SCHEDULE: ${CLEAN_PENDING_ITEMS_SCHEDULE:-30 0 * * *} + PURGE_DELETED_ITEMS_SCHEDULE: ${PURGE_DELETED_ITEMS_SCHEDULE:-45 0 * * *} configs: - source: abra_entrypoint target: /abra-entrypoint.sh @@ -182,7 +193,8 @@ services: - django_sk - django_sp - oidc_rpcs - - minio_rp + - rustfs_rp + - rustfs_ru - postgres_p - email_pass @@ -228,49 +240,38 @@ services: networks: - backend - minio: - image: bitnamilegacy/minio:2025.7.23-debian-12-r5 - user: root - environment: *minio-env + rustfs: + image: rustfs/rustfs:1.0.3 + environment: *rustfs-env healthcheck: - test: ["CMD", "mc", "ready", "local"] - interval: 1s + test: ["CMD", "curl", "-sf", "http://localhost:9000/health"] + interval: 5s timeout: 20s - retries: 300 + retries: 60 networks: - backend - proxy - command: /opt/bitnami/scripts/minio/run.sh - entrypoint: > - sh -c "/minio-initialize.sh & exec /opt/bitnami/scripts/minio/entrypoint.sh \"$$@\"" -- volumes: - - minio:/bitnami/minio/data - configs: - - source: abra_entrypoint - target: /abra-entrypoint.sh - mode: 0555 - - source: minio_initialize - target: /minio-initialize.sh - mode: 0555 + - rustfs:/data secrets: - - minio_rp - - minio_ru + - rustfs_rp + - rustfs_ru deploy: labels: - "traefik.enable=true" - "traefik.docker.network=proxy" - - "traefik.http.routers.${STACK_NAME}_minio.rule=Host(`${MINIO_DOMAIN}`)" - - "traefik.http.routers.${STACK_NAME}_minio.entrypoints=web-secure" - - "traefik.http.routers.${STACK_NAME}_minio.tls=true" - - "traefik.http.routers.${STACK_NAME}_minio.tls.certresolver=${LETS_ENCRYPT_ENV}" - - "traefik.http.services.${STACK_NAME}_minio.loadbalancer.server.port=9000" - - "traefik.http.middlewares.${STACK_NAME}_minio-cors.headers.accessControlAllowOriginList=https://${DOMAIN}" - - "traefik.http.middlewares.${STACK_NAME}_minio-cors.headers.accessControlAllowMethods=GET,POST,PUT,DELETE,OPTIONS" - - "traefik.http.middlewares.${STACK_NAME}_minio-cors.headers.accessControlAllowHeaders=*" - - "traefik.http.middlewares.${STACK_NAME}_minio-cors.headers.accessControlExposeHeaders=ETag,Content-Length" - - "traefik.http.middlewares.${STACK_NAME}_minio-cors.headers.accessControlMaxAge=600" - - "traefik.http.middlewares.${STACK_NAME}_minio-cors.headers.addVaryHeader=true" - - "traefik.http.routers.${STACK_NAME}_minio.middlewares=${STACK_NAME}_minio-cors" + - "traefik.http.routers.${STACK_NAME}_rustfs.rule=Host(`${RUSTFS_DOMAIN}`)" + - "traefik.http.routers.${STACK_NAME}_rustfs.entrypoints=web-secure" + - "traefik.http.routers.${STACK_NAME}_rustfs.tls=true" + - "traefik.http.routers.${STACK_NAME}_rustfs.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.services.${STACK_NAME}_rustfs.loadbalancer.server.port=9000" + - "traefik.http.middlewares.${STACK_NAME}_rustfs-cors.headers.accessControlAllowOriginList=https://${DOMAIN}" + - "traefik.http.middlewares.${STACK_NAME}_rustfs-cors.headers.accessControlAllowMethods=GET,POST,PUT,DELETE,OPTIONS" + - "traefik.http.middlewares.${STACK_NAME}_rustfs-cors.headers.accessControlAllowHeaders=*" + - "traefik.http.middlewares.${STACK_NAME}_rustfs-cors.headers.accessControlExposeHeaders=ETag,Content-Length" + - "traefik.http.middlewares.${STACK_NAME}_rustfs-cors.headers.accessControlMaxAge=600" + - "traefik.http.middlewares.${STACK_NAME}_rustfs-cors.headers.addVaryHeader=true" + - "traefik.http.routers.${STACK_NAME}_rustfs.middlewares=${STACK_NAME}_rustfs-cors" - "backupbot.backup=${ENABLE_BACKUPS:-true}" collabora: @@ -380,7 +381,7 @@ networks: volumes: postgres: - minio: + rustfs: configs: nginx_conf: @@ -396,9 +397,9 @@ configs: migrate: name: ${STACK_NAME}_migrate_${MIGRATE_VERSION} file: migrate.sh - minio_initialize: - name: ${STACK_NAME}_minio_initialize_${MINIO_INITIALIZE_VERSION} - file: minio-initialize.sh + s3_initialize: + name: ${STACK_NAME}_s3_initialize_${S3_INITIALIZE_VERSION} + file: s3-initialize.sh onlyoffice_conf: name: ${STACK_NAME}_onlyoffice_conf_${ONLYOFFICE_CONF_VERSION} file: onlyoffice-config.json.tmpl @@ -417,12 +418,12 @@ secrets: postgres_p: external: true name: ${STACK_NAME}_postgres_p_${SECRET_POSTGRES_P_VERSION} - minio_rp: + rustfs_rp: external: true - name: ${STACK_NAME}_minio_rp_${SECRET_MINIO_RP_VERSION} - minio_ru: + name: ${STACK_NAME}_rustfs_rp_${SECRET_RUSTFS_RP_VERSION} + rustfs_ru: external: true - name: ${STACK_NAME}_minio_ru_${SECRET_MINIO_RU_VERSION} + name: ${STACK_NAME}_rustfs_ru_${SECRET_RUSTFS_RU_VERSION} collabora_p: external: true name: ${STACK_NAME}_collabora_p_${SECRET_COLLABORA_P_VERSION} diff --git a/migrate-from-minio.sh b/migrate-from-minio.sh new file mode 100755 index 0000000..aa41f24 --- /dev/null +++ b/migrate-from-minio.sh @@ -0,0 +1,96 @@ +#!/bin/sh +set -e + +# One-shot copy of all objects from the old MinIO service (deployed with +# compose.minio-migration.yml) into the RustFS object storage. +# Only the latest version of each object is copied, and only objects missing +# on RustFS: objects changed on RustFS since (e.g. edited documents) are never +# overwritten. Once a run completes, a marker object is written and further +# runs are refused, unless called with --force. + +# Load secrets into environment +source /abra-entrypoint.sh -e + +python - "$@" <<'PYEOF' +import os +import sys + +import boto3 +from botocore.exceptions import ClientError + +bucket = os.environ.get("AWS_STORAGE_BUCKET_NAME", "drive-media-storage") +marker = "_minio-migration-done" +force = "--force" in sys.argv[1:] + + +def secret(name): + with open(f"/run/secrets/{name}") as f: + return f.read().strip() + + +def client(endpoint, access_key, secret_key): + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=os.environ.get("AWS_S3_REGION_NAME", "us-east-1"), + ) + + +def exists(s3, key): + try: + s3.head_object(Bucket=bucket, Key=key) + return True + except ClientError as error: + if error.response["Error"]["Code"] in ("404", "NoSuchKey", "NotFound"): + return False + raise + + +# the old MinIO uses the minio_* secrets, RustFS the rustfs_* secrets +src = client( + os.environ.get("MINIO_MIGRATION_ENDPOINT_URL", "http://minio:9000"), + secret("minio_ru"), + secret("minio_rp"), +) +dst = client( + os.environ["AWS_S3_ENDPOINT_URL"], + os.environ["AWS_S3_ACCESS_KEY_ID"], + os.environ["AWS_S3_SECRET_ACCESS_KEY"], +) + +if exists(dst, marker) and not force: + sys.exit( + "migrate-from-minio: the migration already completed, refusing to run again.\n" + "Objects deleted on RustFS since then would be copied back from MinIO.\n" + "Use 'migrate_from_minio --force' to run anyway (existing objects are never overwritten)." + ) + +try: + src.head_bucket(Bucket=bucket) +except Exception as error: + sys.exit(f"migrate-from-minio: cannot reach bucket '{bucket}' on minio: {error}") + +copied = skipped = 0 +for page in src.get_paginator("list_objects_v2").paginate(Bucket=bucket): + for obj in page.get("Contents", []): + key = obj["Key"] + # upload_fileobj is atomic: an object only exists once fully copied + if exists(dst, key): + skipped += 1 + continue + source = src.get_object(Bucket=bucket, Key=key) + extra = {"ContentType": source.get("ContentType", "binary/octet-stream")} + if source.get("Metadata"): + extra["Metadata"] = source["Metadata"] + if source.get("ContentDisposition"): + extra["ContentDisposition"] = source["ContentDisposition"] + dst.upload_fileobj(source["Body"], bucket, key, ExtraArgs=extra) + copied += 1 + if copied % 100 == 0: + print(f"migrate-from-minio: {copied} objects copied...") + +dst.put_object(Bucket=bucket, Key=marker, Body=b"") +print(f"migrate-from-minio: done, {copied} copied, {skipped} already present") +PYEOF diff --git a/minio-initialize.sh b/minio-initialize.sh deleted file mode 100644 index d1242a6..0000000 --- a/minio-initialize.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -set -e - -# Use an isolated mc config dir: the bundled Bitnami image runs its own -# internal mc client (as the "minio" user) against the default /.mc path, -# which collides with this script running as root against the same path. -export MC_CONFIG_DIR=/tmp/mc-drive-init - -# Wait for minio to be ready (up to 60 seconds) -i=0 -while ! mc ready local 2>/dev/null; do - i=$((i+1)) - if [ "$i" -ge 60 ]; then - echo "minio-initialize: timed out waiting for minio to be ready" >&2 - exit 1 - fi - sleep 1 -done - -MINIO_ROOT_USER="$(cat /run/secrets/minio_ru)" -MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_rp)" - -mc alias set drive http://localhost:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}" - -# Idempotent: skip if bucket already exists -if mc ls drive/drive-media-storage > /dev/null 2>&1; then - echo "minio-initialize: bucket 'drive-media-storage' already exists, skipping" - exit 0 -fi - -echo "minio-initialize: creating bucket 'drive-media-storage'..." -mc mb drive/drive-media-storage -mc version enable drive/drive-media-storage -echo "minio-initialize: done" diff --git a/nginx.conf.tmpl b/nginx.conf.tmpl index 0d19a95..04a3dda 100644 --- a/nginx.conf.tmpl +++ b/nginx.conf.tmpl @@ -56,9 +56,9 @@ server { proxy_set_header X-Amz-Date $authDate; proxy_set_header X-Amz-Content-SHA256 $authContentSha256; - # Get resource from Minio - proxy_pass http://minio:9000/drive-media-storage/; - proxy_set_header Host minio:9000; + # Get resource from the RustFS object storage + proxy_pass http://rustfs:9000/drive-media-storage/; + proxy_set_header Host rustfs:9000; } # Proxy auth for media-preview @@ -74,9 +74,9 @@ server { proxy_set_header X-Amz-Date $authDate; proxy_set_header X-Amz-Content-SHA256 $authContentSha256; - # Get resource from Minio - proxy_pass http://minio:9000/drive-media-storage/; - proxy_set_header Host minio:9000; + # Get resource from the RustFS object storage + proxy_pass http://rustfs:9000/drive-media-storage/; + proxy_set_header Host rustfs:9000; } location /media-auth { diff --git a/release/next b/release/next new file mode 100644 index 0000000..531d708 --- /dev/null +++ b/release/next @@ -0,0 +1,25 @@ +Upgrade to drive v0.25.0. + +**Breaking change:** following upstream, the S3 object storage changed from MinIO (`bitnamilegacy/minio`, unmaintained) to RustFS (`rustfs/rustfs`). RustFS cannot read the MinIO data, so existing media files must be copied once: + +``` +abra app config # add: + # RUSTFS_DOMAIN="${MINIO_DOMAIN}" + # SECRET_RUSTFS_RP_VERSION=v1 + # SECRET_RUSTFS_RU_VERSION=v1 + # COMPOSE_FILE="compose.yml:compose.minio-migration.yml" +abra app secret generate rustfs_ru v1 +abra app secret generate rustfs_rp v1 +abra app upgrade +abra app cmd backend migrate_from_minio +abra app config # remove the COMPOSE_FILE line +abra app deploy --force +``` + +See README "Migrating from MinIO" for details. RustFS uses the new `rustfs_ru`/`rustfs_rp` secrets and `RUSTFS_DOMAIN`. The old `minio_ru`/`minio_rp` secrets and `MINIO_DOMAIN` are only needed during the migration. + +Other changes: +- The S3 bucket is now created by the backend on startup (`abra app cmd backend s3_initialize`), the `minio_initialize` command is gone. +- Maintenance commands `clean_pending_items` (daily 0:30) and `purge_deleted_items` (daily 0:45) are now scheduled by celery beat. Configure with `CLEAN_PENDING_ITEMS_SCHEDULE` / `PURGE_DELETED_ITEMS_SCHEDULE`. Note that this permanently purges items deleted from the trash after the upstream grace periods (30 + 7 days). +- New `SDK_ENABLED` setting (default `false`) for the SDK picker used by other La Suite apps. +- The celery workers now get the S3 access key secret too. diff --git a/s3-initialize.sh b/s3-initialize.sh new file mode 100755 index 0000000..f7ac356 --- /dev/null +++ b/s3-initialize.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -e + +# Creates the media bucket on the RustFS object storage, with versioning +# enabled. Runs inside the backend container, which ships boto3. + +# Load secrets into environment +source /abra-entrypoint.sh -e + +python - <<'PYEOF' +import os +import sys +import time + +import boto3 +from botocore.exceptions import ClientError, EndpointConnectionError + +bucket = os.environ.get("AWS_STORAGE_BUCKET_NAME", "drive-media-storage") +s3 = boto3.client( + "s3", + endpoint_url=os.environ["AWS_S3_ENDPOINT_URL"], + aws_access_key_id=os.environ["AWS_S3_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_S3_SECRET_ACCESS_KEY"], + region_name=os.environ.get("AWS_S3_REGION_NAME", "us-east-1"), +) + +# Wait for the object storage to be ready (up to 60 seconds) +for i in range(60): + try: + s3.list_buckets() + break + except (ClientError, EndpointConnectionError): + time.sleep(1) +else: + sys.exit("s3-initialize: timed out waiting for the object storage") + +# Idempotent: skip if bucket already exists +try: + s3.head_bucket(Bucket=bucket) + print(f"s3-initialize: bucket '{bucket}' already exists, skipping") + sys.exit(0) +except ClientError: + pass + +print(f"s3-initialize: creating bucket '{bucket}'...") +s3.create_bucket(Bucket=bucket) +s3.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"} +) +print("s3-initialize: done") +PYEOF -- 2.54.0