From 100fb038e3ed1049ee1c035234463fdf05a0b9f1 Mon Sep 17 00:00:00 2001 From: mirsal Date: Fri, 19 Mar 2021 16:03:13 +0000 Subject: [PATCH] Untested port of peertube to coop-cloud missing: * live streaming and RTMP proxying with traefik * webseed cross-instance redundancy * option to use S3-compatible storage * testing, there might be silly bugs Did I mention this is untested? --- .env.sample | 12 +++ compose.yml | 101 ++++++++++++++++++++- entrypoint.sh | 26 ++++++ nginx.conf.tmpl | 234 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 372 insertions(+), 1 deletion(-) create mode 100644 entrypoint.sh create mode 100644 nginx.conf.tmpl diff --git a/.env.sample b/.env.sample index 88dce54..ebcdc22 100644 --- a/.env.sample +++ b/.env.sample @@ -4,3 +4,15 @@ DOMAIN=peertube.example.com ## Domain aliases #EXTRA_DOMAINS=', `www.peertube.example.com`' LETS_ENCRYPT_ENV=production + +PEERTUBE_WEBSERVER_PORT: 443 +PEERTUBE_WEBSERVER_HTTPS: "true" +PEERTUBE_ADMIN_EMAIL: admin@example.com +PEERTUBE_SIGNUP_ENABLED: "false" +PEERTUBE_TRANSCODING_ENABLED: "true" +PEERTUBE_SMTP_HOSTNAME: smtpd +PEERTUBE_SMTP_PORT: 25 +PEERTUBE_SMTP_FROM: peertube@example.com +PEERTUBE_SMTP_TLS: "false" +PEERTUBE_SMTP_DISABLE_STARTTLS: "true" +PEERTUBE_TRUST_PROXY: '["127.0.0.1", "loopback", "172.16.0.0/12", "10.0.0.0/8"]' diff --git a/compose.yml b/compose.yml index 662a96c..c9a9ef0 100644 --- a/compose.yml +++ b/compose.yml @@ -2,10 +2,21 @@ version: "3.8" services: - app: + nginx: image: nginx:1.19.2 networks: - proxy + - app + volumes: + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + - app-data:/var/www/peertube/storage:ro + - app-assets:/var/www/peertube/peertube-latest/client/dist:ro + - nginx-tmp:/var/lib/nginx/client-tmp:rw + - nginx-cache:/var/cache/nginx/peertube-media-cache:rw + configs: + - source: nginx_config + target: /etc/nginx/conf.d/default.conf deploy: restart_policy: condition: on-failure @@ -19,13 +30,101 @@ services: #- "traefik.http.routers.${STACK_NAME}.middlewares=${STACK_NAME}-redirect" #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLForceHost=true" #- "traefik.http.middlewares.${STACK_NAME}-redirect.headers.SSLHost=${DOMAIN}" + depends_on: + - app healthcheck: test: ["CMD", "curl", "-f", "http://localhost"] interval: 30s timeout: 10s retries: 10 + start_period: 10s + + app: + image: chocobozzz/peertube:v3.1.0-rc.1-buster + environment: + PEERTUBE_WEBSERVER_HOSTNAME: ${DOMAIN} + PEERTUBE_DB_HOSTNAME: db + PEERTUBE_DB_USERNAME: peertube + PEERTUBE_DB_PASSWORD_FILE: /run/secrets/db_password + PEERTUBE_REDIS_HOSTNAME: redis + volumes: + - app-data:/data + - app-config:/config + - app-assets:/app/client/dist + secrets: + - db_password + configs: + - source: entrypoint_conf + target: /docker-entrypoint.sh + mode: 0555 + entrypoint: /docker-entrypoint.sh + networks: + app: + postfix_smtp: + redis: + db: + depends_on: + - db + - redis + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000"] + interval: 30s + timeout: 10s + retries: 10 start_period: 1m + db: + image: postgres:10-alpine + environment: + POSTGRES_USER: peertube + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + POSTGRES_DB: peertube + secrets: + - db_password + volumes: + - postgres-data:/var/lib/postgresql/data + networks: + db: + + redis: + image: redis:4-alpine + volumes: + - redis-data:/data + networks: + redis: + networks: proxy: external: true + app: + driver: overlay + attachable: true + redis: + driver: overlay + attachable: true + db: + driver: overlay + attachable: true + +volumes: + app-data: + app-config: + app-assets: + nginx-tmp: + nginx-cache: + postgres-data: + redis-data: + +configs: + nginx_config: + name: ${STACK_NAME}_nginx_config_${NGINX_CONFIG_VERSION} + file: nginx.conf.tmpl + template_driver: golang + app_entrypoint: + name: ${STACK_NAME}_app_entrypoint_${APP_ENTRYPOINT_VERSION} + file: entrypoint.sh + +secrets: + db_password: + external: true + name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION} diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..d9b80f7 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -e + +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +file_env "PEERTUBE_DB_PASSWORD" + +exec /entrypoint.sh "$@" diff --git a/nginx.conf.tmpl b/nginx.conf.tmpl new file mode 100644 index 0000000..acbf15f --- /dev/null +++ b/nginx.conf.tmpl @@ -0,0 +1,234 @@ +upstream backend { + server {{ env "STACK_NAME" }}_app:9000; +} + +proxy_cache_path /var/cache/nginx/peertube-media-cache levels=1:2 keys_zone=peertube_media_cache:5m max_size=40g inactive=72h use_temp_path=off; + +server { + listen 80; + server_name _; + + ## + # Application + ## + + location @api { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + + client_max_body_size 100k; # default is 1M + + proxy_connect_timeout 10m; + proxy_send_timeout 10m; + proxy_read_timeout 10m; + send_timeout 10m; + + proxy_pass http://backend; + } + + location / { + try_files /dev/null @api; + } + + location = /api/v1/videos/upload { + limit_except POST HEAD { deny all; } + + # This is the maximum upload size, which roughly matches the maximum size of a video file. + # Note that temporary space is needed equal to the total size of all concurrent uploads. + # This data gets stored in /var/lib/nginx by default, so you may want to put this directory + # on a dedicated filesystem. + client_max_body_size 12G; # default is 1M + add_header X-File-Maximum-Size 8G always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size) + + try_files /dev/null @api; + } + + location ~ ^/api/v1/(videos|video-playlists|video-channels|users/me) { + client_max_body_size 3M; # default is 1M + add_header X-File-Maximum-Size 2M always; # inform backend of the set value in bytes before mime-encoding (x * 1.4 >= client_max_body_size) + + try_files /dev/null @api; + } + + ## + # Websocket + ## + + location @api_websocket { + proxy_http_version 1.1; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_pass http://backend; + } + + location /socket.io { + try_files /dev/null @api_websocket; + } + + location /tracker/socket { + # Peers send a message to the tracker every 15 minutes + # Don't close the websocket before then + proxy_read_timeout 15m; # default is 60s + + try_files /dev/null @api_websocket; + } + + ## + # Performance optimizations + # For extra performance please refer to https://github.com/denji/nginx-tuning + ## + + root /var/www/peertube/storage; + + # Enable compression for JS/CSS/HTML, for improved client load times. + # It might be nice to compress JSON/XML as returned by the API, but + # leaving that out to protect against potential BREACH attack. + gzip on; + gzip_vary on; + gzip_types # text/html is always compressed by HttpGzipModule + text/css + application/javascript + font/truetype + font/opentype + application/vnd.ms-fontobject + image/svg+xml; + gzip_min_length 1000; # default is 20 bytes + gzip_buffers 16 8k; + gzip_comp_level 2; # default is 1 + + client_body_timeout 30s; # default is 60 + client_header_timeout 10s; # default is 60 + send_timeout 10s; # default is 60 + keepalive_timeout 10s; # default is 75 + resolver_timeout 10s; # default is 30 + reset_timedout_connection on; + proxy_ignore_client_abort on; + + tcp_nopush on; # send headers in one piece + tcp_nodelay on; # don't buffer data sent, good for small data bursts in real time + + # If you have a small /var/lib partition, it could be interesting to store temp nginx uploads in a different place + # See https://nginx.org/en/docs/http/ngx_http_core_module.html#client_body_temp_path + client_body_temp_path /var/lib/nginx/client-tmp; + + # Bypass PeerTube for performance reasons. Optional. + # Should be consistent with client-overrides assets list in /server/controllers/client.ts + location ~ ^/client/(assets/images/(icons/icon-36x36\.png|icons/icon-48x48\.png|icons/icon-72x72\.png|icons/icon-96x96\.png|icons/icon-144x144\.png|icons/icon-192x192\.png|icons/icon-512x512\.png|logo\.svg|favicon\.png))$ { + add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year + + root /var/www/peertube; + + try_files /storage/client-overrides/$1 /peertube-latest/client/dist/$1 @api; + } + + # Bypass PeerTube for performance reasons. Optional. + location ~ ^/client/(.*\.(js|css|png|svg|woff2|otf|ttf|woff|eot))$ { + add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year + + alias /var/www/peertube/peertube-latest/client/dist/$1; + } + + # Bypass PeerTube for performance reasons. Optional. + location ~ ^/static/(thumbnails|avatars)/ { + if ($request_method = 'OPTIONS') { + add_header Access-Control-Allow-Origin '*'; + add_header Access-Control-Allow-Methods 'GET, OPTIONS'; + add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + add_header Access-Control-Max-Age 1728000; # Preflight request can be cached 20 days + add_header Content-Type 'text/plain charset=UTF-8'; + add_header Content-Length 0; + return 204; + } + + add_header Access-Control-Allow-Origin '*'; + add_header Access-Control-Allow-Methods 'GET, OPTIONS'; + add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + add_header Cache-Control "public, max-age=7200"; # Cache response 2 hours + + rewrite ^/static/(.*)$ /$1 break; + + try_files $uri @api; + } + + # Bypass PeerTube for performance reasons. Optional. + location ~ ^/static/(webseed|redundancy|streaming-playlists)/ { + limit_rate_after 5M; + + # Clients usually have 4 simultaneous webseed connections, so the real limit is 3MB/s per client + set $peertube_limit_rate 800k; + + # Increase rate limit in HLS mode, because we don't have multiple simultaneous connections + if ($request_uri ~ -fragmented.mp4$) { + set $peertube_limit_rate 5M; + } + + # Use this line with nginx >= 1.17.0 + limit_rate $peertube_limit_rate; + # Or this line if your nginx < 1.17.0 + #set $limit_rate $peertube_limit_rate; + + if ($request_method = 'OPTIONS') { + add_header Access-Control-Allow-Origin '*'; + add_header Access-Control-Allow-Methods 'GET, OPTIONS'; + add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + add_header Access-Control-Max-Age 1728000; # Preflight request can be cached 20 days + add_header Content-Type 'text/plain charset=UTF-8'; + add_header Content-Length 0; + return 204; + } + + if ($request_method = 'GET') { + add_header Access-Control-Allow-Origin '*'; + add_header Access-Control-Allow-Methods 'GET, OPTIONS'; + add_header Access-Control-Allow-Headers 'Range,DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'; + add_header X-Cache-Status $upstream_cache_status; + add_header X-Cache-Range $slice_range; + add_header X-Fuuu ok; + + add_header X-Accel-Expires 604800; + + # Don't spam access log file with byte range requests + access_log off; + } + + # Enabling the sendfile directive eliminates the step of copying the data into the buffer + # and enables direct copying data from one file descriptor to another. + sendfile on; + sendfile_max_chunk 1M; # prevent one fast connection from entirely occupying the worker process. should be > 800k. + aio threads; + + proxy_connect_timeout 70; + proxy_read_timeout 1200; + proxy_send_timeout 1200; + + proxy_buffering on; + proxy_buffers 8 128k; + proxy_cache peertube_media_cache; + + slice 64k; + proxy_cache_key $host$uri$is_args$args$slice_range; + proxy_set_header Range $slice_range; + proxy_cache_valid 200 206 1h; + proxy_cache_revalidate on; + + proxy_http_version 1.1; + proxy_redirect off; + + + # Use this in tandem with fuse-mounting i.e. https://docs.joinpeertube.org/admin-remote-storage + # to serve files directly from a public bucket without proxying. + # Assumes you have buckets named after the storage subdirectories, i.e. 'videos', 'redundancy', etc. + #set $cdn https://bucket-name.cdn.cloud; + #rewrite ^/static/webseed/(.*)$ $cdn/videos/$1 redirect; + #rewrite ^/static/(.*)$ $cdn/$1 redirect; + rewrite ^/static/webseed/(.*)$ /videos/$1 break; + rewrite ^/static/(.*)$ /$1 break; + + try_files $uri @api; + #proxy_pass https://bucket-name.s3.api-endpoint.cloud; + } +}