Merge branch 'master' into ccchaos

docs: Add page about user roles

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+github: snikket-im
+liberapay: Snikket
+custom: https://snikket.org/donate/

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Snikket Server changelog
 
+## UNRELEASED
+
+- Increase shared file size limit from 16MB to 100MB
+- Allow configurable storage quota for shared files
+- Initial support for "limited" user accounts
+- Support for group chat notifications on iOS
+- Configurable port range for TURN service
+- Ability to see basic server metrics in the web admin interface
+- Support for advanced monitoring/alerting via Prometheus
+
+### Upgrading
+
+If you are using a reverse proxy in front of Snikket, ensure it can
+handle the new upload limit (for example, in nginx the `client_max_body_size`
+option).
+
 ## beta.20210519
 
 - Allow custom HTTP bind interface

Dockerfile

@@ -24,20 +24,21 @@ RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         software-properties-common ca-certificates \
         gpg gpg-agent \
-        ansible python-passlib python3-passlib \
+        ansible python3-passlib \
         libcap2-bin build-essential\
-    && rm -rf /var/lib/apt/lists/* \
     && c_rehash \
-    && ansible-playbook -c local -i localhost, --extra-vars "ansible_python_interpreter=/usr/bin/python2" /opt/ansible/snikket.yml \
-    && apt-get remove -y \
+    && ansible-playbook -c local -i localhost, --extra-vars "ansible_python_interpreter=/usr/bin/python3" /opt/ansible/snikket.yml \
+    && apt-get remove --purge -y \
          ansible \
          software-properties-common \
          gpg gpg-agent \
-         python-passlib python3-passlib \
+         python3-passlib \
          mercurial libcap2-bin build-essential \
+         python3 python3.7-minimal \
     && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* \
     && rm -rf /var/cache/*
 
-RUN echo "Snikket $BUILD_SERIES.$BUILD_ID" > /usr/lib/prosody/prosody.version
+RUN echo "Snikket $BUILD_SERIES $BUILD_ID" > /usr/lib/prosody/prosody.version
 
 VOLUME ["/snikket"]

Makefile

@@ -1,15 +1,15 @@
-.PHONY: all docker
+.PHONY: all docker site
 
-DOCS=$(docs/**.md)
+DOCS := $(docs/**.md)
 
 all: docker
 
 docker:
-	docker build -t snikket -f docker/Dockerfile .
+	docker build -t snikket .
 
 site: mkdocs.yml $(DOCS)
 	echo $(DOCS)
-	mkdocs
+	mkdocs build
 
 docs/_po/snikket-server-docs.pot: po4a.conf $(DOCS)
 	po4a \

README.md

@@ -1,19 +1,35 @@
-# Snikket builder
+# Snikket server images
 
 This is the source repository for building [Snikket service](https://snikket.org/service/)
 Docker images.
 
-## Requirements
+Snikket is an open-source self-hosted personal messaging service. It aims to
+provide an alternative to proprietary and centralized messaging platforms
+while supporting all the expected features and being easy to use.
+
+For more information see the [Snikket website](https://snikket.org/).
+
+## Getting Started with Snikket
+
+For instructions on getting started with Snikket, see the [Snikket installation
+guide](https://snikket.org/service/quickstart/) on our website.
+
+## Building images
+
+This section is for people who want to build their own images of Snikket, e.g.
+for development purposes.
+
+### Requirements
 
  - GNU make
  - docker (tested on 19.03.5)
  - ansible (tested on 2.7 (debian buster))
 
-## Building
+### Building
 
 Run `make`
 
-## Running
+### Running
 
 The easiest way is to use docker-compose. Copy the file `snikket.conf.example` to
 `snikket.conf` and edit the values in it. Then run:

ansible/files/bin/create-invite

@@ -3,6 +3,7 @@
 SHOW_QR=0
 if [ "$1" == "--qr" ]; then
 	SHOW_QR=1;
+	shift;
 fi
 
 URL=$(prosodyctl mod_invites generate "$SNIKKET_DOMAIN" "$@")

ansible/files/bin/start-coturn.sh

@@ -5,8 +5,8 @@ if [ "$SNIKKET_TWEAK_TURNSERVER" = "0" ]; then
 	exit 0;
 fi
 
-CERTFILE="/snikket/letsencrypt/live/$SNIKKET_DOMAIN/fullchain.pem";
-KEYFILE="/snikket/letsencrypt/live/$SNIKKET_DOMAIN/privkey.pem";
+CERTFILE="${SNIKKET_CERTFILE:-/snikket/letsencrypt/live/$SNIKKET_DOMAIN/fullchain.pem}";
+KEYFILE="${SNIKKET_KEYFILE:-/snikket/letsencrypt/live/$SNIKKET_DOMAIN/privkey.pem}";
 
 echo "Waiting for certificates to become available..."
 while ! test -f "$CERTFILE" -a -f "$KEYFILE"; do
@@ -16,7 +16,11 @@ done
 
 TURN_EXTERNAL_IP="$(snikket-turn-addresses "$SNIKKET_DOMAIN")"
 
+min_port="${SNIKKET_TWEAK_TURNSERVER_MIN_PORT:-49152}"
+max_port="${SNIKKET_TWEAK_TURNSERVER_MAX_PORT:-65535}"
+
 exec /usr/bin/turnserver -c /etc/turnserver.conf --prod \
      --static-auth-secret="$(cat /snikket/prosody/turn-auth-secret-v2)" \
      --cert="$CERTFILE" --pkey "$KEYFILE" -r "$SNIKKET_DOMAIN" \
+     --min-port "$min_port" --max-port "$max_port" \
      -X "$TURN_EXTERNAL_IP"

ansible/files/prosody.cfg.lua

@@ -1,13 +1,16 @@
 local DOMAIN = assert(ENV_SNIKKET_DOMAIN, "Please set the SNIKKET_DOMAIN environment variable")
 
 local RETENTION_DAYS = tonumber(ENV_SNIKKET_RETENTION_DAYS) or 7;
+local UPLOAD_STORAGE_GB = tonumber(ENV_SNIKKET_UPLOAD_STORAGE_GB);
+
+local CERT_PATH = ENV_SNIKKET_CERTFILE or "/etc/prosody/certs/"..DOMAIN..".crt";
+local KEY_PATH = ENV_SNIKKET_KEYFILE or "/etc/prosody/certs/"..DOMAIN..".key";
 
 if prosody.process_type == "prosody" and not prosody.config_loaded then
 	-- Wait at startup for certificates
 	local lfs, socket = require "lfs", require "socket";
-	local cert_path = "/etc/prosody/certs/"..DOMAIN..".crt";
 	local counter = 0;
-	while not lfs.attributes(cert_path, "mode") do
+	while not lfs.attributes(CERT_PATH, "mode") do
 		counter = counter + 1;
 		if counter == 1 or counter%6 == 0 then
 			print("Waiting for certificates...");
@@ -28,6 +31,13 @@ data_path = "/snikket/prosody"
 
 pidfile = "/var/run/prosody/prosody.pid"
 
+admin_shell_prompt = ("prosody [%s]> "):format(DOMAIN)
+
+-- Aggressive GC to reduce resource consumption. These values are not
+-- incredibly scientific, but should be good for a small private server.
+-- They should be reviewed on the upgrade to Lua 5.4.
+gc = { threshold = 100, speed = 750 }
+
 modules_enabled = {
 
 	-- Generally required
@@ -54,9 +64,7 @@ modules_enabled = {
 
 	-- Push notifications
 		"cloud_notify";
-		"cloud_notify_encrypted";
-		"cloud_notify_priority_tag";
-		"cloud_notify_filters";
+		"cloud_notify_extensions";
 
 	-- HTTP modules
 		"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP"
@@ -76,6 +84,15 @@ modules_enabled = {
 		"update_notify";
 		"turncredentials";
 		"admin_shell";
+		"isolate_host";
+		"snikket_client_id";
+		"snikket_ios_preserve_push";
+		"snikket_restricted_users";
+		"lastlog2";
+
+	-- Spam/abuse management
+		"spam_reporting"; -- Allow users to report spam/abuse
+		"watch_spam_reports"; -- Alert admins of spam/abuse reports by users
 
 	-- TODO...
 		--"groups"; -- Shared roster support
@@ -109,6 +126,10 @@ modules_enabled = {
 		"http_oauth2";
 		"http_admin_api";
 		"rest";
+
+	-- Monitoring & maintenance
+		"measure_process";
+		"measure_active_users";
 }
 
 registration_watchers = {} -- Disable by default
@@ -132,6 +153,9 @@ registration_invite_only = true
 -- over what happens when a user invites someone.
 allow_contact_invites = false
 
+-- Disallow restricted users to create invitations to the server
+deny_user_invites_by_roles = { "prosody:restricted" }
+
 invites_page = ENV_SNIKKET_INVITE_URL or ("https://"..DOMAIN.."/invite/{invite.token}/");
 invites_page_external = true
 
@@ -160,7 +184,18 @@ authorization = "internal"
 storage = "internal"
 statistics = "internal"
 
-certificates = "certs"
+if ENV_SNIKKET_TWEAK_PROMETHEUS == "1" then
+	-- When using Prometheus, it is desirable to let the prometheus scraping
+	-- drive the sampling of metrics
+	statistics_interval = "manual"
+else
+	-- When not using Prometheus, we need an interval so that the metrics can
+	-- be shown by the web portal. The HTTP admin API exposure does not force
+	-- a collection as it is only interested in very few specific metrics.
+	statistics_interval = 60
+end
+
+-- certificates = "certs"
 
 group_default_name = ENV_SNIKKET_SITE_NAME or DOMAIN
 
@@ -178,16 +213,25 @@ end
 http_default_host = DOMAIN
 http_host = DOMAIN
 http_external_url = "https://"..DOMAIN.."/"
-http_max_content_size = 1024 * 1024 * 16 -- 16MB
 
 if ENV_SNIKKET_TWEAK_TURNSERVER ~= "0" or ENV_SNIKKET_TWEAK_TURNSERVER_DOMAIN then
 	turncredentials_host = ENV_SNIKKET_TWEAK_TURNSERVER_DOMAIN or DOMAIN
 	turncredentials_secret = ENV_SNIKKET_TWEAK_TURNSERVER_SECRET or assert(io.open("/snikket/prosody/turn-auth-secret-v2")):read("*l");
 end
 
+-- Allow restricted users access to push notification servers
+isolate_except_domains = { "push.snikket.net", "push-ios.snikket.net" }
+
 VirtualHost (DOMAIN)
 	authentication = "internal_hashed"
 
+	ssl = {
+		ciphers = "EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS:!RC4";
+		certificate = CERT_PATH;
+		key = KEY_PATH;
+	};
+
+
 	http_files_dir = "/var/www"
 	http_paths = {
 		files = "/";
@@ -196,6 +240,12 @@ VirtualHost (DOMAIN)
 		invites_register = "/register";
 	}
 
+	if ENV_SNIKKET_TWEAK_PROMETHEUS == "1" then
+		modules_enabled = {
+			"prometheus";
+		}
+	end
+
 	welcome_message = [[Hi, welcome to Snikket on $host! Thanks for joining us.]]
 	.."\n\n"
 	..[[For help and enquiries related to this service you may contact the admin via email: ]]
@@ -209,11 +259,21 @@ Component ("groups."..DOMAIN) "muc"
 		"muc_local_only";
 		"vcard_muc";
 		"muc_defaults";
+		"muc_offline_delivery";
+		"snikket_restricted_users";
+		"muc_auto_reserve_nicks";
 	}
 	restrict_room_creation = "local"
 	muc_local_only = { "general@groups."..DOMAIN }
-	muc_room_default_persistent = true
+
+	-- Default configuration for rooms (typically overwritten by the client)
 	muc_room_default_allow_member_invites = true
+	muc_room_default_persistent = true
+	muc_room_default_public = false
+
+	-- Enable push notifications for offline group members by default
+	-- (this also requires mod_muc_auto_reserve_nicks in practice)
+	muc_offline_delivery_default = true
 
 	default_mucs = {
 		{
@@ -232,13 +292,24 @@ Component ("groups."..DOMAIN) "muc"
 		}
 	}
 
-Component ("share."..DOMAIN) "http_upload"
+Component ("share."..DOMAIN) "http_file_share"
 	-- For backwards compat, allow HTTP upload on the base domain
 	if ENV_SNIKKET_TWEAK_SHARE_DOMAIN ~= "1" then
 		http_host = "share."..DOMAIN
 		http_external_url = "https://share."..DOMAIN.."/"
 	end
-	http_upload_file_size_limit = 1024 * 1024 * 16 -- 16MB
-	http_upload_expire_after = 60 * 60 * 24 * RETENTION_DAYS -- N days
+
+	-- 128 bits (i.e. 16 bytes) is the maximum length of a GCM auth tag, which
+	-- is appended to encrypted uploads according to XEP-0454. This ensures we
+	-- allow files up to the size limit even if they are encrypted.
+	http_file_share_size_limit = (1024 * 1024 * 100) + 16 -- 100MB + 16 bytes
+	http_file_share_expire_after = 60 * 60 * 24 * RETENTION_DAYS -- N days
+
+	if UPLOAD_STORAGE_GB then
+		http_file_share_global_quota = 1024 * 1024 * 1024 * UPLOAD_STORAGE_GB
+	end
+	http_paths = {
+		file_share = "/upload"
+	}
 
 Include (ENV_SNIKKET_TWEAK_EXTRA_CONFIG or "/snikket/prosody/*.cfg.lua")

ansible/files/turnserver.conf

@@ -43,8 +43,9 @@ alt-tls-listening-port=0
 # Lower and upper bounds of the UDP relay endpoints:
 # (default values are 49152 and 65535)
 #
-min-port=49152
-max-port=65535
+# THESE ARE OVERRIDDEN BY start-turn.sh!
+#min-port=49152
+#max-port=65535
 	
 # TURN REST API flag.
 # Flag that sets a special authorization option that is based upon authentication secret.

ansible/snikket.yml

@@ -7,9 +7,9 @@
   vars:
     prosody:
       package: "prosody-trunk"
-      build: "1436"
+      build: "1544"
     prosody_modules:
-      revision: "8e58a1b78336"
+      revision: "4abb33a15897"
   tasks:
     - import_tasks: tasks/prosody.yml
     - import_tasks: tasks/supervisor.yml

ansible/tasks/prosody.yml

@@ -10,6 +10,7 @@
     url: "https://packages.prosody.im/debian/pubkey.asc"
 - name: "Add Prosody package repo"
   apt_repository:
+    filename: prosody
     repo: "deb https://packages.prosody.im/debian buster main"
 - name: "Detect dpkg architecture name"
   shell: dpkg --print-architecture
@@ -78,14 +79,15 @@
   loop:
     - mod_smacks
     - mod_cloud_notify
+    - mod_cloud_notify_extensions
     - mod_cloud_notify_encrypted
     - mod_cloud_notify_priority_tag
     - mod_cloud_notify_filters
     - mod_block_registrations
     - mod_compact_resource
     - mod_conversejs
-    - mod_http_upload
-    - mod_lastlog
+    - mod_migrate_http_upload
+    - mod_lastlog2
     - mod_limit_auth
     - mod_password_policy
     - mod_roster_allinall
@@ -115,7 +117,15 @@
     - mod_groups_muc_bookmarks
     - mod_muc_defaults
     - mod_muc_local_only
+    - mod_muc_offline_delivery
     - mod_http_host_status_check
+    - mod_measure_process
+    - mod_prometheus
+    - mod_spam_reporting
+    - mod_watch_spam_reports
+    - mod_isolate_host
+    - mod_muc_auto_reserve_nicks
+    - mod_measure_active_users
 
 - name: Enable wanted modules (snikket-modules)
   file:
@@ -127,6 +137,9 @@
     - mod_update_notify
     - mod_invites_default_group
     - mod_invites_bootstrap
+    - mod_snikket_client_id
+    - mod_snikket_ios_preserve_push
+    - mod_snikket_restricted_users
 
 - name: "Install lua-ossl for encrypted push notifications"
   apt:
@@ -136,7 +149,8 @@
 
 - name: "Fetch luaunbound source"
   get_url:
-    url: https://code.zash.se/dl/luaunbound/luaunbound-0.5.tar.gz
+    #url: https://code.zash.se/dl/luaunbound/luaunbound-0.5.tar.gz
+    url: https://matthewwild.co.uk/uploads/luaunbound-0.5.tar.gz
     sha256sum: a6564ac1cca6bb350576eb2a5cfa03adb0aafd4f99d6cd491bd8028d046c62a7
     dest: /tmp/luaunbound-0.5.tar.gz
 

ansible/tasks/scripts.yml

@@ -5,3 +5,8 @@
     src: "../files/bin/"
     dest: "/usr/local/bin/"
     mode: 0755
+
+- name: "Install qrencode"
+  apt:
+    name: qrencode
+    state: present

docker/entrypoint.sh

@@ -48,4 +48,8 @@ if test -f /snikket/prosody/turn-auth-secret; then
 	rm /snikket/prosody/turn-auth-secret;
 fi
 
+if test -d /snikket/prosody/http_upload; then
+	prosodyctl mod_migrate_http_upload "share.$SNIKKET_DOMAIN" "$SNIKKET_DOMAIN"
+fi
+
 exec supervisord -c /etc/supervisor/supervisord.conf

docs/advanced/config.md

@@ -0,0 +1,144 @@
+# Advanced Configuration
+
+In most situations, the configuration options shown in the example config in
+[snikket-selfhosted](https://github.com/snikket-im/snikket-selfhosted/blob/main/snikket.conf.example) should suffice. In some cases of more complex requirements (such as running behind a reverse proxy), it may be required to tweak more options.
+
+## Note well
+
+- **Some of these options may break your setup**
+
+  Do not set them unless you know what you're doing. Particularly the options with `TWEAK` in their name are to be looked at carefully.
+
+- Options *only* documented here may change their behaviour between releases without further notice
+
+  There is no guarantee about any of the options documented *only* here. Some are experimental, some are reserved for specific uncommon use cases (for which the support may be dropped eventually), others only exist to glue Snikket components together and should not be touched at all.
+
+Also, it is very likely not complete.
+
+## Configuration Option Reference
+
+This reference is in no particular order. Most importantly, it is certainly not in the order of "things you should try to mess with come first".
+
+### `SNIKKET_DOMAIN`
+
+The domain name of your Snikket instance. Do not change this after it was once set.
+
+### `SNIKKET_RETENTION_DAYS`
+
+The number of days (as integer) for which your server should preserve messages so that all devices of a user can catch up, even if they end up being disconnected from the internet for a while.
+
+The Snikket Server stores all messages which are sent to any user for the given number of days. As end-to-end encryption is used, no plaintext is generally stored, only encrypted messages. These messages are then decrypted only on the devices of the specific user.
+
+It is recommended to set this number not too small. If a device is offline for longer than the number of days this option is set to, it will not receive all messages, which is generally a bad user experience. Note that it does no matter if any other device has received the messages: If a user has only a single device and is offline for more days than the retention period is set to, they will lose messages.
+
+On the other hand, storing too many messages on the server causes impacts on server performance and data hygiene in general. The default of seven is considered reasonable in the sense that most users won't be offline for longer than that.
+
+Changing this option to a lower value will delete messages from the server. Changing this option to a higher value will allow messages existing on the server to be retained for longer.
+
+### `SNIKKET_UPLOAD_STORAGE_GB`
+
+Use this option to place a limit on the amount of storage Snikket will use for files shared by users. You can use this to prevent your server's disk capacity being consumed if users upload many large files. By default there is no limit.
+
+If the limit is reached, users will be unable to upload new files until older files are cleared by Snikket after the configured retention period (or the limit is increased).
+
+Example:
+
+```
+# Allow no more than 1.5GB disk space to be used by uploaded files
+SNIKKET_UPLOAD_STORAGE_GB=1.5
+```
+
+The amount of file storage used is affected by the configured retention period (7 days by default) - i.e. longer retention periods will mean files are stored for longer, and more space will be used. Take this into account when choosing a value.
+
+### `SNIKKET_LOGLEVEL`
+
+Control the detail level of the log output of the snikket server.
+
+Valid options are `error`, `warn`, `info` (the default) and `debug`. The `debug` log level is very detailed. It may quickly fill your disk and also contain more sensitive information.
+
+### `SNIKKET_SITE_NAME`
+
+A human-friendly name for your server. Defaults to the value of `SNIKKET_DOMAIN`.
+
+### `SNIKKET_UPDATE_CHECK`
+
+By default, Snikket sends anonymous requests for the latest release via DNS, to provide you with a notification when a new release is available (which may contain important security fixes). This behaviour can be disabled by setting the option to `0`.
+
+This will not expose your server's IP address or domain name to the Snikket org, as it will generally be proxied through your or your hosters Internet Service Provider's DNS servers.
+
+### `SNIKKET_ADMIN_EMAIL`
+
+Email address of the admin. This will be sent to all new users as contact information.
+
+### `SNIKKET_WEB_AVATAR_CACHE_TTL`
+
+The time (in seconds) for which the web portal will allow avatars to be cached by browsers.
+
+
+## Arcane Configuration Reference
+
+**The options below this line are even more arcane than the options above. Do not touch unless you truly know what you're doing.**
+
+### `SNIKKET_TWEAK_INTERNAL_HTTP_PORT`
+
+The TCP port on which the internal HTTP API listens on. The default is `5280`. Do not change this without also changing `SNIKKET_WEB_PROSODY_ENDPOINT` accordingly.
+
+### `SNIKKET_TWEAK_INTERNAL_HTTP_INTERFACE`
+
+The IP address on which the internal HTTP API listens on. The default is `127.0.0.1`, so that the API is only accessible from the same server. Changing this may be a security risk as some general system information is accessible without authentication.
+
+### `SNIKKET_INVITE_URL`
+
+The URL template for invitation links. The server needs to know under which address the invitation service is hosted.
+
+Changing this will most likely break your invitation flow, so better don't.
+
+### `TWEAK_SNIKKET_BOOTSTRAP_INDEX`
+
+Just do not set this.
+
+### `TWEAK_SNIKKET_BOOTSTRAP_SECRET`
+
+Also better do not set this.
+
+### `SNIKKET_TWEAK_IPV6`
+
+Enable IPv6 support.
+
+By default, IPv6 is disabled because most container runtimes default to it being disabled. Enabling IPv6 in the server could cause issues if the container runtime does not support it.
+
+### `SNIKKET_TWEAK_PROMETHEUS`
+
+If you are monitoring your Snikket server using [Prometheus](https://prometheus.io/) and scraping the metrics endpoint, you should set this to `1` and let it at its default otherwise.
+
+If this is set to `1` without Snikket server being scraped by Prometheus, the System Health panel in the web portal will not work correctly. If this is not set to `1` when Snikket is being scraped by Prometheus, the numbers seen by Prometheus may not be accurate at the time they are being sampled, as Snikket server will in that case sample data only every 60s, no matter how often or when you scrape.
+
+The default is safe for non-Prometheus setups.
+
+### `SNIKKET_TWEAK_TURNSERVER`
+
+By default, Snikket starts a STUN/TURN server. If this option is set to `0`, it will not do that. You will have to run your own STUN/TURN server and configure `SNIKKET_TWEAK_TURNSERVER_DOMAIN` and `SNIKKET_TWEAK_TURNSERVER_SECRET` accordingly.
+
+If `SNIKKET_TWEAK_TURNSERVER` is set to `0` and `SNIKKET_TWEAK_TURNSERVER_DOMAIN` is not set, no STUN/TURN server will be offered to your users. Terrible idea to do that, will break audio/video calls in all but the most ideal situations.
+
+### `SNIKKET_TWEAK_TURNSERVER_DOMAIN`
+
+Hostname of the STUN/TURN server to use.
+
+Defaults to the Snikket domain, as snikket-server runs contains its own STUN/TURN server.
+
+### `SNIKKET_TWEAK_TURNSERVER_SECRET`
+
+Shared secret to use with the STUN/TURN server for authentication of clients.
+
+Defaults to a secret which is generated once at first installation. Only override this if you also set `SNIKKET_TWEAK_TURNSERVER` to `0` and set `SNIKKET_TWEAK_TURNSERVER_DOMAIN` to a STUN/TURN server you operate manually.
+
+### `SNIKKET_TWEAK_SHARE_DOMAIN`
+
+Expose the file share service at the `SNIKKET_DOMAIN` instead of at `share.SNIKKET_DOMAIN`.
+
+This nowadays conflicts with the web portal, so you should not set it.
+
+### `SNIKKET_TWEAK_EXTRA_CONFIG`
+
+Path or glob for extra configuration files to load.

docs/advanced/firewall.md

@@ -4,6 +4,80 @@
 
 Snikket currently requires the following ports to be open/forwarded:
 
-- **TCP only:** 80, 443, 5222, 5269, 5000
-- **TCP and UDP:** 3478, 3479, 5349, 5350
-- **UDP only:** 49152-65535
+
+ 
+  
+ |**TCP only**   |                                                                                    |
+ | :------------ | :--------------------------------------------------------------------------------- |
+ | 80/443        | Web Interface And Group File Sharing Service (HTTP(S))                             |                                                                                                 
+ | 5222          | Client App Connections (Client to Server) (XMPP-c2s)                               |               
+ | 5269          | Federation With Other Snikket Servers (Server to Server) (XMPP-s2s)                |                                                                                                      
+ | 5000          | File Transfer Proxy (proxy65)                                                      |
+  
+ 
+ |**TCP and UDP**|                                                                                    |
+ | :-------------| :--------------------------------------------------------------------------------- |
+ | 3478/3479     | Audio/Video Data Proxy Negotiation and IP discovery <br /> (STUN/TURN)                    |
+ | 5349/5350     | Audio/Video Data Proxy Negotiations and IP Discovery over TLS <br /> (STUN/TURN over TLS) |
+
+
+ |**UDP only**  |                                                                                    |
+ | :----------- | :----------------------------------------------------------------------------------|
+ | 49152-65535  | Audio/Video Data Proxy (Turn Data, see below)                                      |
+
+
+## Changing the turnserver port range
+
+The STUN/TURN server is required for audio/video (A/V) calls to work reliably on all kinds of "difficult" client networks. For this, a relay connection is established which routes the (encrypted) A/V data via your Snikket server. As generally the number of concurrent calls is not known and it needs to compete with ports already in use on the machine, the TURN server defaults to a range with a high number of ports (about 16 thousand). See below for recommendations on picking a smaller number of ports.
+
+However, some appliances will not allow forwarding a large range of UDP ports as normally required for TURN. If you have to forward ports through such an appliance, you can tweak the port range used by the STUN/TURN server using the following two configuration options:
+
+* `SNIKKET_TWEAK_TURNSERVER_MIN_PORT`: Set the lower bound of the port range (default: 49152)
+* `SNIKKET_TWEAK_TURNSERVER_MAX_PORT`: Set the upper bound of the port range (default: 65535)
+
+Both numbers must be larger than 1024 and smaller than or equal to 65535. Keeping them above 40000 is generally recommended for network standards reasons. Obviously, the min number must be less than or equal to the max number.
+
+Example for a range of 1024 ports (in your snikket.conf):
+
+```
+SNIKKET_TWEAK_TURNSERVER_MIN_PORT=60000
+SNIKKET_TWEAK_TURNSERVER_MAX_PORT=61023
+```
+
+Make sure to restart the `snikket` container after changing this option and ideally test A/V calls with two phones on different mobile data providers (those are generally most tricky to get working).
+
+### How many ports does the TURN service need?
+
+In general, you can safely assume that a call will never need more than four ports at the same time. That means that with 200 ports, you could in theory initiate up to 50 concurrent calls on your Snikket instance.
+
+However, these ports are a system-wide resource. A port may only be used by a single application at the same time (this is an oversimplification). That means that if your server machine is "rather busy", "many" of the ports in the range you designate for the TURN service may be in use already by other applications. This in turn means that a call may randomly fail to establish based on whether enough ports are available in the range you chose.
+
+Unless you are running an *extremely* busy service on your server, you should be fine if you plan wih 10% headroom. <!-- I checked how many "high ports" (5 digits) were open on the search.jabber.network xmppd at a random point in time, and they were just 800. Given that the high port range has 50k ports and that most users are not going to run a busy service as that, it should be fine. -->
+
+That means that if you have 20 users and want to allow them to start calls at the same time (ignoring *who* they'd call), you should plan for 80 ports, plus 10% head room, gives you about 90 ports.
+
+## Configuring UFW to Allow Ports for Snikket
+
+[UFW](https://wiki.ubuntu.com/UncomplicatedFirewall), the Uncomplicated Firewall, is a user-friendly interface to the more complicated iptables commands that control a Linux systems's firewall. 
+
+It is possible to manually add each of the above ports with `ufw` commands like the following: `# ufw allow 5000/tcp comment 'File Transfer Proxy (proxy65)'`, however, doing so is tedious and clutters the output of `# ufw status`. A better way is to create a custom ufw application, which we will call "Snikket" and have ufw add rules for that application. This is not only easier and declarative but also has the advantage of yielding a clean `# ufw status` report that looks as follows:
+
+```
+To       Action    From 
+--       ------    ----
+Snikket  ALLOW     Anywhere
+```
+
+Create the following file at `/etc/ufw/applications.d/ufw-snikket`. I have opted to open UDP ports 6000-6200 in the following example, but you should change this to reflect which TURN ports your Snikket configuration specifies.
+
+```
+[Snikket]
+title=Snikket Server
+description=Simple XMPP Server
+ports=80/tcp|443/tcp|5222/tcp|5269/tcp|5000/tcp|3478|3479|5349|5350|6000:6200/udp
+```
+
+Add the new rule:
+`# ufw allow snikket`
+
+Running `# ufw status` should now show Snikket as a rule. If you want to see all the specific ports that have been allowed by adding this rule you can run `# ufw status verbose`.

docs/advanced/reverse_proxy.md

@@ -20,7 +20,7 @@ need to instruct it to forward Snikket traffic to Snikket.
 It is important to get certificates correct when deploying Snikket behind a reverse
 proxy. Snikket needs to obtain certificates from Let's Encrypt in order to secure
 the non-HTTP services it provides. Be careful that your reverse proxy does not
-requests from Let's Encrypt that are intended for the Snikket service.
+intercept requests from Let's Encrypt that are intended for the Snikket service.
 
 # Configuration
 
@@ -56,6 +56,21 @@ server {
   listen 80;
   listen [::]:80;
 
+  server_name chat.example.com;
+  server_name groups.chat.example.com;
+  server_name share.chat.example.com;
+
+  location / {
+      proxy_pass http://localhost:5080/;
+      proxy_set_header      Host              $host;
+      proxy_set_header      X-Forwarded-For   $proxy_add_x_forwarded_for;
+
+      # A bit of headroom over the 16MB accepted by Prosody.
+      client_max_body_size 20M;
+  }
+}
+
+server {
   # Accept HTTPS connections
   listen [::]:443 ssl ipv6only=on;
   listen 443 ssl;
@@ -67,13 +82,25 @@ server {
   server_name share.chat.example.com;
 
   location / {
-      proxy_pass http://localhost:5080/;
-      proxy_set_header  Host            $host;
-      proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_pass https://localhost:5443/;
+      proxy_set_header      Host              $host;
+      proxy_set_header      X-Forwarded-For   $proxy_add_x_forwarded_for;
+      # REMOVE THIS IF YOU CHANGE `localhost` TO ANYTHING ELSE ABOVE
+      proxy_ssl_verify      off;
+      proxy_set_header      X-Forwarded-Proto https;
+      proxy_ssl_server_name on;
+
+      # A bit of headroom over the 16MB accepted by Prosody.
+      client_max_body_size 20M;
   }
 }
 ```
 
+**Note:** You may modify the first server block to include a redirect to HTTPS
+instead of proxying plain-text HTTP traffic. When doing that, take care to
+proxy `.well-known/acme-challenge` even in plain text to allow Snikket to
+obtain certificates.
+
 ### sslh
 
 sslh is a little different to the other servers listed here, as it is not a web server. However it is able
@@ -112,13 +139,13 @@ protocols:
 
 ### apache
 
-**Note**: The following configuration is for reverse proxying from another machine 
+**Note**: The following configuration is for reverse proxying from another machine
 (other from the one hosting Snikket containers). A prerequisite is a mechanism to sync
 Snikket-managed letsencrypt TLS key and cert to `/opt/chat/letsencrypt`. This is required because
 Apache 2.4 is not able to revproxying based on SNI, routing encrypted TLS directly to the Snikket machine.
 If the containers are on the same machine
 of the reverse proxy, you have to tweak HTTP/S ports as indicated before, and you don't need
-to proxy over SSL. 
+to proxy over SSL.
 
 ```
         <VirtualHost *:443>
@@ -135,7 +162,7 @@ to proxy over SSL.
                 CustomLog ${APACHE_LOG_DIR}/chat.example.com-ssl_access.log combined
 
                 SSLEngine on
-		
+
 		#
                 SSLCertificateFile /opt/chat/letsencrypt/chat.example.com/cert.pem
                 SSLCertificateKeyFile /opt/chat/letsencrypt/chat.example.com/privkey.pem
@@ -156,14 +183,14 @@ to proxy over SSL.
         	ServerAlias share.chat.example.com
 
         	ServerAdmin webmaster@localhost
-        
+
 		DocumentRoot /var/www/chat
 
         	ProxyPreserveHost On
 
         	ProxyPass           / http://chat.example.com/
         	ProxyPassReverse    / http://chat.example.com/
-        
+
         	ErrorLog ${APACHE_LOG_DIR}/chat.example.com_error.log
         	CustomLog ${APACHE_LOG_DIR}/chat.example.com_access.log combined
 

docs/features/user_roles.md

@@ -0,0 +1,53 @@
+---
+title: User roles
+---
+
+# User roles
+
+Snikket allows you to select a role for users, each role granting different
+permissions.
+
+Each user may have one of three roles:
+
+## Administrator
+
+This is the default role of the first user (if you're reading this, that's
+probably you!).
+
+Administrators have full control over the server, settings, users and circles.
+These features can be accessed primarily through the admin panel in the
+Snikket web interface.
+
+## Normal
+
+This is the default role for most users. It gives access to all
+non-administrative server functionality.
+
+## Limited
+
+Limited users have various restrictions. The purpose of this role is to
+allow granting someone an account on the server, only for the purposes of
+communicating with other people on that server. This can be useful to provide
+a guest or child account, for example.
+
+In particular, limited users are not allowed to:
+
+- Communicate with users on other servers
+- Join group chats on other servers
+- Create public channels (including on the current server)
+- Invite new users to the server (regardless of whether this is enabled for
+  normal users).
+
+### Caveats
+
+The current support for limited users has some known issues. It is designed to
+prevent casual misuse of the server, but it is not intended to be a foolproof
+security measure. For example, limited users are still able to *receive*
+messages and contact requests from other servers, even though they cannot send
+them to other servers. It is expected that we will restrict incoming traffic
+for limited users in a future release, after further testing.
+
+Also note that limited accounts may have issues using non-Snikket mobile apps
+that use push notifications, depending on the design of the app. This is
+because the restrictions may prevent the app communicating with its'
+developer's push notification services over XMPP.

docs/setup/quickstart.md

@@ -31,10 +31,10 @@ For the server, you can use a VPS from a provider such as [DigitalOcean](https:/
 or you can use a physical device such as a Raspberry Pi. Note that if you run your server at home (which is _really_ cool!) you may need to forward some ports on your
 router.
 
-{{< infobox warning >}}
-**Important:** Snikket provides a built-in web server that must be accessible on port 80. Therefore this guide assumes you are _not_ running any existing
-websites on the same server. We are working to remove this requirement in a future version.
-{{< /infobox >}}
+!!! warning
+
+    **Important:** Snikket provides a built-in web server that must be accessible on port 80. Therefore this guide assumes you are _not_ running any existing
+    websites on the same server. We are working to remove this requirement in a future version.
 
 ## Get Started
 
@@ -172,6 +172,4 @@ Follow the link to open the invitation, and follow the instructions get signed i
 You can create as many links as you want and share them with people. Each link can
 only be used once. Don't forget to drop the `--admin` part to create normal user accounts!
 
-{{< infobox primary >}}
 That's it! How did it go? Let us know at feedback@snikket.org
-{{< /infobox >}}

docs/setup/troubleshooting.md

@@ -0,0 +1,147 @@
+---
+title: "Troubleshooting"
+---
+
+# Self-hosted Snikket troubleshooting
+
+Problems with your Snikket setup? Don't worry! Most people don't experience
+any issues, but if you do, it's likely something simple. This page describes
+problems you might encounter, and how to solve them.
+
+## General problems
+
+### "Snikket is starting" page does not go away
+
+If this page stays for more than a few minutes, there was probably an
+issue obtaining certificates for your Snikket service. For more
+information on diagnosing certificate issues, see the
+['Certificates' section](#certificates) later on this page.
+
+### Unable to share large files
+
+If you find that your users cannot share large files through Snikket,
+there could be a couple of reasons:
+
+- If you are using Snikket behind a reverse proxy, ensure that the proxy
+  does not place a limit on the size of uploads. Check our [reverse proxy
+  guide](../../advanced/reverse_proxy/) for more information.
+- If the file is over 100MB, Snikket will attempt a direct device-to-device
+  transfer. This requires you and your recipient to be online at the
+  same time, and it only works between two users (not in groups). Also
+  note that direct transfers are not currently supported to or from iOS
+  devices.
+- To share files over 100MB with a Snikket group or iOS users, we
+  recommend a dedicated file transfer service. You can find a list of
+  standalone [self-hosted file transfer services](https://github.com/awesome-selfhosted/awesome-selfhosted#file-transfer---single-click--drag-n-drop-upload), use a system
+  such as NextCloud, or select one of the many free online file transfer
+  services.
+
+### Invitations are always expired
+
+If all invitation links show as expired immediately after you create them:
+
+- Check you copied the entire URL correctly.
+- Ensure that you don't have an XMPP server or other service running on
+  the same system as Snikket using port 5280.
+- If you use a reverse proxy, check that it is correctly forwarding
+  requests to Snikket. See our [reverse proxy guide](../../advanced/reverse_proxy/)
+  for more info.
+
+### Not responsible for this domain
+
+If you see an error in the app reporting that the server is "not
+responsible for this domain":
+
+- Check that you do not have another XMPP server running on the same
+  system as Snikket. It may be using the ports that Snikket needs.
+- Check that your DNS setup is correct, and you do not have SRV records
+  left over from a previous XMPP installation on the same domain. If you
+  recently modified your DNS records, you may need to wait a while for
+  DNS caches to expire the old records.
+
+## Certificate problems
+
+Certificates are an important part of securing connections to your
+Snikket.
+
+Snikket automatically obtains certificates from Let's Encrypt, and keeps
+them up to date. This usually works without problems, but it can be
+sensitive to a number of things that might cause it to fail.
+
+### Common causes
+
+Common causes of an inability to obtain or renew certificates:
+
+#### Missing or incorrect DNS records
+
+Snikket needs 3 DNS records to be added. Ensure you followed the steps
+from the installation guide correctly, particularly the
+[DNS configuration](https://snikket.org/service/quickstart/#step-1-dns).
+
+If your server supports IPv6, you may also add that to DNS (using an
+AAAA record). If you do this, you *must* tell Snikket by adding the
+following line to your snikket.conf:
+
+```
+SNIKKET_TWEAK_IPV6=1
+```
+
+#### Port 80 blocked
+
+Ensure that port 80 is open and accessible. You can review a [list of
+ports required by Snikket](../../advanced/firewall/). Port 80 is required
+to be open by Let's Encrypt so they can verify your domain.
+
+On a VPS or in a cloud environment, your provider may require you to
+manually open ports, e.g. in their web dashboard. If you are running in
+a LAN, you may need to forward ports in your router's web interface.
+
+Finally, check the firewall on the server itself (e.g. ufw, iptables or
+nftables).
+
+#### Incorrect reverse proxy configuration
+
+If you have a reverse proxy set up (e.g. to run Snikket on the same server
+as other websites or services), it needs to correctly forward requests
+to Snikket on both http and https.
+
+See our [Snikket reverse proxy documentation](../../advanced/reverse_proxy/)
+for more information on correctly configuring reverse proxies.
+
+### Certificate debugging commands
+
+#### Checking for errors
+
+If you think you have everything set up correctly and you're not sure what the
+problem could be, check the error log:
+
+```
+docker-compose exec snikket_certs cat /var/log/letsencrypt/errors.log
+```
+
+If you get a "No such file or directory" error when running the above command,
+inspect the debug log instead:
+
+```
+docker-compose exec snikket_certs cat /var/log/letsencrypt/letsencrypt.log | grep detail
+```
+
+#### Trying again
+
+Once you have fixed any problems, you can force a new attempt with the
+following command:
+
+```
+docker-compose exec snikket_certs /etc/cron.daily/certbot
+```
+
+If that command says that no certificates are due for renewal, but you need to
+trigger a renewal anyway, run:
+
+```
+docker-compose exec snikket_certs su letsencrypt -- -c "certbot renew --config-dir /snikket/letsencrypt --cert-path /etc/ssl/certbot --force-renew"
+```
+
+Note that Let's Encrypt has strict [rate limits](https://letsencrypt.org/docs/rate-limits/) -
+do not run these commands more often than necessary, or you may find yourself
+unable to get new certificates for a while.

mkdocs.yml

@@ -8,6 +8,7 @@ nav:
   - index.md
   - Setup:
     - setup/quickstart.md
+    - setup/troubleshooting.md
   - Advanced:
     - advanced/dns.md
     - advanced/firewall.md

snikket-modules/mod_snikket_client_id/mod_snikket_client_id.lua

@@ -0,0 +1,13 @@
+-- This module assigns a client_id to sessions if they are using a "Snikket.*"
+-- resource identifier. We assume that a resource string in this format is
+-- static for the same client instance across every session.
+--
+-- In the future it is anticipated that this "hack" will be replaced by SASL 2
+-- (XEP-0388) and/or Bind 2 (XEP-0386), however this is not yet implemented in
+-- Prosody or any clients.
+
+module:hook("resource-bind", function (event)
+	local id = event.session.resource:match("^Snikket%..+$");
+	if not id then return; end
+	event.session.client_id = id;
+end, 1000);

snikket-modules/mod_snikket_ios_preserve_push/mod_snikket_ios_preserve_push.lua

@@ -0,0 +1,23 @@
+-- The Snikket iOS client does not perform a push registration ("enable") on
+-- every new connection (it connects every time the app is opened, so we want
+-- to reduce round-trips and latency). This module attempts to locate a push
+-- registration associated with the connecting client, and load it onto the
+-- session so that mod_cloud_notify can find it.
+
+local push_store = module:open_store("cloud_notify");
+
+module:hook("resource-bind", function (event)
+	local session = event.session;
+	local client_id = session.client_id;
+	if not client_id then return; end
+	local push_registrations = push_store:get(session.username);
+	if not push_registrations then return; end
+	for push_identifier, push_registration in pairs(push_registrations) do
+		if push_registration.client_id == client_id then
+			session.push_identifier = push_identifier;
+			session.push_settings = push_registration;
+			module:log("debug", "Restored push registration for %s (%s)", client_id, push_identifier);
+			break;
+		end
+	end
+end, 10);

snikket-modules/mod_snikket_restricted_users/mod_snikket_restricted_users.lua

@@ -0,0 +1,55 @@
+local jid_bare = require "util.jid".bare;
+local um_get_roles = require "core.usermanager".get_roles;
+
+local function load_main_host(module)
+	-- Check whether a user should be isolated from remote JIDs
+	-- If not, set a session flag that allows them to bypass mod_isolate_host
+	local function check_user_isolated(event)
+		local session = event.session;
+		if not session.no_host_isolation then
+			local bare_jid = jid_bare(session.full_jid);
+			local roles = um_get_roles(bare_jid, module.host);
+			if roles == false then return; end
+			if not roles or not roles["prosody:restricted"] then
+				-- Bypass isolation for all unrestricted users
+				session.no_host_isolation = true;
+			end
+		end
+	end
+
+	-- Add low-priority hook to run after the check_user_isolated default
+	-- behaviour in mod_isolate_host
+	module:hook("resource-bind", check_user_isolated, -0.5);
+end
+
+local function load_groups_host(module)
+	local primary_host = module.host:gsub("^%a+%.", "");
+
+	local function is_restricted(user_jid)
+		local roles = um_get_roles(user_jid, primary_host);
+		return not roles or roles["prosody:restricted"];
+	end
+
+	module:hook("muc-config-submitted/muc#roomconfig_publicroom", function (event)
+		if not is_restricted(event.actor) then return; end
+		-- Don't allow modification of this value by restricted users
+		return true;
+	end, 5);
+
+	module:hook("muc-config-form", function (event)
+		if not is_restricted(event.actor) then return; end -- Don't restrict admins
+		-- Hide the option from the config form for restricted users
+		local form = event.form;
+		for i = #form, 1, -1 do
+			if form[i].name == "muc#roomconfig_publicroom" then
+				table.remove(form, i);
+			end
+		end
+	end);
+end
+
+if module:get_host_type() == "component" and module:get_option_string("component_module") == "muc" then
+	load_groups_host(module);
+else
+	load_main_host(module);
+end

snikket-modules/mod_update_check/mod_update_check.lua

@@ -12,10 +12,11 @@ local check_interval = module:get_option_number("update_check_interval", 86400);
 local version_info = {};
 
 do
-	local version = prosody.version;
-	local branch, bugfix = version:match("(%S+)%.(%d+)$");
-	if branch then
-		version_info.branch, version_info.level = branch, bugfix;
+	local version_string = prosody.version;
+	-- "dev 128-00000", "release v2021.05r2"
+	local series, version = version_string:match("(%w+) (%S+)$");
+	if series then
+		version_info.branch, version_info.level = series, version;
 	end
 end