Update CHANGELOG with new version number

Updated reverse_proxy.md Apace section.

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Snikket Server changelog
 
+## beta.20210519
+
+- Allow custom HTTP bind interface
+- Add docker health checks
+- Fix warnings about obsolete letsencrypt user
+- Add bootstrap API to create initiatal invite in an automated way
+- Switch to libunbound for DNS resolution (more robust)
+- Add environment variables to disable/replace the built-in TURN service
+
+## beta.20210205
+
+- Fix destruction of circle group chats when a circle
+    is deleted or fails to be created
+- Add circle group chats to bookmarks of newly-added members
+- Add trailing '/' to invite URLs for compatibility with some
+    URL parsers
+
 ## beta.20210202
 
 - Support for Raspberry Pi and other ARM-based systems
@@ -22,7 +39,7 @@ then put the new one in its place. For example:
 mv docker-compose.yml docker-compose.old.yml
 wget -O docker-compose.yml https://snikket.org/service/resources/docker-compose.beta.yml
 docker-compose pull
-docker-compose up -d
+docker-compose up -d --remove-orphans
 ```
 
 You may also want to check out our new repository of scripts to help

ansible/files/bin/start-coturn.sh

@@ -1,5 +1,10 @@
 #!/bin/sh
 
+if [ "$SNIKKET_TWEAK_TURNSERVER" = "0" ]; then
+	echo "TURN server disabled by environment, not launching.";
+	exit 0;
+fi
+
 CERTFILE="/snikket/letsencrypt/live/$SNIKKET_DOMAIN/fullchain.pem";
 KEYFILE="/snikket/letsencrypt/live/$SNIKKET_DOMAIN/privkey.pem";
 
@@ -11,7 +16,6 @@ done
 
 TURN_EXTERNAL_IP="$(snikket-turn-addresses "$SNIKKET_DOMAIN")"
 
-
 exec /usr/bin/turnserver -c /etc/turnserver.conf --prod \
      --static-auth-secret="$(cat /snikket/prosody/turn-auth-secret-v2)" \
      --cert="$CERTFILE" --pkey "$KEYFILE" -r "$SNIKKET_DOMAIN" \

ansible/files/prosody.cfg.lua

@@ -61,6 +61,7 @@ modules_enabled = {
 	-- HTTP modules
 		"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP"
 		"websocket"; -- XMPP over WebSockets
+		"http_host_status_check"; -- Health checks over HTTP
 
 	-- Other specific functionality
 		"limits"; -- Enable bandwidth limiting for XMPP connections
@@ -95,11 +96,14 @@ modules_enabled = {
 		"invites_register_api";
 		"invites_tracking";
 		"invites_default_group";
+		"invites_bootstrap";
 
 		"firewall";
 
 	-- Circles
+		"groups_internal";
 		"groups_migration";
+		"groups_muc_bookmarks";
 
 	-- For the web portal
 		"http_oauth2";
@@ -113,7 +117,7 @@ registration_notification = "New user registered: $username"
 reload_global_modules = { "http" }
 
 http_ports  = { ENV_SNIKKET_TWEAK_INTERNAL_HTTP_PORT or 5280 }
-http_interfaces = { "127.0.0.1" }
+http_interfaces = { ENV_SNIKKET_TWEAK_INTERNAL_HTTP_INTERFACE or "127.0.0.1" }
 
 https_ports = {};
 
@@ -128,9 +132,12 @@ registration_invite_only = true
 -- over what happens when a user invites someone.
 allow_contact_invites = false
 
-invites_page = ENV_SNIKKET_INVITE_URL or ("https://"..DOMAIN.."/invite/{invite.token}");
+invites_page = ENV_SNIKKET_INVITE_URL or ("https://"..DOMAIN.."/invite/{invite.token}/");
 invites_page_external = true
 
+invites_bootstrap_index = tonumber(ENV_TWEAK_SNIKKET_BOOTSTRAP_INDEX)
+invites_bootstrap_secret = ENV_TWEAK_SNIKKET_BOOTSTRAP_SECRET
+
 c2s_require_encryption = true
 s2s_require_encryption = true
 s2s_secure_auth = true
@@ -173,8 +180,10 @@ http_host = DOMAIN
 http_external_url = "https://"..DOMAIN.."/"
 http_max_content_size = 1024 * 1024 * 16 -- 16MB
 
-turncredentials_host = DOMAIN
-turncredentials_secret = assert(io.open("/snikket/prosody/turn-auth-secret-v2")):read("*l");
+if ENV_SNIKKET_TWEAK_TURNSERVER ~= "0" or ENV_SNIKKET_TWEAK_TURNSERVER_DOMAIN then
+	turncredentials_host = ENV_SNIKKET_TWEAK_TURNSERVER_DOMAIN or DOMAIN
+	turncredentials_secret = ENV_SNIKKET_TWEAK_TURNSERVER_SECRET or assert(io.open("/snikket/prosody/turn-auth-secret-v2")):read("*l");
+end
 
 VirtualHost (DOMAIN)
 	authentication = "internal_hashed"
@@ -232,4 +241,4 @@ Component ("share."..DOMAIN) "http_upload"
 	http_upload_file_size_limit = 1024 * 1024 * 16 -- 16MB
 	http_upload_expire_after = 60 * 60 * 24 * RETENTION_DAYS -- N days
 
-Include "/snikket/prosody/*.cfg.lua"
+Include (ENV_SNIKKET_TWEAK_EXTRA_CONFIG or "/snikket/prosody/*.cfg.lua")

ansible/files/supervisord.conf

@@ -27,7 +27,8 @@ umask=002
 [program:coturn]
 command=start-coturn.sh
 startsecs=0
-autorestart=true
+autorestart=unexpected
+exitcodes=0
 stopwaitsecs=30
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -41,4 +42,3 @@ stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 redirect_stderr=true
 umask=002
-

ansible/snikket.yml

@@ -4,6 +4,12 @@
 - hosts: all
   become: yes
   gather_facts: no
+  vars:
+    prosody:
+      package: "prosody-trunk"
+      build: "1436"
+    prosody_modules:
+      revision: "8e58a1b78336"
   tasks:
     - import_tasks: tasks/prosody.yml
     - import_tasks: tasks/supervisor.yml

ansible/tasks/prosody.yml

@@ -11,9 +11,12 @@
 - name: "Add Prosody package repo"
   apt_repository:
     repo: "deb https://packages.prosody.im/debian buster main"
+- name: "Detect dpkg architecture name"
+  shell: dpkg --print-architecture
+  register: dpkg_arch
 - name: "Install Prosody package"
   apt:
-    name: prosody-trunk
+    deb: "https://packages.prosody.im/debian/pool/main/p/{{ prosody.package }}/{{ prosody.package }}_1nightly{{ prosody.build }}-1~buster_{{ dpkg_arch.stdout }}.deb"
     state: present
     install_recommends: yes
 - name: "Deploy Prosody config"
@@ -64,10 +67,10 @@
   hg:
     repo: https://hg.prosody.im/prosody-modules
     dest: /usr/local/lib/prosody-modules
-    revision: 94805a7e7b30
+    revision: "{{ prosody_modules.revision }}"
     purge: yes
     update: yes
-- name: Enable wanted modules
+- name: Enable wanted modules (prosody-modules)
   file:
     state: link
     src: "/usr/local/lib/prosody-modules/{{item}}"
@@ -109,10 +112,12 @@
     - mod_invites_register_api
     - mod_invites_tracking
     - mod_groups_internal
+    - mod_groups_muc_bookmarks
     - mod_muc_defaults
     - mod_muc_local_only
+    - mod_http_host_status_check
 
-- name: Enable wanted modules
+- name: Enable wanted modules (snikket-modules)
   file:
     state: link
     src: "/usr/local/lib/snikket-modules/{{item}}"
@@ -121,9 +126,51 @@
     - mod_update_check
     - mod_update_notify
     - mod_invites_default_group
+    - mod_invites_bootstrap
 
 - name: "Install lua-ossl for encrypted push notifications"
   apt:
     name: lua-luaossl
     state: present
     install_recommends: no
+
+- name: "Fetch luaunbound source"
+  get_url:
+    url: https://code.zash.se/dl/luaunbound/luaunbound-0.5.tar.gz
+    sha256sum: a6564ac1cca6bb350576eb2a5cfa03adb0aafd4f99d6cd491bd8028d046c62a7
+    dest: /tmp/luaunbound-0.5.tar.gz
+
+- name: "Extract luaunbound"
+  unarchive:
+    src: /tmp/luaunbound-0.5.tar.gz
+    remote_src: yes
+    dest: /tmp
+
+- name: "Install libunbound-dev"
+  apt:
+    name:
+      - libunbound8
+      - libunbound-dev
+      - liblua5.2-dev
+    state: present
+
+- name: "Build luaunbound"
+  make:
+    chdir: /tmp/luaunbound-0.5
+
+- name: "Install luaunbound"
+  make:
+    chdir: /tmp/luaunbound-0.5
+    target: install
+
+- name: "Remove luaunbound source"
+  file:
+    path: /tmp/luaunbound-0.5
+    state: absent
+
+- name: "Remove libunbound-dev"
+  apt:
+    name:
+      - libunbound-dev
+      - liblua5.2-dev
+    state: absent

docker/Dockerfile

@@ -14,6 +14,8 @@ ADD docker/entrypoint.sh /bin/entrypoint.sh
 RUN chmod 770 /bin/entrypoint.sh
 ENTRYPOINT ["/bin/entrypoint.sh"]
 
+HEALTHCHECK CMD lua -l socket -e 'assert(socket.connect(os.getenv"SNIKKET_TWEAK_INTERNAL_HTTP_INTERFACE" or "127.0.0.1",os.getenv"SNIKKET_TWEAK_INTERNAL_HTTP_PORT" or "5280"))'
+
 ADD ansible /opt/ansible
 
 ADD snikket-modules /usr/local/lib/snikket-modules
@@ -23,7 +25,7 @@ RUN apt-get update \
         software-properties-common ca-certificates \
         gpg gpg-agent \
         ansible python-passlib python3-passlib \
-        libcap2-bin \
+        libcap2-bin build-essential\
     && rm -rf /var/lib/apt/lists/* \
     && c_rehash \
     && ansible-playbook -c local -i localhost, --extra-vars "ansible_python_interpreter=/usr/bin/python2" /opt/ansible/snikket.yml \
@@ -32,7 +34,7 @@ RUN apt-get update \
          software-properties-common \
          gpg gpg-agent \
          python-passlib python3-passlib \
-         mercurial libcap2-bin \
+         mercurial libcap2-bin build-essential \
     && apt-get autoremove -y \
     && rm -rf /var/cache/*
 

docker/entrypoint.sh

@@ -30,9 +30,6 @@ PGID=${PGID:=$(stat -c %g /snikket)}
 if [ "$PUID" != 0 ] && [ "$PGID" != 0 ]; then
 	usermod  -o -u "$PUID" prosody
 	groupmod -o -g "$PGID" prosody
-
-	usermod  -o -u "$PUID" letsencrypt
-	groupmod -o -g "$PGID" letsencrypt
 fi
 
 if ! test -d /snikket/prosody; then
@@ -41,16 +38,6 @@ fi
 
 chown -R prosody:prosody /var/spool/anacron /var/run/prosody /snikket/prosody /etc/prosody
 
-if ! test -d /snikket/letsencrypt; then
-	install -o letsencrypt -g letsencrypt -m 750 -d /snikket/letsencrypt;
-fi
-
-install -o letsencrypt -g letsencrypt -m 750 -d /var/lib/letsencrypt;
-install -o letsencrypt -g letsencrypt -m 750 -d /var/log/letsencrypt;
-install -o letsencrypt -g letsencrypt -m 755 -d /var/www/.well-known/acme-challenge;
-
-chown -R letsencrypt:letsencrypt /snikket/letsencrypt
-
 ## Generate secret for coturn auth if necessary
 if ! test -f /snikket/prosody/turn-auth-secret-v2; then
 	head -c 32 /dev/urandom | base64 > /snikket/prosody/turn-auth-secret-v2;

docs/advanced/reverse_proxy.md

@@ -109,3 +109,65 @@ protocols:
 );
 
 ```
+
+### apache
+
+**Note**: The following configuration is for reverse proxying from another machine 
+(other from the one hosting Snikket containers). A prerequisite is a mechanism to sync
+Snikket-managed letsencrypt TLS key and cert to `/opt/chat/letsencrypt`. This is required because
+Apache 2.4 is not able to revproxying based on SNI, routing encrypted TLS directly to the Snikket machine.
+If the containers are on the same machine
+of the reverse proxy, you have to tweak HTTP/S ports as indicated before, and you don't need
+to proxy over SSL. 
+
+```
+        <VirtualHost *:443>
+
+                ServerName chat.example.com
+                ServerAlias groups.chat.example.com
+                ServerAlias share.chat.example.com
+
+                ServerAdmin webmaster@localhost
+
+                DocumentRoot /var/www/chat
+
+                ErrorLog ${APACHE_LOG_DIR}/chat.example.com-ssl_error.log
+                CustomLog ${APACHE_LOG_DIR}/chat.example.com-ssl_access.log combined
+
+                SSLEngine on
+		
+		#
+                SSLCertificateFile /opt/chat/letsencrypt/chat.example.com/cert.pem
+                SSLCertificateKeyFile /opt/chat/letsencrypt/chat.example.com/privkey.pem
+                SSLCertificateChainFile /opt/chat/letsencrypt/chat.example.com/chain.pem
+
+                SSLProxyEngine On
+                ProxyPreserveHost On
+
+                ProxyPass           / https://chat.example.com/
+                ProxyPassReverse    / https://chat.example.com/
+
+        </VirtualHost>
+
+	<VirtualHost *:80>
+
+        	ServerName chat.example.com
+        	ServerAlias groups.chat.example.com
+        	ServerAlias share.chat.example.com
+
+        	ServerAdmin webmaster@localhost
+        
+		DocumentRoot /var/www/chat
+
+        	ProxyPreserveHost On
+
+        	ProxyPass           / http://chat.example.com/
+        	ProxyPassReverse    / http://chat.example.com/
+        
+        	ErrorLog ${APACHE_LOG_DIR}/chat.example.com_error.log
+        	CustomLog ${APACHE_LOG_DIR}/chat.example.com_access.log combined
+
+	</VirtualHost>
+
+```
+

docs/advanced/update_notifications.md

@@ -0,0 +1,101 @@
+# Update notifications
+
+This is an informational technical document about the update notification
+system in Snikket server.
+
+## Why are update notifications important?
+
+It is now widely known that [outdated software][OWASP-A9] is one of the
+biggest risk factors in securing systems on the internet. Therefore the
+Snikket server will alert all admins to available updates and important
+notices from the Snikket team.
+
+We believe it is up to you to decide when and how to update your service.
+But we will provide you with the tools you need to make that easy, fast
+and painless.
+
+## How are they implemented?
+
+To preserve your privacy, private Snikket servers do not make requests
+directly to our servers. Instead we put the necessary information about
+current releases and security updates into our DNS records.
+
+## Why did you choose DNS?
+
+The obvious choice was HTTP, and this is how most traffic on the internet
+is conveyed these days. But we opted for DNS due to the following advantages:
+
+- DNS is designed for serving small amounts of data from one place to many
+- Due to caching, and its connectionless nature, DNS is more scalable
+- Queries will often travel via an intermediate resolver, so we
+    typically won't have access to your server's IP address
+- A DNS query contains very little information, whereas HTTP will always
+    leak the IP address, and by default will often leak other headers.
+
+But it also has some known downsides. In particular DNS is not secure by
+default. Intermediaries may observe or drop the query, or even modify the
+response.
+
+The following conclusions were made about the downsides:
+
+- Observability: an intermediary seeing outbound queries to our DNS
+    records may deduce that your server is running Snikket. This should
+    not be a problem in itself - there are many ways to detect if a server
+    is running Snikket (load up its web page for a start!).
+- Availability: an intermediary may block queries for our DNS records.
+    This would prevent a server admin from receiving update notifications,
+    which is bad (they may be tricked into thinking they are up to date).
+    However using another protocol such as HTTP(S) would not prevent this
+    focused attack.
+- Integrity: the data returned to the Snikket server may be modified or
+    spoofed by an intermediary. This would allow them to trigger false
+    update notifications. We have designed the system so that the risk is
+    minimized - the update notifications will always include a link to the
+    real announcement on snikket.org (if any). It is not possible to direct
+    admins to arbitrary URLs.
+
+It is possible that in the future we will add support for DNSSEC or manually
+sign the data provided in our DNS records.
+
+It is also possible that we will move to another mechanism in the future, if
+a more suitable one can be found.
+
+## The details
+
+Snikket releases are organized into 'channels', e.g. 'dev', 'alpha', 'beta',
+'stable'. Your server will work out the channel it belongs to, and make a DNS
+query to:
+
+```
+TXT _channel.update.snikket.net
+```
+
+The response will look like:
+
+```
+"latest=3"
+"secure=2"
+"msg=0"
+```
+
+This response indicates that version '3' is the latest, but version '2' is the
+last release with no known security vulnerabilities is '2'. The `msg` field
+allows us to send important announcements that may not be included in a release.
+
+A Snikket server will use the returned information to determine whether the
+administrators need to be notified, and generate a message if necessary. Since
+the server has no further information, the message will include a link to the
+relevant announcement on the snikket.org website by calculating the URL to use.
+
+## Disabling update checks
+
+We strongly recommend you leave update notifications enabled so that you are
+notified promptly about important releases and announcements. However if you
+plan to receive these another way, you may disable them by adding to your
+snikket.conf:
+
+```
+SNIKKET_UPDATE_CHECK=0
+```
+
+[OWASP-A9]: https://owasp.org/www-project-top-ten/2017/A9_2017-Using_Components_with_Known_Vulnerabilities

docs/setup/upgrading.md

@@ -0,0 +1,26 @@
+---
+title: "Upgrading your Snikket server"
+date: 2021-05-19T14:32:02Z
+---
+
+Upgrading to a new Snikket release is typically very easy.
+
+## snikket-selfhosted
+
+If you installed Snikket using the [snikket-selfhosted][] scripts, simply run:
+
+    cd /opt/snikket
+    git pull
+    ./scripts/update.sh
+
+## Snikket quickstart
+
+If you're using a version installed from the [original quickstart][] guide on
+the website, use these commands instead:
+
+    cd /etc/snikket
+    docker-compose pull
+    docker-compose up -d
+
+[snikket-selfhosted]: https://github.com/snikket-im/snikket-selfhosted
+[original quickstart]: https://snikket.org/service/quickstart/

snikket-modules/mod_invites_bootstrap/mod_invites_bootstrap.lua

@@ -0,0 +1,66 @@
+--luacheck: ignore 143/module
+
+local http_formdecode = require "net.http".formdecode;
+
+local secret = module:get_option_string("invites_bootstrap_secret");
+if not secret then return; end
+
+local invites_bootstrap_store = module:open_store("invites_bootstrap");
+
+-- This should be a non-negative integer higher than any set for the
+-- previous bootstrap event (if any)
+local current_index = module:get_option_number("invites_bootstrap_index");
+
+local invites = module:depends("invites");
+module:depends("http");
+
+local function handle_request(event)
+	local query_params = http_formdecode(event.request.url.query);
+
+	if not query_params.token or query_params.token ~= secret then
+		return 403;
+	end
+
+	local bootstrap_records = invites_bootstrap_store:get() or {};
+	if #bootstrap_records > 0 then
+		local last_bootstrap = bootstrap_records[#bootstrap_records];
+		if current_index == last_bootstrap.index then
+			event.response.headers.Location = last_bootstrap.result;
+			return 303;
+		elseif current_index < last_bootstrap.index then
+			return 410;
+		end
+	end
+
+	-- Create invite
+	local invite, invite_err = invites.create_account(nil, {
+		roles = { ["prosody:admin"] = true };
+		groups = { "default" };
+		source = "api/token/bootstrap-"..current_index;
+	});
+	if not invite then
+		module:log("error", "Failed to create bootstrap invite! %s", invite_err);
+		return 500;
+	end
+
+	-- Record this bootstrap event (to prevent replay)
+	table.insert(bootstrap_records, {
+		index = current_index;
+		timestamp = os.time();
+		result = invite.landing_page or invite.uri;
+	});
+	local record_ok, record_err = invites_bootstrap_store:set(nil, bootstrap_records);
+	if not record_ok then
+		module:log("error", "Failed to store bootstrap record: %s", record_err);
+		return 500;
+	end
+
+	event.response.headers.Location = invite.landing_page or invite.uri;
+	return 303;
+end
+
+module:provides("http", {
+	route = {
+		GET = handle_request;
+	};
+});