feat: serve default robots txt

This implements the high performance backend for Nextcloud Talk, which is nessecary if it needs to handle more people (video) calling.
More Details about it: https://nextcloud-talk.readthedocs.io/en/latest/quick-install/

The current implementation is sadly limited to be used once per host, so this might need some additional love in the future, if someone needs it more flexible.

The related Nextcloud PR: coop-cloud/nextcloud#56

Co-authored-by: Apfelwurm <Alexander@volzit.de>
Reviewed-on: coop-cloud/traefik#66
Co-authored-by: p4u1 <p4u1_f4u1@riseup.net>
Co-committed-by: p4u1 <p4u1_f4u1@riseup.net>

.env.sample

@@ -170,4 +170,18 @@ COMPOSE_FILE="compose.yml"
 
 ## Garage
 #COMPOSE_FILE="$COMPOSE_FILE:compose.garage.yml"
-#GARAGE_RPC_ENABLED=1
+#GARAGE_RPC_ENABLED=1
+
+## Nextcloud Talk HPB
+#COMPOSE_FILE="$COMPOSE_FILE:compose.nextcloud-talk-hpb.yml"
+#NEXTCLOUD_TALK_HPB_ENABLED=1
+
+## Anubis
+#COMPOSE_FILE="$COMPOSE_FILE:compose.anubis.yml"
+#ANUBIS_COOKIE_DOMAIN=example.com
+#ANUBIS_DOMAIN=anubis.example.com
+#ANUBIS_REDIRECT_DOMAINS=
+#ANUBIS_OG_PASSTHROUGH=true
+#ANUBIS_OG_EXPIRY_TIME=1h
+#ANUBIS_OG_CACHE_CONSIDER_HOST=true
+#ANUBIS_SERVE_ROBOTS_TXT=true

README.md

@@ -55,4 +55,17 @@ Letsencrypt DNS challenges.
      Access Token, in which case use compose.gandi-personal-access-token.yml.
 6. Redeploy Traefik, using e.g. `abra app deploy YOURAPPDOMAIN -f`
 
+## Blocking scrapers with [Anubis](https://anubis.techaro.lol/)
+
+Uncomment the lines on the Anubis section of the configuration.  Set
+a domain name for the cookies and a domain that will serve Anubis
+redirection service.  Optionally and for [added
+security](https://anubis.techaro.lol/docs/admin/configuration/redirect-domains),
+set a list of the domain names for the apps that are going to be
+protected.
+
+After deploying these changes, go to each recipe that supports Anubis
+and follow the process there.  **Enabling Anubis here is not enough for
+protection your apps.**
+
 [`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra

abra.sh

@@ -1,3 +1,3 @@
-export TRAEFIK_YML_VERSION=v27
+export TRAEFIK_YML_VERSION=v28
 export FILE_PROVIDER_YML_VERSION=v10
 export ENTRYPOINT_VERSION=v5

compose.anubis.yml

@@ -0,0 +1,29 @@
+---
+version: "3.8"
+services:
+  app:
+    deploy:
+      labels:
+      - "traefik.http.middlewares.anubis.forwardauth.address=http://anubis:8080/.within.website/x/cmd/anubis/api/check"
+  anubis:
+    image: "ghcr.io/techarohq/anubis:v1.24.0"
+    environment:
+      BIND: ":8080"
+      TARGET: " "
+      REDIRECT_DOMAINS: "${ANUBIS_REDIRECT_DOMAINS}"
+      COOKIE_DOMAIN: "${ANUBIS_COOKIE_DOMAIN}"
+      PUBLIC_URL: "https://${ANUBIS_DOMAIN}"
+      OG_PASSTHROUGH: "${ANUBIS_OG_PASSTHROUGH}"
+      OG_EXPIRY_TIME: "${ANUBIS_OG_EXPIRY_TIME}"
+      OG_CACHE_CONSIDER_HOST: "${ANUBIS_OG_CACHE_CONSIDER_HOST}"
+      SERVE_ROBOTS_TXT: "${ANUBIS_SERVE_ROBOTS_TXT}"
+    networks:
+    - proxy
+    deploy:
+      labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.anubis.rule=Host(`${ANUBIS_DOMAIN}`)"
+      - "traefik.http.routers.anubis.tls.certresolver=${LETS_ENCRYPT_ENV}"
+      - "traefik.http.routers.anubis.entrypoints=web-secure"
+      - "traefik.http.services.anubis.loadbalancer.server.port=8080"
+      - "traefik.http.routers.anubis.service=anubis"

compose.nextcloud-talk-hpb.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+  app:
+    environment:
+      - NEXTCLOUD_TALK_HPB_ENABLED
+    ports:
+      - "3478:3478/udp"
+      - "3478:3478/tcp"

compose.yml

@@ -48,7 +48,7 @@ services:
         - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}"
         - "traefik.http.routers.${STACK_NAME}.service=api@internal"
         - "traefik.http.routers.${STACK_NAME}.middlewares=security@file"
-        - "coop-cloud.${STACK_NAME}.version=3.8.0+v3.6.5"
+        - "coop-cloud.${STACK_NAME}.version=3.9.0+v3.6.5"
         - "coop-cloud.${STACK_NAME}.timeout=${TIMEOUT:-120}"
         - "backupbot.backup=${ENABLE_BACKUPS:-true}"
 

traefik.yml.tmpl

@@ -105,6 +105,12 @@ entrypoints:
   matrix-federation:
     address: ":9001"
   {{ end }}
+  {{ if eq (env "NEXTCLOUD_TALK_HPB_ENABLED") "1" }}
+  nextcloud-talk-hpb:
+    address: ":3478"
+  nextcloud-talk-hpb-udp:
+    address: ":3478/udp"
+  {{ end }}
 
 ping:
   entryPoint: web