feat: serve default robots txt

.env.sample

@@ -174,4 +174,14 @@ COMPOSE_FILE="compose.yml"
 
 ## Nextcloud Talk HPB
 #COMPOSE_FILE="$COMPOSE_FILE:compose.nextcloud-talk-hpb.yml"
-#NEXTCLOUD_TALK_HPB_ENABLED=1
+#NEXTCLOUD_TALK_HPB_ENABLED=1
+
+## Anubis
+#COMPOSE_FILE="$COMPOSE_FILE:compose.anubis.yml"
+#ANUBIS_COOKIE_DOMAIN=example.com
+#ANUBIS_DOMAIN=anubis.example.com
+#ANUBIS_REDIRECT_DOMAINS=
+#ANUBIS_OG_PASSTHROUGH=true
+#ANUBIS_OG_EXPIRY_TIME=1h
+#ANUBIS_OG_CACHE_CONSIDER_HOST=true
+#ANUBIS_SERVE_ROBOTS_TXT=true

README.md

@@ -55,4 +55,17 @@ Letsencrypt DNS challenges.
      Access Token, in which case use compose.gandi-personal-access-token.yml.
 6. Redeploy Traefik, using e.g. `abra app deploy YOURAPPDOMAIN -f`
 
+## Blocking scrapers with [Anubis](https://anubis.techaro.lol/)
+
+Uncomment the lines on the Anubis section of the configuration.  Set
+a domain name for the cookies and a domain that will serve Anubis
+redirection service.  Optionally and for [added
+security](https://anubis.techaro.lol/docs/admin/configuration/redirect-domains),
+set a list of the domain names for the apps that are going to be
+protected.
+
+After deploying these changes, go to each recipe that supports Anubis
+and follow the process there.  **Enabling Anubis here is not enough for
+protection your apps.**
+
 [`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra

compose.anubis.yml

@@ -0,0 +1,29 @@
+---
+version: "3.8"
+services:
+  app:
+    deploy:
+      labels:
+      - "traefik.http.middlewares.anubis.forwardauth.address=http://anubis:8080/.within.website/x/cmd/anubis/api/check"
+  anubis:
+    image: "ghcr.io/techarohq/anubis:v1.24.0"
+    environment:
+      BIND: ":8080"
+      TARGET: " "
+      REDIRECT_DOMAINS: "${ANUBIS_REDIRECT_DOMAINS}"
+      COOKIE_DOMAIN: "${ANUBIS_COOKIE_DOMAIN}"
+      PUBLIC_URL: "https://${ANUBIS_DOMAIN}"
+      OG_PASSTHROUGH: "${ANUBIS_OG_PASSTHROUGH}"
+      OG_EXPIRY_TIME: "${ANUBIS_OG_EXPIRY_TIME}"
+      OG_CACHE_CONSIDER_HOST: "${ANUBIS_OG_CACHE_CONSIDER_HOST}"
+      SERVE_ROBOTS_TXT: "${ANUBIS_SERVE_ROBOTS_TXT}"
+    networks:
+    - proxy
+    deploy:
+      labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.anubis.rule=Host(`${ANUBIS_DOMAIN}`)"
+      - "traefik.http.routers.anubis.tls.certresolver=${LETS_ENCRYPT_ENV}"
+      - "traefik.http.routers.anubis.entrypoints=web-secure"
+      - "traefik.http.services.anubis.loadbalancer.server.port=8080"
+      - "traefik.http.routers.anubis.service=anubis"