feat: serve default robots txt

.env.sample

@@ -176,6 +176,12 @@ COMPOSE_FILE="compose.yml"
 #COMPOSE_FILE="$COMPOSE_FILE:compose.nextcloud-talk-hpb.yml"
 #NEXTCLOUD_TALK_HPB_ENABLED=1
 
-## Iocane
-#COMPOSE_FILE="$COMPOSE_FILE:compose.iocane.yml"
-#IOCANE_ENABLED=1
+## Anubis
+#COMPOSE_FILE="$COMPOSE_FILE:compose.anubis.yml"
+#ANUBIS_COOKIE_DOMAIN=example.com
+#ANUBIS_DOMAIN=anubis.example.com
+#ANUBIS_REDIRECT_DOMAINS=
+#ANUBIS_OG_PASSTHROUGH=true
+#ANUBIS_OG_EXPIRY_TIME=1h
+#ANUBIS_OG_CACHE_CONSIDER_HOST=true
+#ANUBIS_SERVE_ROBOTS_TXT=true

README.md

@@ -55,4 +55,17 @@ Letsencrypt DNS challenges.
      Access Token, in which case use compose.gandi-personal-access-token.yml.
 6. Redeploy Traefik, using e.g. `abra app deploy YOURAPPDOMAIN -f`
 
+## Blocking scrapers with [Anubis](https://anubis.techaro.lol/)
+
+Uncomment the lines on the Anubis section of the configuration.  Set
+a domain name for the cookies and a domain that will serve Anubis
+redirection service.  Optionally and for [added
+security](https://anubis.techaro.lol/docs/admin/configuration/redirect-domains),
+set a list of the domain names for the apps that are going to be
+protected.
+
+After deploying these changes, go to each recipe that supports Anubis
+and follow the process there.  **Enabling Anubis here is not enough for
+protection your apps.**
+
 [`abra`]: https://git.autonomic.zone/autonomic-cooperative/abra

compose.anubis.yml

@@ -0,0 +1,29 @@
+---
+version: "3.8"
+services:
+  app:
+    deploy:
+      labels:
+      - "traefik.http.middlewares.anubis.forwardauth.address=http://anubis:8080/.within.website/x/cmd/anubis/api/check"
+  anubis:
+    image: "ghcr.io/techarohq/anubis:v1.24.0"
+    environment:
+      BIND: ":8080"
+      TARGET: " "
+      REDIRECT_DOMAINS: "${ANUBIS_REDIRECT_DOMAINS}"
+      COOKIE_DOMAIN: "${ANUBIS_COOKIE_DOMAIN}"
+      PUBLIC_URL: "https://${ANUBIS_DOMAIN}"
+      OG_PASSTHROUGH: "${ANUBIS_OG_PASSTHROUGH}"
+      OG_EXPIRY_TIME: "${ANUBIS_OG_EXPIRY_TIME}"
+      OG_CACHE_CONSIDER_HOST: "${ANUBIS_OG_CACHE_CONSIDER_HOST}"
+      SERVE_ROBOTS_TXT: "${ANUBIS_SERVE_ROBOTS_TXT}"
+    networks:
+    - proxy
+    deploy:
+      labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.anubis.rule=Host(`${ANUBIS_DOMAIN}`)"
+      - "traefik.http.routers.anubis.tls.certresolver=${LETS_ENCRYPT_ENV}"
+      - "traefik.http.routers.anubis.entrypoints=web-secure"
+      - "traefik.http.services.anubis.loadbalancer.server.port=8080"
+      - "traefik.http.routers.anubis.service=anubis"

compose.iocane.yml

@@ -1,19 +0,0 @@
-version: "3.8"
-
-services:
-  app:
-    environment:
-      - IOCANE_ENABLED
-
-  iocane:
-    image: git.madhouse-project.org/iocaine/iocaine:latest
-    ports:
-      - '42069:42069'
-    environment:
-      - RUST_LOG=iocaine=warn
-    labels:
-      - 'traefik.http.middlewares.iocaine.plugin.iocaine.iocaineHttpUrl=http://iocaine:42069'
-      - 'traefik.http.middlewares.iocaine.plugin.iocaine.methods=GET,HEAD'
-    networks:
-      - internal
-    #TODO healthcheck for iocane?

traefik.yml.tmpl

@@ -42,10 +42,6 @@ entrypoints:
         allowEncodedPercent: true
         allowEncodedQuestionMark: true
         allowEncodedHash: true
-      {{ if eq (env "IOCANE_ENABLED") "1" }}
-      middlewares:
-        - iocane@file
-      {{ end }}
   {{ if eq (env "GITEA_SSH_ENABLED") "1" }}
   gitea-ssh:
     address: ":2222"
@@ -155,10 +151,3 @@ certificatesResolvers:
           - "1.1.1.1:53"
           - "9.9.9.9:53"
       {{ end }}
-
-{{ if eq (env "IOCANE_ENABLED") "1" }}
-experimental:
-  localPlugins:
-    iocaine:
-      moduleName: "git.mstar.dev/mstar/traefik-iocaine-middleware"
-{{ end }}