---
weight: 100
---
Host: {{ site.hostname }}
Sitemap: {{ 'sitemap.xml' | absolute_url }}

{% if jekyll.env == 'production' %}
{% if site.data.robots.disallow_ai_crawlers %}
{{ site.data.robots.ai_robots_txt | fetch }}
{% endif %}
{% endif %}

User-Agent: *
{% assign collections = site.collections | where: 'output', true -%}
{%- for collection in collections -%}
  {%- assign docs = collection.docs | where: 'sitemap', false -%}
  {%- for doc in docs -%}
Disallow: {{ doc.url }}
{% endfor -%}
{%- endfor -%}
{%- for locale in site.locales -%}
  {%- if locale == site.locale -%}{%- continue -%}{%- endif -%}
  {%- assign collection = site.collections | find: 'label', locale -%}
  {%- assign docs = collection.docs | where: 'sitemap', false -%}
  {%- for doc in docs -%}
Disallow: {{ doc.url }}
{% endfor -%}
{%- endfor -%}
