diff --git a/docs/en/robots.txt b/docs/en/robots.txt new file mode 100644 index 00000000..7b943e3d --- /dev/null +++ b/docs/en/robots.txt @@ -0,0 +1,41 @@ +User-agent: GPTBot +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: Claude-Web +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: anthropic-ai +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Amazonbot +Allow: / + +User-agent: Bytespider +Allow: / + +User-agent: Applebot-Extended +Allow: / + +User-agent: * +Content-Signal: ai-train=yes, search=yes, ai-input=no +Allow: / + +Sitemap: https://gtfs.org/sitemap.xml