User-agent: *
Allow: /
Allow: /tests/auth/robots-accessible-login
Disallow: /tests/canonicals/blocked-target
Disallow: /intl/darwin/blocked/
Disallow: /tests/redirects/chains/robots-disallowed-final/blocked-final
Disallow: /tests/redirects/chains/robots-disallowed-middle/blocked-hop
Disallow: /tests/robots/conflicts/blocked-target

User-agent: Amazonbot
Disallow: /tests/ai-modern-crawlers/blocked/amazonbot

User-agent: ChatGPT-User
Disallow: /tests/ai-modern-crawlers/blocked/chatgpt-user

User-agent: Claude-Web
Disallow: /tests/ai-modern-crawlers/blocked/claude-web

User-agent: ClaudeBot
Disallow: /tests/ai-modern-crawlers/blocked/claudebot

User-agent: cohere-ai
Disallow: /tests/ai-modern-crawlers/blocked/cohere-ai

User-agent: Google-Extended
Disallow: /tests/ai-modern-crawlers/blocked/google-extended

User-agent: GPTBot
Disallow: /tests/ai-modern-crawlers/blocked/gptbot

User-agent: OAI-SearchBot
Disallow: /tests/ai-modern-crawlers/blocked/oai-searchbot

User-agent: PerplexityBot
Disallow: /tests/ai-modern-crawlers/blocked/perplexitybot

Sitemap: https://crawler-test-site.uk.sitebulb.com/sitemap.xml
Sitemap: https://crawler-test-site.uk.sitebulb.com/tests/international/cluster-c/sitemap.xml
Sitemap: https://crawler-test-site.uk.sitebulb.com/tests/sitemaps/fixtures/robots-referenced-404.xml
Sitemap: https://crawler-test-site.uk.sitebulb.com/tests/sitemaps/fixtures/valid-single-url.xml