# anonym.community robots.txt # https://anonym.community/robots.txt User-agent: * Allow: / Disallow: /.git/ Disallow: /api/ Disallow: /*.env Crawl-delay: 1 # --- Search Engine Crawlers --- User-agent: Googlebot Allow: / User-agent: bingbot Allow: / User-agent: YandexBot Allow: / User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: NaverBot Allow: / User-agent: SeznamBot Allow: / User-agent: Sogou Allow: / User-agent: Sogou web spider Allow: / User-agent: Exabot Allow: / User-agent: Slurp Allow: / User-agent: PetalBot Allow: / # --- Social Media Crawlers --- User-agent: FacebookBot Allow: / User-agent: LinkedInBot Allow: / User-agent: Twitterbot Allow: / User-agent: Slackbot Allow: / User-agent: WhatsApp Allow: / User-agent: Applebot Allow: / User-agent: MicrosoftPreview Allow: / # --- AI Crawlers (OpenAI) --- User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / # User-agent: Operator — deprecated July 2025, now uses standard Chrome UA # --- AI Crawlers (Anthropic) --- User-agent: ClaudeBot Allow: / User-agent: Claude-SearchBot Allow: / User-agent: Claude-User Allow: / # --- AI Crawlers (Google) --- User-agent: Google-Extended Allow: / User-agent: Google-CloudVertexBot Allow: / User-agent: Gemini-Deep-Research Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: Googlebot-News Allow: / User-agent: Storebot-Google Allow: / User-agent: Google-InspectionTool Allow: / User-agent: GoogleOther Allow: / User-agent: GoogleOther-Image Allow: / User-agent: GoogleOther-Video Allow: / # --- AI Crawlers (Meta) --- User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / # --- AI Crawlers (Apple) --- User-agent: Applebot-Extended Allow: / # --- AI Crawlers (xAI / Grok) --- User-agent: GrokBot Allow: / User-agent: xAI-Grok Allow: / User-agent: Grok-DeepSearch Allow: / # --- AI Crawlers (Amazon) --- User-agent: Amazonbot Allow: / User-agent: AmazonBuyForMe Allow: / # --- AI Crawlers (Perplexity) --- User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # --- AI Crawlers (ByteDance) --- User-agent: Bytespider Allow: / User-agent: TikTokSpider Allow: / # --- AI Crawlers (Other) --- User-agent: CCBot Allow: / User-agent: MistralAI-User Allow: / User-agent: DeepSeekBot Allow: / User-agent: DuckAssistBot Allow: / User-agent: AI2Bot Allow: / User-agent: Ai2Bot-Dolma Allow: / User-agent: Diffbot Allow: / User-agent: YouBot Allow: / User-agent: webzio-extended Allow: / User-agent: ImagesiftBot Allow: / User-agent: Timpibot Allow: / User-agent: Devin Allow: / User-agent: NovaAct Allow: / User-agent: img2dataset Allow: / User-agent: iAskSpider Allow: / User-agent: FirecrawlAgent Allow: / User-agent: cohere-ai Allow: / # --- SEO & Research Crawlers --- User-agent: SemrushBot Allow: / User-agent: AhrefsBot Allow: / User-agent: TurnitinBot Allow: / User-agent: MSIE Allow: / User-agent: ia_archiver Allow: / User-agent: archive.org_bot Allow: / # --- Discovery Files --- Sitemap: https://anonym.community/sitemap.xml # LLM content files: # https://anonym.community/llms.txt (structured index) # https://anonym.community/llms-full.txt (full content) # Security policy: # https://anonym.community/.well-known/security.txt # IndexNow key (instant indexing for Bing/Yandex/Naver): # https://anonym.community/47ec0f50a7c4a68ff71e96341450b9df.txt