User-agent: * Allow: / Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-content/cache/ Disallow: /wp-content/uploads/wc-logs/ Disallow: /search/ Disallow: /trackback/ Disallow: /comments/feed/ Disallow: /*/feed/ Disallow: /*/attachment/ Disallow: /tag/* Disallow: /category/* Disallow: /author-sitemap.xml Disallow: /project_tag-sitemap.xml Disallow: /tf_header_footer-sitemap.xml # ============================== # OPENAI (ChatGPT / GPT Models) # ============================== # GPT model training crawler User-agent: GPTBot Allow: / # User-driven browsing from ChatGPT User-agent: ChatGPT-User Allow: / # ChatGPT-User v2 User-agent: ChatGPT-User/2.0 Allow: / # ChatGPT Search indexing User-agent: OAI-SearchBot Allow: / # ChatGPT Operator agent User-agent: chatgpt-operator Allow: / # ============================== # ANTHROPIC (Claude) # ============================== # Main Claude training crawler User-agent: ClaudeBot Allow: / # Anthropic AI crawler User-agent: anthropic-ai Allow: / # Claude web browsing User-agent: Claude-Web Allow: / # Claude user browsing User-agent: claude-user Allow: / # ============================== # XAI (Grok - Elon Musk / X) # ============================== User-agent: GrokBot Allow: / User-agent: xAI-Grok Allow: / User-agent: Grok Allow: / # ============================== # GOOGLE AI (Gemini / Vertex AI) # ============================== # AI training for Gemini models User-agent: Google-Extended Allow: / # Gemini AI assistant crawler User-agent: GoogleGemini Allow: / # Google internal R&D crawler User-agent: GoogleOther Allow: / # Google Cloud Vertex AI User-agent: Google-CloudVertexBot Allow: / # Gemini Deep Research User-agent: Gemini-Deep-Research Allow: / # ============================== # MICROSOFT (Copilot / Bing AI) # ============================== User-agent: MicrosoftCopilot Allow: / User-agent: bingbot Allow: / # ============================== # PERPLEXITY AI # ============================== User-agent: PerplexityBot Allow: / # Perplexity user-triggered fetcher User-agent: Perplexity-User Allow: / # ============================== # META (Facebook / Instagram AI) # ============================== User-agent: FacebookBot Allow: / User-agent: meta-externalagent Allow: / # ============================== # APPLE (Siri / Spotlight / Apple Intelligence) # ============================== User-agent: Applebot Allow: / # Apple AI training opt-in User-agent: Applebot-Extended Allow: / # ============================== # BYTEDANCE / TIKTOK (Doubao) # ============================== User-agent: Bytespider Allow: / User-agent: TikTokSpider Allow: / # ============================== # DEEPSEEK # ============================== User-agent: DeepSeek Allow: / User-agent: deepseekbot Allow: / # ============================== # MISTRAL AI (Le Chat) # ============================== User-agent: MistralLeChat Allow: / User-agent: mistralai-user Allow: / # ============================== # COHERE AI # ============================== User-agent: cohere-ai Allow: / # ============================== # HUGGING FACE # ============================== User-agent: HuggingChat Allow: / # ============================== # YOU.COM # ============================== User-agent: YouBot Allow: / # ============================== # DUCKDUCKGO AI (DuckAssist) # ============================== User-agent: DuckAssistBot Allow: / # ============================== # QUORA (Poe) # ============================== User-agent: Poe Allow: / # ============================== # DIFFBOT (AI Data Extraction) # ============================== User-agent: Diffbot Allow: / # ============================== # OMGILI / WEBZIO (AI Training Data) # ============================== User-agent: omgili Allow: / User-agent: omgilibot Allow: / User-agent: webzio-extended Allow: / # ============================== # TIMPI (Decentralized AI Search) # ============================== User-agent: Timpibot Allow: / # ============================== # AI2 - ALLEN INSTITUTE # ============================== User-agent: ai2bot Allow: / # ============================== # KANGAROO LLM # ============================== User-agent: Kangaroo Bot Allow: / # ============================== # SENTIONE (Sentibot) # ============================== User-agent: Sentibot Allow: / # ============================== # ICC-CRAWLER (NICT Research) # ============================== User-agent: ICC-Crawler Allow: / # ============================== # DIGITALOCEAN AI # ============================== User-agent: digitaloceangenai-crawler Allow: / # ============================== # COMMON CRAWL (Open Dataset) # ============================== User-agent: CCBot Allow: / # ============================== # LINKEDIN (Microsoft) # ============================== User-agent: LinkedInBot Allow: / # ============================== # PINTEREST AI # ============================== User-agent: Pinterestbot Allow: / # ============================== # OTHER AI PLATFORMS # ============================== User-agent: Chatsonic Allow: / User-agent: Kimi Allow: / User-agent: GigaChat Allow: / User-agent: CharacterAI Allow: / User-agent: ZapierAgents Allow: / User-agent: Wonderchat Allow: / User-agent: BreezeAI Allow: / User-agent: XiaoIce Allow: / User-agent: EvoAI Allow: / User-agent: Replika Allow: / User-agent: Kuki Allow: / # ============================== # AI CHATBOT & AUTOMATION BOTS # ============================== User-agent: DriftChatbot Allow: / User-agent: IntercomAI Allow: / User-agent: AdaSupport Allow: / User-agent: Tidio Allow: / User-agent: TidioBotPro Allow: / User-agent: ManyChat Allow: / User-agent: BotStar Allow: / User-agent: DialogflowCX Allow: / User-agent: Rasa Allow: / User-agent: IBMWatsonAssistant Allow: / User-agent: LivePersonAI Allow: / User-agent: Bold360AI Allow: / User-agent: ZendeskAnswerBot Allow: / User-agent: SalesforceEinstein Allow: / User-agent: ServiceNowVirtualAgent Allow: / User-agent: FreshchatAI Allow: / User-agent: TARSChatbot Allow: / User-agent: Chatfuel Allow: / User-agent: FlowXO Allow: / User-agent: ManyRequestsBot Allow: / User-agent: Landbot Allow: / User-agent: SnatchBot Allow: / User-agent: Pandorabots Allow: / User-agent: Botpress Allow: / User-agent: FlowAI Allow: / User-agent: OctaneAI Allow: / User-agent: MobileMonkey Allow: / User-agent: WhosOnAI Allow: / User-agent: HelpshiftBot Allow: / User-agent: 24-7AIBot Allow: / User-agent: LiveChatAI Allow: / User-agent: VerloopAI Allow: / User-agent: AivoAIChatbot Allow: / # ============================== # SITEMAPS & LLMS.TXT # ============================== Sitemap: https://nlkdconcretecoatings.com/sitemap_index.xml # LLMs: https://nlkdconcretecoatings.com/llms.txt