# Allow OpenAI's GPT models (e.g. ChatGPT, GPT-4o) — used in enterprise procurement, integrations, summarization
User-Agent: gptbot
Allow: /

# Allow Google's Gemini (via Google-Extended) — used in Google AI Overviews, Workspace integrations
User-Agent: Google-Extended
Allow: /

# Allow Anthropic Claude (Sonnet/Haiku) — growing enterprise usage for AI summaries and safe content parsing
User-Agent: anthropic-ai
Allow: /

# Allow Meta's LLaMA (LLaMA 2/3/4) — top open-source model adopted by large organizations
User-Agent: meta-llama
Allow: /

# Allow Perplexity — AI-powered search engine increasingly used by IT and procurement managers
User-Agent: perplexitybot
Allow: /

# Allow Cohere — LLM provider focused on enterprise document embedding and retrieval
User-Agent: cohere-ai
Allow: /

# Allow AI21 Labs — known for enterprise use and structured text generation
User-Agent: ai21labs
Allow: /

# Allow IBM Granite — IBM's trusted enterprise-grade LLM, embedded into Watsonx for services/solutions
User-Agent: ibm-granite
Allow: /

# Allow Mistral — highly efficient open-source LLMs being adopted for hybrid infrastructure deployments
User-Agent: mistral
Allow: /

# Allow Hugging Face — serves Falcon, BLOOM, and many enterprise open LLMs
User-Agent: huggingface
Allow: /

# Allow Aleph Alpha — trusted in European enterprise AI deployments, good for multilingual contexts
User-Agent: aleph-alpha
Allow: /

# Allow Writer — used in enterprise product description generation and ecommerce copy
User-Agent: writer
Allow: /

# Allow xAI's Grok — growing influence due to live data integration (esp. in B2B social ecosystems)
User-Agent: xai-grok
Allow: /

# Allow You.com AI assistant — often used in product discovery and ecommerce comparisons
User-Agent: yousearch
Allow: /

# Allow Claude web-crawler — another Anthropic signal (variant)
User-Agent: claude-web
Allow: /

# Allow LlamaIndex — framework used to connect private datasets (e.g. product catalogs) to LLMs
User-Agent: llama-index
Allow: /

# Allow OpenRouter — serves multiple top LLMs like Mixtral, Claude, GPT via API gateway
User-Agent: openrouter
Allow: /

# ========== DISALLOWED BELOW ==========

# Block ModelScope — mostly experimental Alibaba research models with limited western enterprise adoption
User-Agent: modelscope
Disallow: /

# Block Semantic Kernel — Microsoft orchestration framework, not intended as a crawler
User-Agent: semantic-kernel
Disallow: /

# Block Stability AI — more focused on image/video generation than enterprise ecommerce LLM use
User-Agent: stabilityai
Disallow: /