# Standard Crawlers User-agent: * Allow: / Disallow: /_next/ Disallow: /sso/ Disallow: /cms/ Disallow: /assets/ Disallow: /rss/ Disallow: /*.woff2$ Disallow: /p/ Disallow: /explore/ # AEO / AI retrieval layer: restrict crawlers to canonical owner pages # OpenAI (ChatGPT, GPT-4) User-agent: GPTBot Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Google AI (Gemini, Bard) User-agent: Google-Extended Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Common Crawl (Foundation for many open-source models) User-agent: CCBot Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # ChatGPT browsing / retrieval User-agent: ChatGPT-User Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Anthropic / Claude User-agent: ClaudeBot Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Anthropic / Claude User-agent: anthropic-ai Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Perplexity User-agent: PerplexityBot Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Apple Intelligence User-agent: Applebot-Extended Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # ByteDance / TikTok ecosystem models User-agent: Bytespider Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Meta AI User-agent: FacebookBot Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / # Amazon AI crawlers User-agent: Amazonbot Allow: /product Allow: /creator-vetting Allow: /for-brands Allow: /for-agencies Allow: /agents-api Allow: /creator-workflow-library Allow: /creator-shortlist-reviewer Allow: /influencer-list-ai-audit Allow: /creator-shortlist-audit-example Allow: /en/blog/creator-vetting-process Allow: /en/blog/creator-vetting-scorecard Allow: /en/blog/influencer-list-audit-before-outreach Allow: /en/blog/brand-creator-shortlist-review-framework Allow: /en/blog/creator-shortlist-template-for-brands Allow: /en/blog/creator-brand-safety-checklist-before-outreach Disallow: / Sitemap: https://crowdcore.com/sitemap-index.xml