User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-json/wp/v2/users
Disallow: /search/
Disallow: /?s=
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/

# AI Crawlers Specific Allow
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: FacebookBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Bingbot
Allow: /

# Special AI and Research Crawlers
User-agent: CCBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Bytespider
Crawl-delay: 10

# Crawl Delay for Aggressive Bots
Crawl-delay: 5

# Sitemaps for All Search Engines
Sitemap: https://www.redgifs.it.com/sitemap_index.xml
Sitemap: https://www.redgifs.it.com/wp-sitemap.xml
Sitemap: https://www.redgifs.it.com/post-sitemap.xml
Sitemap: https://www.redgifs.it.com/page-sitemap.xml