# robots.txt for https://avassa.io/
# Last updated: 2025-10-21
# Purpose: Optimize crawl efficiency, prevent duplicate feeds/search URLs,
# and enable visibility in AI search and generative systems.

User-agent: *
Disallow: /search/*
Allow: /search/
Disallow: /feed/
Disallow: /rss2/
Disallow: /wp-json/wp/v2/posts*
Disallow: /?s=
Allow: /

# Sitemap declaration
Sitemap: https://avassa.io/sitemap_index.xml

# --- AI and LLM Crawlers ---
# Allow trusted AI crawlers for inclusion in AI search experiences.
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

# --- Crawlers to block ---
# Known scrapers or low-value bots
User-agent: Bytespider
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# End of file