# robots.txt for bisnapi.id
# Updated: 2026-05-26
# Content Signals (https://contentsignals.org/)
# search=yes: allow indexing for search results
# ai-train=no: disallow AI training on our content
# ai-input=no: disallow real-time AI grounding/RAG without permission
User-agent: *
Content-Signal: search=yes, ai-train=no, ai-input=no

# AI training crawlers — disallow
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /


# Allow all bots to crawl everything
User-agent: *
Allow: /

# Sitemap location
Sitemap: https://bisnapi.id/sitemap-index.xml

# Crawl-delay for polite bots (optional)
# Crawl-delay: 1

# Disallow error pages from indexing
Disallow: /400
Disallow: /404
Disallow: /500

# Allow comparison pages (important for SEO)
Allow: /vs/

# Allow all other pages
Allow: /about
Allow: /features
Allow: /pricing
Allow: /security
Allow: /hiring
Allow: /compliance
Allow: /produk/
Allow: /industri/
Allow: /legal/
Allow: /gratis/

# Allow new pages
Allow: /rakjat
Allow: /rakjat-kita
Allow: /why-black
Allow: /sustainable