User-agent: *
Allow: /
Disallow: /admin/
Disallow: /login/
Disallow: /private/
Disallow: /test/
Disallow: /tmp/
Disallow: /backup/
Disallow: /cgi-bin/
Disallow: /user/
Disallow: /membership/profile/
Disallow: /api/
Disallow: /thanks
Disallow: /_framework/
Disallow: /_content/


# Sitemap
Sitemap: https://jayakutub.biz.id/sitemap.xml

# Google bots
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /uploads/
Allow: /gallery

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Prevent scraping
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /