# Robots.txt for Markaba News
# This file tells search engine crawlers which pages or files the crawler can or can't request from your site.

# Main site crawling rules
User-agent: *
Allow: /

# Disallow access to admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /login
Disallow: /_next/
Disallow: /static/

# Disallow access to error pages
Disallow: /404
Disallow: /500

# Disallow access to search results with parameters to avoid duplicate content
Disallow: /search?*

# Sitemap locations
Sitemap: https://www.markaba.news/api/sitemap-index.xml

# Block malicious bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SemrushBot
Disallow: /