# =========================================
# robots.txt for Obelisk Infotech
# Optimized for SEO, Content & Video Indexing (2025)
# =========================================

User-agent: *
# Allow full site crawling
Allow: /

# Block admin, system & duplicate pages
Disallow: /wp-admin/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /?s=
Disallow: /search/
Disallow: /tag/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /feed/
Disallow: /comments/
Disallow: /*?replytocom
Disallow: /*add-to-cart=*

# Allow JS, CSS, and Media files for rendering
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.mp4$

# Crawl-delay (optional: prevent server overload)
Crawl-delay: 5

# =========================================
# Sitemap Section
# Include all sitemaps for search engines
# =========================================

# Main XML Sitemap
Sitemap: https://www.obeliskinfotech.com/sitemap.xml

# Video Sitemap for Indexing Video Content
Sitemap: https://www.obeliskinfotech.com/video-sitemap.xml

# =========================================
# Specific Bot Rules
# =========================================

# Allow Google Image Bot (for image search visibility)
User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# Allow Google Mobile Bot (for mobile indexing)
User-agent: Googlebot-Mobile
Allow: /

# Allow Bing & Yahoo bots
User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: GPTBot
Allow: /

# Disallow spam bots & scrapers
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

# =========================================
# Notes:
# - Ensure your sitemap URLs are accessible
# - Validate robots.txt at: https://search.google.com/test/robots
# - Use Search Console → “robots.txt Tester”
# =========================================