# NextyHost - Production robots.txt (SEO-Optimized + Secure)
# Valid for: Google, Bing, Yandex, Baidu, DuckDuckGo, Naver, Sogou, all major crawlers
# Last updated: 2026-09-29

# ==========================================
# GLOBAL RULES - All compliant crawlers
# ==========================================
User-agent: *
Allow: /

# ---- Security: Hide internals, never index ----
Disallow: /app/
Disallow: /includes/
Disallow: /404.php
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /temp/
Disallow: /.git/
Disallow: /.svn/
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /yarn.lock
Disallow: /.env
Disallow: /config/
Disallow: /logs/
Disallow: /backup/
Disallow: /backups/
Disallow: /header
Disallow: /footer
Disallow: /stats
Disallow: /plan-card
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /*.log$

# Allow CSS/JS for rendering (Google/Bing need these for rendering)
Allow: /assets/css/*.css
Allow: /assets/js/*.js
Allow: /assets/images/
Allow: /assets/fonts/

# ---- Don't crawl utility/parameter URLs ----
Disallow: /*?utm_*
Disallow: /*?fbclid*
Disallow: /*?gclid*
Disallow: /*?ref=*
Disallow: /*?sort=*
Disallow: /*?page=*
Disallow: /*?currency=*
Disallow: /*?format=*

# WordPress/Common attack paths (not used, but block proactively)
Disallow: /wp-
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# ==========================================
# SPECIFIC BOT RULES (overrides global)
# ==========================================

# ---- Google (primary) ----
User-agent: Googlebot
Allow: /
Allow: /assets/images/
Allow: /assets/css/
Allow: /assets/js/
Allow: /assets/fonts/
Crawl-delay: 0

# ---- Bing / MSN ----
User-agent: Bingbot
Allow: /
Allow: /assets/images/
Allow: /assets/css/
Allow: /assets/js/
Crawl-delay: 10

# ---- Yandex ----
User-agent: Yandex
Allow: /
Allow: /assets/images/
Crawl-delay: 10

# ---- Baidu ----
User-agent: Baiduspider
Allow: /
Crawl-delay: 10

# ---- DuckDuckGo ----
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 10

# ---- Naver ----
User-agent: Yeti
Allow: /
Crawl-delay: 10

# ---- Sogou ----
User-agent: Sogou
Allow: /
Crawl-delay: 10

# ---- Social/Media crawlers (OG/Twitter cards) ----
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: Discordbot
User-agent: Slackbot
User-agent: TelegramBot
User-agent: Applebot
Allow: /

# ---- AI / LLM crawlers (GEO: explicitly allowed) ----
# Generative engine crawlers may read public content for AI answers
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: Applebot-Extended
Allow: /

# ==========================================
# HOST & SITEMAP
# ==========================================
# Host directive deprecated (only Yandex used it); use canonical + sitemap instead
Sitemap: https://nextyhost.com/sitemap.xml

# ==========================================
# CLEAN PARAMS (Yandex/Google Search Console)
# ==========================================
# Uncomment and register in Yandex Webmaster / GSC if you use these params:
# Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content / 
# Clean-param: fbclid&gclid&ref&sort&page&currency /

# ==========================================
# NOTES
# ==========================================
# - All public pages are crawlable via Allow: /
# - CSS/JS allowed for rendering (Core Web Vitals, Mobile-First)
# - Internal/app files blocked for security
# - Parameter URLs blocked to prevent duplicate content
# - Sitemap: https://nextyhost.com/sitemap.xml (59 URLs, updated 2026-10-02)
# - Monitor: GSC > Indexing > Pages, Bing Webmaster > Site Explorer