# Robots.txt for Apyrn.com
# Last updated: 2024-12-09

# Allow all search engines to crawl the site
User-agent: *
Allow: /

# Disallow specific directories and files
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: /_next/
Disallow: /404
Disallow: /500
Disallow: /build/
Disallow: /node_modules/
Disallow: /.env
Disallow: /package.json
Disallow: /package-lock.json

# Allow important static files
Allow: /static/css/
Allow: /static/js/
Allow: /static/media/
Allow: /*.css$
Allow: /*.js$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.ico$

# Allow important pages and directories
Allow: /tools/
Allow: /products/
Allow: /services/
Allow: /contact
Allow: /about
Allow: /blog/

# AI Discovery Resources
Allow: /ai-sitemap.xml
Allow: /ai-discovery.html
Allow: /structured-data.json

# Specific rules for Google
User-agent: Googlebot
Allow: /

# Specific rules for Bing
User-agent: Bingbot
Allow: /

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemaps - All sitemaps for better indexing
Sitemap: https://apyrn.com/sitemap-index.xml
Sitemap: https://apyrn.com/sitemap.xml
Sitemap: https://apyrn.com/sitemap-tools.xml
Sitemap: https://apyrn.com/sitemap-products.xml
Sitemap: https://apyrn.com/sitemap-services.xml

# Host directive
Host: https://apyrn.com