# Robots.txt - Dental Concepts
# https://www.dental-concepts.com/

User-agent: *
Allow: /
Allow: /wp-admin/admin-ajax.php

# Block internal and temporary folders
Disallow: /new/
Disallow: /old-files/
Disallow: /log/
Disallow: /cgi-bin/
Disallow: /public/ckfinder/
Disallow: /public/ckeditor/
Disallow: /campaign1/
Disallow: /dc-new/
Disallow: /test/
Disallow: /backend/
Disallow: /tmp_ci_sessions/
Disallow: /dc.ninetriangles.com/

# Block admin and login pages
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /login/
Disallow: /dashboard/

# Block utility pages
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /*?print=
Disallow: /*?replytocom=
Disallow: /feed/
Disallow: /*/feed/
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /?s=
Disallow: /search/

# Google Image
User-agent: Googlebot-Image
Allow: /
Disallow: /old-files/
Disallow: /test/
Disallow: /dc-new/

# Google AI
User-agent: Google-Extended
Allow: /

# Bing
User-agent: Bingbot
Allow: /
Disallow: /new/
Disallow: /old-files/
Disallow: /log/
Disallow: /cgi-bin/
Disallow: /public/ckfinder/
Disallow: /public/ckeditor/
Disallow: /campaign1/
Disallow: /dc-new/
Disallow: /test/
Disallow: /tmp_ci_sessions/
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /?s=
Disallow: /search/

User-agent: BingPreview
Allow: /

# Other Search Engines
User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /test/
Disallow: /old-files/

User-agent: Yandex
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /old-files/
Disallow: /test/

User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /old-files/
Disallow: /test/

# AI Search Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: YouBot
Allow: /

User-agent: ia_archiver
Allow: /

User-agent: Bytespider
Allow: /

User-agent: CCBot
Allow: /

# Social Media Crawlers
User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# SEO Tools
User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: rogerbot
Allow: /

User-agent: Screaming Frog SEO Spider
Allow: /

# Block Unwanted Crawlers
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

# XML Sitemap
Sitemap: https://www.dental-concepts.com/sitemap.xml
LLMs-txt: https://www.dental-concepts.com/llms.txt