# Updated Apr 2025

User-Agent: *
Disallow: /cmd/
Disallow: /*.json
Disallow: /*.woff2$
Disallow: /*.woff$																									
Disallow: /.well-known/                                                                                                                                                                                                                                                                                                                                                                    
Disallow: /_next/static/media/
Allow: /_next/static/media/*svg																									
Disallow: /*search$
Disallow: /*comment/

User-Agent: Mediapartners-Google
Disallow: /cmd/
Disallow: /*.json
Disallow: /*.woff2$
Disallow: /*.woff$																									
Disallow: /.well-known/                                                                                                                                                                                                                                                                                                                                                                    
Disallow: /_next/static/media/
Allow: /_next/static/media/*svg																									
Disallow: /*search$
Allow: /*comment/

User-agent: Amazonbot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Bytespider
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: cohere-ai
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: CCBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Deepseek-V3
User-agent: DeepSeek-V3
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: GPTBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: anthropic-ai
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Google-CloudVertexBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Google-Extended
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Applebot-Extended
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: omgilibot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: omgili
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: ClaudeBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Claude-Web
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: PerplexityBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Diffbot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: OAI-SearchBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: NewsNow
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: news-please
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: Meta-ExternalFetcher
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

User-agent: FacebookBot
Disallow: /*story*
Disallow: /*video*
Disallow: /*comment/
Allow: /

# Intelligence Gatherers

User-agent: magpie-crawler
Disallow: /

User-agent: BLP_bbot
Disallow: /

User-agent: sentibot
Disallow: /

User-agent: peer_39_crawler/1.0
User-agent: peer39_crawler
Disallow: /

# SEO Crawlers

User-agent: dotbot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

Sitemap: https:www.lessentiel.lu/sitemaps/fr/articles.xml
Sitemap: https:www.lessentiel.lu/sitemaps/fr/navigation.xml
Sitemap: https:www.lessentiel.lu/sitemaps/de/news.xml
Sitemap: https:www.lessentiel.lu/sitemaps/de/articles.xml
Sitemap: https:www.lessentiel.lu/sitemaps/de/navigation.xml