# QCK URL Shortener - robots.txt
# https://qck.sh

# Allow all search engines
User-agent: *
Allow: /

# Sitemaps
Sitemap: https://qck.sh/sitemap.xml
Sitemap: https://qck.sh/sitemap-index.xml

# LLMS.txt for AI crawlers
# See: https://qck.sh/llms.txt

# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: MSNBot
Allow: /

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 1

# Yandex
User-agent: Yandex
Allow: /
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Facebook
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

# Twitter
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Pinterest
User-agent: Pinterest
Allow: /

# Slack
User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

# Discord
User-agent: Discordbot
Allow: /

# Telegram
User-agent: TelegramBot
Allow: /

# WhatsApp
User-agent: WhatsApp
Allow: /

# Apple
User-agent: Applebot
Allow: /

# AI Crawlers - Welcome!
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Perplexity
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Cohere-ai
Allow: /

# SEO Tools
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

User-agent: MJ12bot
Allow: /
Crawl-delay: 5

User-agent: DotBot
Allow: /
Crawl-delay: 5

# Archive.org - Welcome to preserve our content
User-agent: ia_archiver
Allow: /

# Block known bad bots
User-agent: MegaIndex.ru
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

# Host directive
Host: https://qck.sh

# Note: Short URLs (qck.sh/xxxxx) redirect and are not indexable
# The main site and dashboard are fully crawlable