# This file tells search engines and bots what they are allowed to see on your site.

# This is the default rule, which allows search engines to crawl your site (recommended).
User-agent: *
Allow: /

# If you do not want AI bots to crawl your site, remove the # from the following lines:
#User-agent: AI2Bot
#User-agent: Ai2Bot-Dolma
#User-agent: Amazonbot
#User-agent: anthropic-ai
#User-agent: Applebot-Extended
#User-agent: Bytespider
#User-agent: CCBot
#User-agent: ChatGPT-User
#User-agent: Claude-Web
#User-agent: ClaudeBot
#User-agent: cohere-ai
#User-agent: Diffbot
#User-agent: DuckAssistBot
#User-agent: FacebookBot
#User-agent: FriendlyCrawler
#User-agent: Google-Extended
#User-agent: GoogleOther
#User-agent: GoogleOther-Image
#User-agent: GoogleOther-Video
#User-agent: GPTBot
#User-agent: iaskspider/2.0
#User-agent: ICC-Crawler
#User-agent: ImagesiftBot
#User-agent: img2dataset
#User-agent: ISSCyberRiskCrawler
#User-agent: Kangaroo Bot
#User-agent: Meta-ExternalAgent
#User-agent: Meta-ExternalFetcher
#User-agent: OAI-SearchBot
#User-agent: omgili
#User-agent: omgilibot
#User-agent: PanguBot
#User-agent: PerplexityBot
#User-agent: PetalBot
#User-agent: Scrapy
#User-agent: Sidetrade indexer bot
#User-agent: Timpibot
#User-agent: VelenPublicWebCrawler
#User-agent: Webzio-Extended
#User-agent: YouBot
#Disallow: /


# Genel erişim
User-agent: *
Allow: /

# Özel dosyalar
Disallow: /private/
Disallow: /temp/
Disallow: /drafts/
Disallow: /backup/
Disallow: /cgi-bin/

# Arama sonuçlarını indexletme
Disallow: /search

# URL parametreleri (gereksiz duplicate sayfalar)
Disallow: /*?*

# Belirli dosya tipleri
Disallow: /*.bak$
Disallow: /*.tmp$
Disallow: /*.log$

# Crawl gecikmesi
Crawl-delay: 5

# Sitemap
Sitemap: https://seninsiten.neocities.org/sitemap.xml


# =========================================
# GOOGLE
# =========================================

User-agent: Googlebot
Allow: /

# Google görselleri
User-agent: Googlebot-Image
Allow: /

# Google videoları
User-agent: Googlebot-Video
Allow: /


# =========================================
# BING
# =========================================

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /


# =========================================
# AI BOTLARI
# =========================================

# OpenAI
User-agent: GPTBot
Disallow: /

# Anthropic Claude
User-agent: ClaudeBot
Disallow: /

# Perplexity
User-agent: PerplexityBot
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Facebook AI
User-agent: FacebookBot
Disallow: /

# Bytespider (TikTok)
User-agent: Bytespider
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# Meta dışı AI tarayıcıları
User-agent: ImagesiftBot
Disallow: /


# =========================================
# KÖTÜ/SPAM BOTLAR
# =========================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /