# ============================================================================
# WikiAI.hu robots.txt
# Public content is open to search engines, AI search, user-triggered fetchers,
# model-development crawlers, archives, SEO tools and social preview bots.
# Updated: 2026-09-26
# ============================================================================

Sitemap: https://www.wikiai.hu/sitemap_index.xml

# WikiAI Machine Delivery first-party catalog references
# The canonical WordPress HTML remains authoritative.
# Discovery manifest: https://www.wikiai.hu/wp-content/uploads/wikiai-machine/v1/discovery.json
# Machine JSON index: https://www.wikiai.hu/wp-content/uploads/wikiai-machine/v1/index.json
# Machine HTML index: https://www.wikiai.hu/wp-content/uploads/wikiai-machine/v1/index.html
# Machine Markdown catalog: https://www.wikiai.hu/wp-content/uploads/wikiai-machine/v1/catalog.md
# Catalog API: https://www.wikiai.hu/wp-json/wikiai-machine/v1/catalog
# Discovery API: https://www.wikiai.hu/wp-json/wikiai-machine/v1/discovery

# Default policy for every standards-compliant crawler not named below.
User-agent: *
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# OpenAI: search, user fetch, model development and ad validation.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: OAI-AdsBot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Anthropic: search, user fetch and model development.
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: ClaudeBot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Google Search, Gemini, Vertex AI and user-triggered Google agents.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: Google-InspectionTool
User-agent: Storebot-Google
User-agent: Google-CloudVertexBot
User-agent: Google-Extended
User-agent: Google-Agent
User-agent: Google-GeminiNotebook
User-agent: Google-Read-Aloud
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Microsoft Bing and preview services; Bing also powers Copilot discovery.
User-agent: bingbot
User-agent: BingPreview
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Perplexity search and user-triggered retrieval.
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Mistral Vibe: search index, user-triggered retrieval and model training.
User-agent: MistralAI-Index
User-agent: MistralAI-User
User-agent: MistralAI-Training
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Apple Search, Siri, Safari and foundation-model usage control.
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Meta AI, Facebook search/preview and user-triggered retrieval.
User-agent: meta-externalagent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: facebookexternalhit
User-agent: Facebot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Other AI, answer-engine and public-dataset crawlers.
User-agent: Amazonbot
User-agent: CCBot
User-agent: Bytespider
User-agent: YouBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: AI2Bot
User-agent: AI2Bot-Dolma
User-agent: Diffbot
User-agent: Timpibot
User-agent: omgili
User-agent: omgilibot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Independent and regional search engines.
# Brave Search publishes no separate crawler token and follows Googlebot rules.
User-agent: DuckDuckBot
User-agent: Qwantify
User-agent: MojeekBot
User-agent: Kagibot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Slurp
User-agent: PetalBot
User-agent: SeznamBot
User-agent: NaverBot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Web archives.
User-agent: ia_archiver
User-agent: archive.org_bot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# SEO and site-audit crawlers.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: Screaming Frog SEO Spider
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# Social and messaging link-preview crawlers.
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterestbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: Discordbot
User-agent: Slackbot
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /


# robots.txt controls crawl permission. It cannot guarantee indexing, ranking,
# citation, recommendation or inclusion in AI-generated answers.