From f65c91f296da97573659f9310bd0abc8855471bf Mon Sep 17 00:00:00 2001 From: BigMarc <79053307+BigMarc@users.noreply.github.com> Date: Sat, 26 Sep 2026 14:26:36 +0200 Subject: [PATCH] fix(seo): name all AI answer-engine crawlers in robots.txt Adds the bots missing from the explicit allow list (of GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, PerplexityBot, Perplexity-User, Google-Extended). Crawl behaviour is unchanged: they were already allowed via User-agent: *. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/app/robots.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/robots.ts b/src/app/robots.ts index c9767d1..eb12e9a 100644 --- a/src/app/robots.ts +++ b/src/app/robots.ts @@ -1,6 +1,6 @@ import type { MetadataRoute } from "next"; import { absoluteUrl } from "@/lib/content"; export default function robots(): MetadataRoute.Robots { - const agents = ["*", "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended"]; + const agents = ["*", "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Google-Extended"]; return { rules: agents.map((userAgent) => ({ userAgent, allow: "/" })), sitemap: absoluteUrl("/sitemap.xml"), host: absoluteUrl("/") }; }