gitadmin/Bunny-Agency-Italy
1
0
Fork 0
mirror of https://github.com/BigMarc/Bunny-Agency-Italy.git synced 2026-10-13 17:00:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(seo): name all AI answer-engine crawlers in robots.txt

Browse Source 
Adds the bots missing from the explicit allow list (of GPTBot, OAI-SearchBot,
ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, PerplexityBot,
Perplexity-User, Google-Extended). Crawl behaviour is unchanged: they were
already allowed via User-agent: *.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
BigMarc 2026-09-26 14:26:36 +02:00
parent a6b0bdcab2
commit f65c91f296
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/app/robots.ts
View File

@ -1,6 +1,6 @@
import type { MetadataRoute } from "next"; import type { MetadataRoute } from "next";
import { absoluteUrl } from "@/lib/content"; import { absoluteUrl } from "@/lib/content";
export default function robots(): MetadataRoute.Robots { export default function robots(): MetadataRoute.Robots {
const agents = ["*", "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended"]; const agents = ["*", "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Google-Extended"];
return { rules: agents.map((userAgent) => ({ userAgent, allow: "/" })), sitemap: absoluteUrl("/sitemap.xml"), host: absoluteUrl("/") }; return { rules: agents.map((userAgent) => ({ userAgent, allow: "/" })), sitemap: absoluteUrl("/sitemap.xml"), host: absoluteUrl("/") };
} }