feat: add sitemap.xml, robots.txt, and llms.txt for SEO

2026-03-13 14:40:41 +01:00
parent b5d408b4e8
commit 2a2a64f6d7
3 changed files with 138 additions and 0 deletions
--- a/app/robots.ts
+++ b/app/robots.ts
@@ -0,0 +1,50 @@
+import type { MetadataRoute } from 'next'
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        allow: [
+          '/',
+          '/about',
+          '/features',
+          '/pricing',
+          '/faq',
+          '/changelog',
+          '/installation',
+          '/integrations',
+        ],
+        disallow: [
+          '/api/',
+          '/admin/',
+          '/sites/',
+          '/notifications/',
+          '/onboarding/',
+          '/org-settings/',
+          '/welcome/',
+          '/auth/',
+          '/actions/',
+          '/share/',
+        ],
+      },
+      {
+        userAgent: 'GPTBot',
+        disallow: ['/'],
+      },
+      {
+        userAgent: 'ChatGPT-User',
+        disallow: ['/'],
+      },
+      {
+        userAgent: 'Google-Extended',
+        disallow: ['/'],
+      },
+      {
+        userAgent: 'CCBot',
+        disallow: ['/'],
+      },
+    ],
+    sitemap: 'https://pulse.ciphera.net/sitemap.xml',
+  }
+}