feat: add sitemap.xml, robots.txt, and llms.txt for SEO

app/robots.ts

@@ -0,0 +1,50 @@
+import type { MetadataRoute } from 'next'
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        allow: [
+          '/',
+          '/about',
+          '/features',
+          '/pricing',
+          '/faq',
+          '/changelog',
+          '/installation',
+          '/integrations',
+        ],
+        disallow: [
+          '/api/',
+          '/admin/',
+          '/sites/',
+          '/notifications/',
+          '/onboarding/',
+          '/org-settings/',
+          '/welcome/',
+          '/auth/',
+          '/actions/',
+          '/share/',
+        ],
+      },
+      {
+        userAgent: 'GPTBot',
+        disallow: ['/'],
+      },
+      {
+        userAgent: 'ChatGPT-User',
+        disallow: ['/'],
+      },
+      {
+        userAgent: 'Google-Extended',
+        disallow: ['/'],
+      },
+      {
+        userAgent: 'CCBot',
+        disallow: ['/'],
+      },
+    ],
+    sitemap: 'https://pulse.ciphera.net/sitemap.xml',
+  }
+}

app/sitemap.ts

@@ -0,0 +1,38 @@
+import type { MetadataRoute } from 'next'
+
+const BASE_URL = 'https://pulse.ciphera.net'
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const integrationSlugs = [
+    'nextjs',
+    'react',
+    'vue',
+    'wordpress',
+  ]
+
+  const publicRoutes = [
+    { url: '', priority: 1.0, changeFrequency: 'weekly' as const },
+    { url: '/about', priority: 0.8, changeFrequency: 'monthly' as const },
+    { url: '/features', priority: 0.9, changeFrequency: 'monthly' as const },
+    { url: '/pricing', priority: 0.9, changeFrequency: 'monthly' as const },
+    { url: '/faq', priority: 0.7, changeFrequency: 'monthly' as const },
+    { url: '/changelog', priority: 0.6, changeFrequency: 'weekly' as const },
+    { url: '/installation', priority: 0.8, changeFrequency: 'monthly' as const },
+    { url: '/integrations', priority: 0.8, changeFrequency: 'monthly' as const },
+  ]
+
+  const integrationRoutes = integrationSlugs.map((slug) => ({
+    url: `/integrations/${slug}`,
+    priority: 0.7,
+    changeFrequency: 'monthly' as const,
+  }))
+
+  const allRoutes = [...publicRoutes, ...integrationRoutes]
+
+  return allRoutes.map((route) => ({
+    url: `${BASE_URL}${route.url}`,
+    lastModified: new Date(),
+    changeFrequency: route.changeFrequency,
+    priority: route.priority,
+  }))
+}