feat: create content package

.gitignore

@@ -45,3 +45,4 @@ npm-debug.log*
 lychee-out.md
 .lycheecache
 .env*.local
+.content-collections

apps/chatbot/package.json

@@ -12,51 +12,51 @@
     "typecheck": "tsc --noEmit --emitDeclarationOnly false"
   },
   "dependencies": {
-    "@ai-sdk/react": "^1.2.0",
-    "@ai-sdk/xai": "^1.2.1",
+    "@ai-sdk/react": "^1.2.5",
+    "@ai-sdk/xai": "^1.2.5",
     "@mdx-js/loader": "^3.1.0",
     "@mdx-js/react": "^3.1.0",
-    "@next/mdx": "^15.2.3",
+    "@next/mdx": "^15.2.4",
     "@thedaviddias/config-next": "workspace:*",
     "@thedaviddias/config-typescript": "workspace:*",
     "@thedaviddias/design-system": "workspace:*",
     "@thedaviddias/flags": "workspace:*",
     "@thedaviddias/observability": "workspace:*",
     "@thedaviddias/supabase": "workspace:*",
     "@thedaviddias/utils": "workspace:*",
-    "ai": "^4.2.0",
+    "ai": "^4.2.10",
     "axios": "^1.8.4",
     "cheerio": "^1.0.0",
     "clsx": "^2.1.1",
     "date-fns": "^4.1.0",
-    "framer-motion": "^12.5.0",
+    "framer-motion": "^12.6.3",
     "fuse.js": "^7.1.0",
     "gray-matter": "^4.0.3",
-    "lucide-react": "^0.483.0",
-    "next": "15.2.3",
-    "react": "^19.0.0",
-    "react-dom": "^19",
+    "lucide-react": "^0.486.0",
+    "next": "15.2.4",
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0",
     "react-markdown": "^10.1.0",
-    "sonner": "^2.0.1",
+    "sonner": "^2.0.3",
     "zod": "^3.24.2"
   },
   "devDependencies": {
     "@swc/jest": "^0.2.37",
-    "@tailwindcss/postcss": "^4.0.15",
+    "@tailwindcss/postcss": "^4.1.0",
     "@testing-library/jest-dom": "^6.6.3",
     "@testing-library/react": "^16.2.0",
     "@testing-library/user-event": "^14.6.1",
     "@types/jest": "^29.5.14",
-    "@types/node": "^22.13.11",
+    "@types/node": "^22.13.17",
     "@types/react": "19.0.12",
     "@types/react-dom": "^19.0.4",
     "identity-obj-proxy": "^3.0.0",
     "jest": "^29.7.0",
     "jest-environment-jsdom": "^29.7.0",
     "jest-watch-typeahead": "^2.2.2",
     "postcss": "^8.5.3",
-    "tailwindcss": "^4.0.15",
-    "ts-jest": "^29.2.6",
+    "tailwindcss": "^4.1.0",
+    "ts-jest": "^29.3.1",
     "typescript": "^5.8.2"
   }
 }

apps/web/.gitignore

@@ -0,0 +1 @@
+.content-collections

apps/web/actions/get-home-page-data.ts

@@ -1,10 +1,10 @@
 'use server'
 
-import { getAllWebsites } from '@/lib/mdx'
+import { getWebsites } from '@/lib/content-loader'
 import { getFeaturedProjects, getRecentlyUpdatedProjects } from '@/lib/project-utils'
 
 export async function getHomePageData() {
-  const allProjects = await getAllWebsites()
+  const allProjects = await getWebsites()
   const featuredProjects = getFeaturedProjects(allProjects, 4)
   const recentlyUpdatedProjects = getRecentlyUpdatedProjects(allProjects, 5)
 

apps/web/app/(files)/rss.xml/route.ts

@@ -1,9 +1,9 @@
-import { type WebsiteMetadata, getAllWebsites } from '@/lib/mdx'
+import { getWebsites } from '@/lib/content-loader'
 
 const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://llmstxthub.com'
 
 export async function GET() {
-  const websitesData = await getAllWebsites()
+  const websitesData = await getWebsites()
 
   const feed = {
     version: 'https://jsonfeed.org/version/1',
@@ -21,7 +21,7 @@ export async function GET() {
     ],
     language: 'en',
     items: [
-      ...websitesData.map((site: WebsiteMetadata) => ({
+      ...websitesData.map(site => ({
         id: site.slug,
         url: `${baseUrl}/websites/${site.slug}`,
         title: site.name,

apps/web/app/api/debug/content-paths/route.ts

@@ -0,0 +1,52 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { type NextRequest, NextResponse } from 'next/server'
+
+/**
+ * Endpoint to check content paths and debug content-collections issues
+ */
+export async function GET(request: NextRequest) {
+  const paths = {
+    cwd: process.cwd(),
+    relPath: path.resolve(process.cwd(), '../../packages/content/data'),
+    absPath: path.resolve(process.cwd(), '..', 'packages/content/data'),
+    exists: {
+      relPath: fs.existsSync(path.resolve(process.cwd(), '../../packages/content/data')),
+      absPath: fs.existsSync(path.resolve(process.cwd(), '..', 'packages/content/data'))
+    },
+    contentDirs: {}
+  }
+
+  // Check if content directories exist
+  try {
+    const contentDir = paths.exists.relPath ? paths.relPath : paths.absPath
+    const dirs = ['websites', 'guides', 'resources', 'legal']
+
+    type ContentDirInfo = {
+      path: string
+      exists: boolean
+      files: number
+    }
+
+    paths.contentDirs = dirs.reduce<Record<string, ContentDirInfo>>((acc, dir) => {
+      const dirPath = path.join(contentDir, dir)
+      acc[dir] = {
+        path: dirPath,
+        exists: fs.existsSync(dirPath),
+        files: fs.existsSync(dirPath) ? fs.readdirSync(dirPath).length : 0
+      }
+      return acc
+    }, {})
+  } catch (error: unknown) {
+    const errorMessage = error instanceof Error ? error.message : String(error)
+    const errorStack = error instanceof Error ? error.stack : undefined
+
+    return NextResponse.json({
+      paths,
+      error: errorMessage,
+      stack: errorStack
+    })
+  }
+
+  return NextResponse.json(paths)
+}

apps/web/app/api/fetch-metadata/route.ts

@@ -1,4 +1,4 @@
-import { getAllWebsites } from '@/lib/mdx'
+import { getWebsites } from '@/lib/content-loader'
 import * as cheerio from 'cheerio'
 import { NextResponse } from 'next/server'
 import normalizeUrl from 'normalize-url'
@@ -14,7 +14,7 @@ function cleanTitle(title: string): string {
 async function fetchMetadata(url: string) {
   try {
     // Check for duplicate websites
-    const existingWebsites = await getAllWebsites()
+    const existingWebsites = await getWebsites()
 
     const normalizedNewUrl = normalizeUrl(url, {
       stripProtocol: true,

apps/web/app/page.tsx

@@ -10,7 +10,7 @@ import { LatestNewsSection } from '@/components/sections/latest-news-section'
 import { LatestUpdatesSection } from '@/components/sections/latest-updates-section'
 import { NewsletterSection } from '@/components/sections/newsletter-section'
 import { ToolsSection } from '@/components/sections/tools-section'
-import { getAllGuides } from '@/lib/mdx'
+import { getGuides } from '@/lib/content-loader'
 import { getBaseUrl } from '@thedaviddias/utils/get-base-url'
 import type { Metadata } from 'next'
 import { CommunitiesSection } from '@/components/sections/communities-section'
@@ -46,7 +46,7 @@ export const metadata: Metadata = {
 
 export default async function Home() {
   const { featuredProjects, recentlyUpdatedProjects } = await getHomePageData()
-  const featuredGuides = await getAllGuides()
+  const featuredGuides = await getGuides()
 
   return (
     <>

apps/web/app/sitemap.ts

@@ -86,7 +86,7 @@ function getStaticRoutes(baseUrl: string): MetadataRoute.Sitemap {
 export default function sitemap(): MetadataRoute.Sitemap {
   const routes: MetadataRoute.Sitemap = []
   const baseUrl = process.env.NEXT_PUBLIC_APP_URL || 'https://llmstxthub.com'
-  const contentDir = join(process.cwd(), '../../content')
+  const contentDir = join(process.cwd(), '../../packages/content/data')
 
   try {
     // Add static routes