Fix llms.txt (#1235)

Co-authored-by: Tobie Morgan Hitchcock <[email protected]>

Author: kearfy

Makefile

@@ -77,11 +77,13 @@ deploy:
 	@echo "Deploy..."
 	aws s3 sync --region eu-west-2 --cache-control "public, max-age=31536000, immutable" --exclude '.DS_Store' --exclude '*' --include '*.webp' --content-type 'image/webp' ./dist/docs/_astro s3://www.surrealdb.com/docs/_astro/
 	aws s3 sync --region eu-west-2 --cache-control "public, max-age=31536000, immutable" --exclude '.DS_Store' --exclude '*.webp' ./dist/docs/_astro s3://www.surrealdb.com/docs/_astro/
+	aws s3 cp   --region eu-west-2 --cache-control "public, max-age=300" ./dist/docs/llms.txt s3://www.surrealdb.com/docs/
 	aws s3 sync --region eu-west-2 --cache-control "public, max-age=30" --delete --exclude '*' --include '*.html' ./dist/docs/ s3://www.surrealdb.com/docs/
 
 .PHONY: stage
 stage:
 	@echo "Stage..."
 	aws s3 sync --region eu-west-2 --cache-control "public, max-age=31536000, immutable" --exclude '.DS_Store' --exclude '*' --include '*.webp' --content-type 'image/webp' ./dist/docs/_astro s3://www.surrealdb.dev/docs/_astro/
 	aws s3 sync --region eu-west-2 --cache-control "public, max-age=31536000, immutable" --exclude '.DS_Store' --exclude '*.webp' ./dist/docs/_astro s3://www.surrealdb.dev/docs/_astro/
+	aws s3 cp   --region eu-west-2 --cache-control "public, max-age=86400" ./dist/docs/llms.txt s3://www.surrealdb.dev/docs/
 	aws s3 sync --region eu-west-2 --cache-control "public, max-age=30" --delete --exclude '*' --include '*.html' ./dist/docs/ s3://www.surrealdb.dev/docs/

astro.config.mjs

@@ -14,7 +14,6 @@ import { autolinkConfig } from './src/util/rehypeHeadingsConfig';
 import { rehypeNotesPlugin } from './src/util/rehypeNotesPlugin.mjs';
 
 import sitemap from '@astrojs/sitemap';
-import llms from 'vite-plugin-llms';
 
 const deployDomain = process.env.DEPLOY_DOMAIN ?? 'surrealdb.com';
 const site = `https://${deployDomain}`;
@@ -48,11 +47,4 @@ export default defineConfig({
         ],
         syntaxHighlight: false,
     },
-    vite: {
-        plugins: [
-            llms({
-                llmsDir: 'llms',
-            }),
-        ],
-    },
 });

bun.lockb

@@ -59,8 +59,7 @@
         "tailwind-merge": "^2.5.4",
         "tailwindcss": "^3.4.15",
         "typescript": "^5.6.3",
-        "unist-util-visit": "^5.0.0",
-        "vite-plugin-llms": "^1.0.2"
+        "unist-util-visit": "^5.0.0"
     },
     "devDependencies": {
         "@biomejs/biome": "1.8.3",

package.json

@@ -60,3 +60,19 @@ Things to remember when working with SurrealDB:
 - [Aeon's Surreal Renaissance](https://surrealdb.com/learn/book)
 - [Surreal Sidekick](https://surrealist.app/cloud/chat)
 
+## Raw markdown content
+
+To find the raw markdown content for a route, follow these instructions:
+- For a route which matches `/docs/sdk/{language}/{...path}`, you can translate it to either of: 
+  - https://raw.githubusercontent.com/surrealdb/docs.surrealdb.com/refs/heads/main/src/content/doc-sdk-{language}/{path}.md(x?)
+  - https://raw.githubusercontent.com/surrealdb/docs.surrealdb.com/refs/heads/main/src/content/doc-sdk-{language}/{path}/index.md(x?)
+- For a route which matches `/docs/{doc}/{...path}`, you can translate it to either of: 
+  - https://raw.githubusercontent.com/surrealdb/docs.surrealdb.com/refs/heads/main/src/content/doc-{doc}/{path}.md(x?)
+  - https://raw.githubusercontent.com/surrealdb/docs.surrealdb.com/refs/heads/main/src/content/doc-{doc}/{path}/index.md(x?)
+
+Notice the ending file extension (`.md(x?)`), by this I mean that the file extension may be either of `.md` or `.mdx`, however `.mdx` is most likely to be the file extension.
+For either of the two criteria, if the ending `/{...path}` is missing, the file under the `content/doc-{...}` folder will be `index.md(x?)`
+
+### Example:
+- Website route: /docs/sdk/javascript/data-types
+- Raw markdown URL: https://raw.githubusercontent.com/surrealdb/docs.surrealdb.com/refs/heads/main/src/content/doc-sdk-javascript/data-types.mdx