fix(markdown): handle JSON parsing errors in processCodeBlocksWithLabel (#4114)

* fix(markdown): handle JSON parsing errors in processCodeBlocksWithLabel function

* feat(markdown): add formatObjectToMarkdownBlock and formatPlaceholder functions for improved markdown handling

- Introduced formatObjectToMarkdownBlock to format objects into markdown code blocks with metadata.
- Added formatPlaceholder to create formatted placeholders for file and symbol types.
- Updated processingPlaceholder to utilize the new formatting functions.
- Enhanced custom AST stringification in markdown processing.
- Added tests for new formatting functions and existing markdown utilities.

* feat(markdown): enhance markdown processing with placeholder handling and unified dependency

- Added a placeholder handler in the markdown processor to manage custom nodes.
- Updated the `createRemarkProcessor` function to include the new placeholder handler.
- Refactored `customAstToString` to streamline AST stringification.
- Introduced a new `unified` dependency for improved markdown processing.
- Added comprehensive tests for new functionality and existing markdown utilities.

* refactor(chat): remove unused log

* chore: remove log

* [autofix.ci] apply automated fixes

* Update ee/tabby-ui/lib/utils/markdown.ts

* Update ee/tabby-ui/lib/utils/markdown.ts

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Zhiming Ma <codes.icy@gmail.com>

Author: 3 people

ee/tabby-ui/lib/utils/chat.ts

@@ -25,7 +25,10 @@ import {
   PLACEHOLDER_FILE_REGEX,
   PLACEHOLDER_SYMBOL_REGEX
 } from '../constants/regex'
-import { convertContextBlockToPlaceholder } from './markdown'
+import {
+  convertContextBlockToPlaceholder,
+  formatObjectToMarkdownBlock
+} from './markdown'
 
 export const isCodeSourceContext = (kind: ContextSourceKind) => {
   return [
@@ -354,8 +357,7 @@ export async function processingPlaceholder(
         })
         let replacement = ''
         if (content) {
-          const fileInfoJSON = JSON.stringify(fileInfo).replace(/"/g, '\\"')
-          replacement = `\n\`\`\`context label='file' object='${fileInfoJSON}'\n${content}\n\`\`\`\n`
+          replacement = formatObjectToMarkdownBlock('file', fileInfo, content)
         }
         processedMessage = processedMessage.replace(match[0], replacement)
         tempMessage = tempMessage.replace(match[0], replacement)
@@ -381,8 +383,11 @@ export async function processingPlaceholder(
         })
         let replacement = ''
         if (content) {
-          const symbolInfoJSON = JSON.stringify(symbolInfo).replace(/"/g, '\\"')
-          replacement = `\n\`\`\`context label='symbol' object='${symbolInfoJSON}'\n${content}\n\`\`\`\n`
+          replacement = formatObjectToMarkdownBlock(
+            'symbol',
+            symbolInfo,
+            content
+          )
         }
         processedMessage = processedMessage.replace(match[0], replacement)
         tempMessage = tempMessage.replace(match[0], replacement)

ee/tabby-ui/lib/utils/markdown.ts

@@ -1,58 +1,37 @@
 import { Parent, Root, RootContent } from 'mdast'
 import { remark } from 'remark'
-import remarkStringify from 'remark-stringify'
+import remarkStringify, { Options } from 'remark-stringify'
 
-/**
- * Custom stringification of AST to preserve special patterns
- * @param ast AST to stringify
- * @returns Plain string representation
- */
-export function customAstToString(ast: Root): string {
-  let result = ''
-  for (const node of ast.children) {
-    result += nodeToString(node) + '\n'
-  }
-  return result.trim()
+const REMARK_STRINGIFY_OPTIONS: Options = {
+  bullet: '*',
+  emphasis: '*',
+  fences: true,
+  listItemIndent: 'one',
+  tightDefinitions: true,
+  handlers: {
+    placeholder: (node: PlaceholderNode) => {
+      return node.value
+    }
+  } as any
 }
 
-/**
- * Convert a single node to string
- * @param node AST node
- * @returns String representation
- */
-function nodeToString(node: any): string {
-  switch (node.type) {
-    case 'paragraph':
-      return paragraphToString(node)
-    case 'text':
-      return node.value
-    default:
-      const processor = remark().use(remarkStringify)
-      return processor.stringify({ type: 'root', children: [node] }).trim()
-  }
+function createRemarkProcessor() {
+  return remark().use(remarkStringify, REMARK_STRINGIFY_OPTIONS)
 }
 
 /**
- * Convert paragraph node to string
- * @param node Paragraph node
- * @returns String representation
+ * Custom stringification of AST using remarkStringify
+ * @param ast AST to stringify
+ * @returns Plain string representation
  */
-function paragraphToString(node: any): string {
-  return childrenToString(node)
+export function customAstToString(ast: Root): string {
+  const processor = createRemarkProcessor()
+  return processor.stringify(ast).trim()
 }
 
 /**
- * Process children of a node and join them
- * @param node Parent node
- * @returns Combined string of all children
+ * Process code blocks with labels and convert them to placeholders
  */
-function childrenToString(node: any): string {
-  if (!node.children || node.children.length === 0) {
-    return ''
-  }
-  return node.children.map((child: any) => nodeToString(child)).join('')
-}
-
 export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
   const newChildren: RootContent[] = []
   for (let i = 0; i < ast.children.length; i++) {
@@ -62,8 +41,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
     if (node.type === 'code' && node.meta) {
       node.meta?.split(' ').forEach(item => {
         const [key, rawValue] = item.split(/=(.+)/)
-        const value = rawValue?.replace(/^['"]|['"]$/g, '') || ''
-        metas[key] = value
+        metas[key] = rawValue
       })
     }
 
@@ -84,31 +62,57 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
         nextNode.position.start.line - node.position.end.line === 1
 
       let finalCommandText = ''
+      let placeholderNode: RootContent | null = null
+      let shouldProcessNode = true
 
-      // processing differet type of context
       switch (metas['label']) {
         case 'changes':
-          finalCommandText = '[[contextCommand:"changes"]]'
+          finalCommandText = '"changes"'
+          placeholderNode = createPlaceholderNode(
+            `[[contextCommand:${finalCommandText}]]`
+          ) as unknown as RootContent
           break
         case 'file':
           if (metas['object']) {
-            const fileObject = JSON.parse(metas['object'].replace(/\\"/g, '"'))
-            finalCommandText = `[[file:${JSON.stringify(fileObject)}]]`
+            try {
+              placeholderNode = createPlaceholderNode(
+                `[[file:${metas['object']}]]`
+              ) as unknown as RootContent
+              if (!placeholderNode) {
+                shouldProcessNode = false
+                newChildren.push(node)
+              }
+            } catch (error) {
+              shouldProcessNode = false
+              newChildren.push(node)
+            }
           }
           break
         case 'symbol':
           if (metas['object']) {
-            const symbolObject = JSON.parse(
-              metas['object'].replace(/\\"/g, '"')
-            )
-            finalCommandText = `[[symbol:${JSON.stringify(symbolObject)}]]`
+            try {
+              placeholderNode = createPlaceholderNode(
+                `[[symbol:${metas['object']}]]`
+              ) as unknown as RootContent
+              if (!placeholderNode) {
+                shouldProcessNode = false
+                newChildren.push(node)
+              }
+            } catch (error) {
+              shouldProcessNode = false
+              newChildren.push(node)
+            }
           }
           break
         default:
           newChildren.push(node)
           continue
       }
 
+      if (!shouldProcessNode) {
+        continue
+      }
+
       if (
         prevNode &&
         prevNode.type === 'paragraph' &&
@@ -123,7 +127,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
           type: 'paragraph',
           children: [
             ...(prevNode.children || []),
-            { type: 'text', value: ` ${finalCommandText} ` },
+            placeholderNode || { type: 'text', value: ` ${finalCommandText} ` },
             ...(nextNode.children || [])
           ]
         } as RootContent)
@@ -136,7 +140,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
         newChildren.push({
           type: 'paragraph',
           children: [
-            { type: 'text', value: `${finalCommandText} ` },
+            placeholderNode || { type: 'text', value: `${finalCommandText} ` },
             ...(nextNode.children || [])
           ]
         } as RootContent)
@@ -145,14 +149,15 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
         prevNode.type === 'paragraph' &&
         isPrevNodeSameLine
       ) {
-        ;(prevNode.children || []).push({
-          type: 'text',
-          value: ` ${finalCommandText}`
-        })
+        ;(prevNode.children || []).push(
+          placeholderNode || { type: 'text', value: ` ${finalCommandText}` }
+        )
       } else {
         newChildren.push({
           type: 'paragraph',
-          children: [{ type: 'text', value: finalCommandText }]
+          children: [
+            placeholderNode || { type: 'text', value: finalCommandText }
+          ]
         } as RootContent)
       }
     } else {
@@ -163,7 +168,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
 }
 
 export function processContextCommand(input: string): string {
-  const processor = remark()
+  const processor = createRemarkProcessor()
   const ast = processor.parse(input) as Root
   ast.children = processCodeBlocksWithLabel(ast)
   return customAstToString(ast)
@@ -172,3 +177,51 @@ export function processContextCommand(input: string): string {
 export function convertContextBlockToPlaceholder(input: string): string {
   return processContextCommand(input)
 }
+
+/**
+ * Format an object into a markdown code block with proper metadata
+ * @param label The label for the code block (e.g., 'file', 'symbol')
+ * @param obj The object to format
+ * @param content The content to include in the code block
+ * @returns A formatted markdown code block string
+ */
+export function formatObjectToMarkdownBlock(
+  label: string,
+  obj: any,
+  content: string
+): string {
+  try {
+    const objJSON = JSON.stringify(obj)
+
+    const codeNode: Root = {
+      type: 'root',
+      children: [
+        {
+          type: 'code',
+          lang: 'context',
+          meta: `label=${label} object=${objJSON}`,
+          value: content
+        } as RootContent
+      ]
+    }
+
+    const processor = createRemarkProcessor()
+
+    const res = '\n' + processor.stringify(codeNode).trim() + '\n'
+    return res
+  } catch (error) {
+    return `\n*Error formatting ${label}*\n`
+  }
+}
+
+export interface PlaceholderNode extends Node {
+  type: 'placeholder'
+  value: string
+}
+
+export function createPlaceholderNode(value: string): PlaceholderNode {
+  return {
+    type: 'placeholder',
+    value: value
+  } as PlaceholderNode
+}

ee/tabby-ui/package.json

@@ -130,6 +130,7 @@
     "swr": "^2.2.4",
     "tabby-chat-panel": "workspace:*",
     "tippy.js": "^6.3.7",
+    "unified": "^11.0.5",
     "urql": "^4.0.6",
     "use-local-storage": "^3.0.0",
     "wonka": "^6.3.4",

ee/tabby-ui/test/utils/markdown.test.ts

@@ -0,0 +1,218 @@
+import { describe, expect, it } from 'vitest'
+import { formatObjectToMarkdownBlock } from '../../lib/utils/markdown'
+import { Filepath } from 'tabby-chat-panel/index';
+
+describe('formatObjectToMarkdownBlock - comprehensive tests', () => {
+  describe('filepath types with standard content', () => {
+    const jsContent = `// Example JavaScript code
+function example() {
+  const greeting = "Hello World!";
+  console.log(greeting);
+  return {
+    message: greeting,
+    timestamp: new Date().getTime()
+  };
+}
+
+// ES6 features
+const arrowFunc = () => {
+  return Promise.resolve(42);
+};
+`;
+
+    it('should format Unix path with git format', () => {
+      const unixGitObj = { 
+        kind: "git", 
+        filepath: '/home/user/projects/example.js', 
+        gitUrl: "https://github.com/tabbyml/tabby" 
+      } as Filepath;
+      
+      const result = formatObjectToMarkdownBlock('file', unixGitObj, jsContent);
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${JSON.stringify(unixGitObj)}`);
+      expect(result).toContain(jsContent);
+      expect(result).toContain('```');
+    });
+
+    it('should format Unix path with uri format', () => {
+      const unixUriObj = { 
+        kind: "uri", 
+        uri: '/home/user/projects/example.js' 
+      } as Filepath;
+      
+      const result = formatObjectToMarkdownBlock('file', unixUriObj, jsContent);
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${JSON.stringify(unixUriObj)}`);
+      expect(result).toContain(jsContent);
+      expect(result).toContain('```');
+    });
+
+    it('should format Windows path with uri format and backslashes', () => {
+      const winUriObj = { 
+        kind: "uri", 
+        uri: 'C:\\Users\\johndoe\\Projects\\example.js' 
+      } as Filepath;
+      
+      const result = formatObjectToMarkdownBlock('file', winUriObj, jsContent);
+      
+      const expectedJson = JSON.stringify(winUriObj).replace(/\\\\/g, '\\\\\\');
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${expectedJson}`);
+      expect(result).toContain(jsContent);
+      expect(result).toContain('```');
+    });
+  });
+
+  describe('markdown content handling', () => {
+    const markdownContent = `# Example Markdown Document
+
+This is a Markdown document with various elements.
+
+## JavaScript Code Example
+
+\`\`\`javascript
+function hello() {
+  console.log('Hello, world!');
+}
+\`\`\`
+
+## Rust Code Example
+
+\`\`\`rust
+fn main() {
+  println!("Hello from Rust!");
+}
+\`\`\`
+
+## Blockquotes and Lists
+
+> This is a blockquote
+
+* List item 1
+* List item 2
+  * Nested list item
+`;
+
+    it('should correctly handle markdown with nested code blocks', () => {
+      const gitObj = { 
+        kind: "git", 
+        filepath: '/home/user/docs/README.md', 
+        gitUrl: "https://github.com/tabbyml/tabby" 
+      } as Filepath;
+      
+      const result = formatObjectToMarkdownBlock('file', gitObj, markdownContent);
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${JSON.stringify(gitObj)}`);
+      expect(result).toContain('```javascript');
+      expect(result).toContain('```rust');
+      expect(result).toContain('# Example Markdown Document');
+      expect(result).toContain('```');
+    });
+
+    it('should correctly handle markdown with Windows paths', () => {
+      const winUriObj = { 
+        kind: "uri", 
+        uri: 'C:\\Users\\johndoe\\Documents\\README.md' 
+      } as Filepath;
+      
+      const result = formatObjectToMarkdownBlock('file', winUriObj, markdownContent);
+      
+      const expectedJson = JSON.stringify(winUriObj).replace(/\\\\/g, '\\\\\\');
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${expectedJson}`);
+      expect(result).toContain('```javascript');
+      expect(result).toContain('```rust');
+      expect(result).toContain('```');
+    });
+  });
+
+  describe('special cases', () => {
+    it('should handle path with additional metadata', () => {
+      const objWithMetadata = { 
+        kind: "git",
+        filepath: '/Users/johndoe/Developer/main.rs',
+        gitUrl: "https://github.com/tabbyml/tabby",
+        line: 5,
+        highlight: true
+      } as Filepath;
+      
+      const rustContent = `// Example Rust code
+fn main() {
+    println!("Hello, Rust!");
+}`;
+      
+      const result = formatObjectToMarkdownBlock('file', objWithMetadata, rustContent);
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${JSON.stringify(objWithMetadata)}`);
+      expect(result).toContain(rustContent);
+      expect(result).toContain('```');
+    });
+
+    it('should handle special characters in paths', () => {
+      const specialPathObj = { 
+        kind: "git",
+        filepath: '/Users/user/Projects/special-chars/file with spaces.js',
+        gitUrl: "https://github.com/tabbyml/tabby",
+        branch: 'feature/new-branch'
+      } as Filepath;
+      
+      const jsContent = 'console.log("Special characters test");';
+      
+      const result = formatObjectToMarkdownBlock('file', specialPathObj, jsContent);
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${JSON.stringify(specialPathObj)}`);
+      expect(result).toContain(jsContent);
+      expect(result).toContain('```');
+    });
+
+    it('should handle complex content types', () => {
+      const winObj = { 
+        kind: "uri", 
+        uri: 'D:\\Projects\\TypeScript\\interfaces.ts' 
+      } as Filepath;
+      
+      const tsContent = `/**
+ * User interface representing a person.
+ */
+interface User {
+  id: number;
+  name: string;
+  email: string;
+  isActive?: boolean; // Optional property
+}
+
+/**
+ * Create a new user with default values.
+ */
+function createUser(name: string, email: string): User {
+  return {
+    id: Math.floor(Math.random() * 1000),
+    name,
+    email,
+    isActive: true
+  };
+}
+
+// Test the function
+const newUser = createUser("John Doe", "john@example.com");
+console.log(newUser);
+`;
+      
+      const result = formatObjectToMarkdownBlock('file', winObj, tsContent);
+      
+      const expectedJson = JSON.stringify(winObj).replace(/\\\\/g, '\\\\\\');
+      
+      expect(result).toContain('```context label=file');
+      expect(result).toContain(`object=${expectedJson}`);
+      expect(result).toContain(tsContent);
+      expect(result).toContain('```');
+    });
+  });
+});