feat(markdown): implement formatting functions for markdown blocks and add tests

- Added `formatObjectToMarkdownBlock`, `escapeBackslashes`, and `formatPlaceholder` functions to handle object formatting and escaping in markdown.
- Refactored `customAstToString` to utilize a dedicated remark processor.
- Introduced unit tests for the new formatting functions to ensure proper functionality and error handling.

Author: Sma1lboy

ee/tabby-ui/components/chat/question-answer.tsx

@@ -142,7 +142,9 @@ function UserMessageCard(props: { message: UserMessage }) {
   }
 
   const processedContent = useMemo(() => {
-    return convertContextBlockToPlaceholder(message.content)
+   const afterConvert = convertContextBlockToPlaceholder(message.content)
+   console.log('afterConvert', afterConvert)
+   return afterConvert
   }, [message.content])
 
   return (

ee/tabby-ui/lib/utils/chat.ts

@@ -25,7 +25,7 @@ import {
   PLACEHOLDER_FILE_REGEX,
   PLACEHOLDER_SYMBOL_REGEX
 } from '../constants/regex'
-import { convertContextBlockToPlaceholder } from './markdown'
+import { convertContextBlockToPlaceholder, formatObjectToMarkdownBlock } from './markdown'
 
 export const isCodeSourceContext = (kind: ContextSourceKind) => {
   return [
@@ -326,7 +326,7 @@ export async function processingPlaceholder(
 ): Promise<string> {
   let processedMessage = message
   if (hasChangesCommand(processedMessage) && options.getChanges) {
-    try {
+    try { 
       const changes = await options.getChanges({})
       const gitChanges = convertChangeItemsToContextContent(changes)
       processedMessage = processedMessage.replaceAll(
@@ -354,8 +354,7 @@ export async function processingPlaceholder(
         })
         let replacement = ''
         if (content) {
-          const fileInfoJSON = JSON.stringify(fileInfo).replace(/"/g, '\\"')
-          replacement = `\n\`\`\`context label='file' object='${fileInfoJSON}'\n${content}\n\`\`\`\n`
+          replacement = formatObjectToMarkdownBlock('file', fileInfo, content)
         }
         processedMessage = processedMessage.replace(match[0], replacement)
         tempMessage = tempMessage.replace(match[0], replacement)
@@ -381,8 +380,7 @@ export async function processingPlaceholder(
         })
         let replacement = ''
         if (content) {
-          const symbolInfoJSON = JSON.stringify(symbolInfo).replace(/"/g, '\\"')
-          replacement = `\n\`\`\`context label='symbol' object='${symbolInfoJSON}'\n${content}\n\`\`\`\n`
+          replacement = formatObjectToMarkdownBlock('symbol', symbolInfo, content)
         }
         processedMessage = processedMessage.replace(match[0], replacement)
         tempMessage = tempMessage.replace(match[0], replacement)

ee/tabby-ui/lib/utils/markdown.ts

@@ -1,56 +1,29 @@
 import { Parent, Root, RootContent } from 'mdast'
 import { remark } from 'remark'
 import remarkStringify from 'remark-stringify'
+import { Options } from 'remark-stringify'
 
-/**
- * Custom stringification of AST to preserve special patterns
- * @param ast AST to stringify
- * @returns Plain string representation
- */
-export function customAstToString(ast: Root): string {
-  let result = ''
-  for (const node of ast.children) {
-    result += nodeToString(node) + '\n'
-  }
-  return result.trim()
+const REMARK_STRINGIFY_OPTIONS: Options = {
+  bullet: '*',
+  emphasis: '*',
+  fences: true,
+  listItemIndent: 'one',
+  tightDefinitions: true
 }
 
-/**
- * Convert a single node to string
- * @param node AST node
- * @returns String representation
- */
-function nodeToString(node: any): string {
-  switch (node.type) {
-    case 'paragraph':
-      return paragraphToString(node)
-    case 'text':
-      return node.value
-    default:
-      const processor = remark().use(remarkStringify)
-      return processor.stringify({ type: 'root', children: [node] }).trim()
-  }
-}
 
-/**
- * Convert paragraph node to string
- * @param node Paragraph node
- * @returns String representation
- */
-function paragraphToString(node: any): string {
-  return childrenToString(node)
+function createRemarkProcessor() {
+  return remark().use(remarkStringify, REMARK_STRINGIFY_OPTIONS)
 }
 
 /**
- * Process children of a node and join them
- * @param node Parent node
- * @returns Combined string of all children
+ * Custom stringification of AST using remarkStringify
+ * @param ast AST to stringify
+ * @returns Plain string representation
  */
-function childrenToString(node: any): string {
-  if (!node.children || node.children.length === 0) {
-    return ''
-  }
-  return node.children.map((child: any) => nodeToString(child)).join('')
+export function customAstToString(ast: Root): string {
+  const processor = createRemarkProcessor()
+  return processor.stringify(ast).trim()
 }
 
 export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
@@ -62,8 +35,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
     if (node.type === 'code' && node.meta) {
       node.meta?.split(' ').forEach(item => {
         const [key, rawValue] = item.split(/=(.+)/)
-        const value = rawValue?.replace(/^['"]|['"]$/g, '') || ''
-        metas[key] = value
+        metas[key] = rawValue
       })
     }
 
@@ -94,10 +66,11 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
         case 'file':
           if (metas['object']) {
             try {
-              const fileObject = JSON.parse(
-                metas['object'].replace(/\\"/g, '"').replace(/\\/g, '/')
-              )
-              finalCommandText = `[[file:${JSON.stringify(fileObject)}]]`
+              finalCommandText = formatPlaceholder('file', metas['object'])
+              if (!finalCommandText) {
+                shouldProcessNode = false
+                newChildren.push(node)
+              }
             } catch (error) {
               shouldProcessNode = false
               newChildren.push(node)
@@ -107,10 +80,11 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
         case 'symbol':
           if (metas['object']) {
             try {
-              const symbolObject = JSON.parse(
-                metas['object'].replace(/\\"/g, '"').replace(/\\/g, '/')
-              )
-              finalCommandText = `[[symbol:${JSON.stringify(symbolObject)}]]`
+              finalCommandText = formatPlaceholder('symbol', metas['object'])
+              if (!finalCommandText) {
+                shouldProcessNode = false
+                newChildren.push(node)
+              }
             } catch (error) {
               shouldProcessNode = false
               newChildren.push(node)
@@ -180,7 +154,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
 }
 
 export function processContextCommand(input: string): string {
-  const processor = remark()
+  const processor = createRemarkProcessor()
   const ast = processor.parse(input) as Root
   ast.children = processCodeBlocksWithLabel(ast)
   return customAstToString(ast)
@@ -189,3 +163,82 @@ export function processContextCommand(input: string): string {
 export function convertContextBlockToPlaceholder(input: string): string {
   return processContextCommand(input)
 }
+
+/**
+ * Format an object into a markdown code block with proper metadata
+ * @param label The label for the code block (e.g., 'file', 'symbol')
+ * @param obj The object to format
+ * @param content The content to include in the code block
+ * @returns A formatted markdown code block string
+ */
+export function formatObjectToMarkdownBlock(
+  label: string,
+  obj: any,
+  content: string
+): string {
+  try {
+    // Convert the object to a JSON string
+    const objJSON = JSON.stringify(obj)
+    
+    let safeContent = content
+    
+    if (safeContent.includes('```')) {
+      safeContent = safeContent.replace(/```/g, '` ` `')
+    }
+    
+    if (safeContent.trimEnd().endsWith('`')) {
+      safeContent = safeContent + ' '
+    }
+    
+    const codeNode: Root = {
+      type: 'root',
+      children: [
+        {
+          type: 'code',
+          lang: 'context',
+          meta: `label=${label} object=${objJSON}`,
+          value: safeContent
+        } as RootContent
+      ]
+    }
+    
+    const processor = createRemarkProcessor()
+    
+    const res = '\n' + processor.stringify(codeNode).trim() + '\n'
+    console.log('res', res)
+    return res;
+  } catch (error) {
+    console.error(`Error formatting ${label} to markdown block:`, error)
+    return `\n*Error formatting ${label}*\n`
+  }
+}
+
+/**
+ * Escape backslashes in a string to handle Windows filepath issues
+ * @param str The string to process
+ * @returns The string with properly escaped backslashes
+ */
+export function escapeBackslashes(str: string): string {
+  if (!str) return ''
+  return str.replace(/\\/g, '\\\\')
+}
+
+/**
+ * Format a placeholder with proper backslash escaping
+ * @param type The type of placeholder (e.g., 'file', 'symbol')
+ * @param objStr The string representation of the object
+ * @returns The formatted placeholder text
+ */
+export function formatPlaceholder(type: string, objStr: string): string {
+  if (!objStr) return ''
+  
+  try {
+    console.log('objStr', JSON.parse(objStr)) 
+    // Escape backslashes to ensure they're preserved
+    const escapedStr = escapeBackslashes(objStr)
+    return `[[${type}:${escapedStr}]]`
+  } catch (error) {
+    console.error(`Error formatting ${type} placeholder:`, error)
+    return ''
+  }
+}