Skip to content

Commit 89d1414

Browse files
committed
feat(markdown): add formatObjectToMarkdownBlock and formatPlaceholder functions for improved markdown handling
- Introduced formatObjectToMarkdownBlock to format objects into markdown code blocks with metadata. - Added formatPlaceholder to create formatted placeholders for file and symbol types. - Updated processingPlaceholder to utilize the new formatting functions. - Enhanced custom AST stringification in markdown processing. - Added tests for new formatting functions and existing markdown utilities.
1 parent b52e1b2 commit 89d1414

File tree

4 files changed

+273
-21
lines changed

4 files changed

+273
-21
lines changed

ee/tabby-ui/components/chat/question-answer.tsx

+3-1
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,9 @@ function UserMessageCard(props: { message: UserMessage }) {
142142
}
143143

144144
const processedContent = useMemo(() => {
145-
return convertContextBlockToPlaceholder(message.content)
145+
const afterConvert = convertContextBlockToPlaceholder(message.content)
146+
console.log('afterConvert', afterConvert)
147+
return afterConvert
146148
}, [message.content])
147149

148150
return (

ee/tabby-ui/lib/utils/chat.ts

+4-6
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import {
2525
PLACEHOLDER_FILE_REGEX,
2626
PLACEHOLDER_SYMBOL_REGEX
2727
} from '../constants/regex'
28-
import { convertContextBlockToPlaceholder } from './markdown'
28+
import { convertContextBlockToPlaceholder, formatObjectToMarkdownBlock } from './markdown'
2929

3030
export const isCodeSourceContext = (kind: ContextSourceKind) => {
3131
return [
@@ -326,7 +326,7 @@ export async function processingPlaceholder(
326326
): Promise<string> {
327327
let processedMessage = message
328328
if (hasChangesCommand(processedMessage) && options.getChanges) {
329-
try {
329+
try {
330330
const changes = await options.getChanges({})
331331
const gitChanges = convertChangeItemsToContextContent(changes)
332332
processedMessage = processedMessage.replaceAll(
@@ -354,8 +354,7 @@ export async function processingPlaceholder(
354354
})
355355
let replacement = ''
356356
if (content) {
357-
const fileInfoJSON = JSON.stringify(fileInfo).replace(/"/g, '\\"')
358-
replacement = `\n\`\`\`context label='file' object='${fileInfoJSON}'\n${content}\n\`\`\`\n`
357+
replacement = formatObjectToMarkdownBlock('file', fileInfo, content)
359358
}
360359
processedMessage = processedMessage.replace(match[0], replacement)
361360
tempMessage = tempMessage.replace(match[0], replacement)
@@ -381,8 +380,7 @@ export async function processingPlaceholder(
381380
})
382381
let replacement = ''
383382
if (content) {
384-
const symbolInfoJSON = JSON.stringify(symbolInfo).replace(/"/g, '\\"')
385-
replacement = `\n\`\`\`context label='symbol' object='${symbolInfoJSON}'\n${content}\n\`\`\`\n`
383+
replacement = formatObjectToMarkdownBlock('symbol', symbolInfo, content)
386384
}
387385
processedMessage = processedMessage.replace(match[0], replacement)
388386
tempMessage = tempMessage.replace(match[0], replacement)

ee/tabby-ui/lib/utils/markdown.ts

+86-14
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,23 @@
11
import { Parent, Root, RootContent } from 'mdast'
22
import { remark } from 'remark'
33
import remarkStringify from 'remark-stringify'
4+
import { Options } from 'remark-stringify'
5+
6+
const REMARK_STRINGIFY_OPTIONS: Options = {
7+
bullet: '*',
8+
emphasis: '*',
9+
fences: true,
10+
listItemIndent: 'one',
11+
tightDefinitions: true
12+
}
13+
14+
15+
function createRemarkProcessor() {
16+
return remark().use(remarkStringify, REMARK_STRINGIFY_OPTIONS)
17+
}
418

519
/**
6-
* Custom stringification of AST to preserve special patterns
20+
* Custom stringification of AST using remarkStringify
721
* @param ast AST to stringify
822
* @returns Plain string representation
923
*/
@@ -27,8 +41,7 @@ function nodeToString(node: any): string {
2741
case 'text':
2842
return node.value
2943
default:
30-
const processor = remark().use(remarkStringify)
31-
return processor.stringify({ type: 'root', children: [node] }).trim()
44+
return createRemarkProcessor().stringify({ type: 'root', children: [node] }).trim()
3245
}
3346
}
3447

@@ -62,8 +75,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
6275
if (node.type === 'code' && node.meta) {
6376
node.meta?.split(' ').forEach(item => {
6477
const [key, rawValue] = item.split(/=(.+)/)
65-
const value = rawValue?.replace(/^['"]|['"]$/g, '') || ''
66-
metas[key] = value
78+
metas[key] = rawValue
6779
})
6880
}
6981

@@ -94,10 +106,11 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
94106
case 'file':
95107
if (metas['object']) {
96108
try {
97-
const fileObject = JSON.parse(
98-
metas['object'].replace(/\\"/g, '"').replace(/\\/g, '/')
99-
)
100-
finalCommandText = `[[file:${JSON.stringify(fileObject)}]]`
109+
finalCommandText = formatPlaceholder('file', metas['object'])
110+
if (!finalCommandText) {
111+
shouldProcessNode = false
112+
newChildren.push(node)
113+
}
101114
} catch (error) {
102115
shouldProcessNode = false
103116
newChildren.push(node)
@@ -107,10 +120,11 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
107120
case 'symbol':
108121
if (metas['object']) {
109122
try {
110-
const symbolObject = JSON.parse(
111-
metas['object'].replace(/\\"/g, '"').replace(/\\/g, '/')
112-
)
113-
finalCommandText = `[[symbol:${JSON.stringify(symbolObject)}]]`
123+
finalCommandText = formatPlaceholder('symbol', metas['object'])
124+
if (!finalCommandText) {
125+
shouldProcessNode = false
126+
newChildren.push(node)
127+
}
114128
} catch (error) {
115129
shouldProcessNode = false
116130
newChildren.push(node)
@@ -180,7 +194,7 @@ export function processCodeBlocksWithLabel(ast: Root): RootContent[] {
180194
}
181195

182196
export function processContextCommand(input: string): string {
183-
const processor = remark()
197+
const processor = createRemarkProcessor()
184198
const ast = processor.parse(input) as Root
185199
ast.children = processCodeBlocksWithLabel(ast)
186200
return customAstToString(ast)
@@ -189,3 +203,61 @@ export function processContextCommand(input: string): string {
189203
export function convertContextBlockToPlaceholder(input: string): string {
190204
return processContextCommand(input)
191205
}
206+
207+
/**
208+
* Format an object into a markdown code block with proper metadata
209+
* @param label The label for the code block (e.g., 'file', 'symbol')
210+
* @param obj The object to format
211+
* @param content The content to include in the code block
212+
* @returns A formatted markdown code block string
213+
*/
214+
export function formatObjectToMarkdownBlock(
215+
label: string,
216+
obj: any,
217+
content: string
218+
): string {
219+
try {
220+
// Convert the object to a JSON string
221+
const objJSON = JSON.stringify(obj)
222+
223+
const codeNode: Root = {
224+
type: 'root',
225+
children: [
226+
{
227+
type: 'code',
228+
lang: 'context',
229+
meta: `label=${label} object=${objJSON}`,
230+
value: content
231+
} as RootContent
232+
]
233+
}
234+
235+
const processor = createRemarkProcessor()
236+
237+
const res = '\n' + processor.stringify(codeNode).trim() + '\n'
238+
console.log('res', res)
239+
return res;
240+
} catch (error) {
241+
console.error(`Error formatting ${label} to markdown block:`, error)
242+
return `\n*Error formatting ${label}*\n`
243+
}
244+
}
245+
246+
247+
248+
/**
249+
* Format a placeholder with proper backslash escaping
250+
* @param type The type of placeholder (e.g., 'file', 'symbol')
251+
* @param objStr The string representation of the object
252+
* @returns The formatted placeholder text
253+
*/
254+
export function formatPlaceholder(type: string, objStr: string): string {
255+
if (!objStr) return ''
256+
257+
try {
258+
return `[[${type}:${objStr}]]`
259+
} catch (error) {
260+
console.error(`Error formatting ${type} placeholder:`, error)
261+
return ''
262+
}
263+
}
+180
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
import { describe, expect, it } from 'vitest'
2+
import {
3+
formatObjectToMarkdownBlock,
4+
escapeBackslashes,
5+
formatPlaceholder,
6+
customAstToString,
7+
processContextCommand
8+
} from '../../lib/utils/markdown'
9+
import { remark } from 'remark'
10+
import remarkStringify from 'remark-stringify'
11+
12+
describe('escapeBackslashes', () => {
13+
it('should escape backslashes in Windows paths', () => {
14+
const windowsPath = 'C:\\Users\\test\\Documents\\project\\file.js'
15+
const expected = 'C:\\\\Users\\\\test\\\\Documents\\\\project\\\\file.js'
16+
expect(escapeBackslashes(windowsPath)).toBe(expected)
17+
})
18+
19+
it('should handle unix paths properly', () => {
20+
const unixPath = '/home/user/projects/file.js'
21+
expect(escapeBackslashes(unixPath)).toBe(unixPath)
22+
})
23+
24+
it('should handle empty strings', () => {
25+
expect(escapeBackslashes('')).toBe('')
26+
})
27+
28+
it('should handle null or undefined', () => {
29+
expect(escapeBackslashes(null as any)).toBe('')
30+
expect(escapeBackslashes(undefined as any)).toBe('')
31+
})
32+
})
33+
34+
describe('formatPlaceholder', () => {
35+
it('should format a file placeholder with Windows path', () => {
36+
const objStr = JSON.stringify({ filepath: 'C:\\Users\\test\\file.js' })
37+
const expected = `[[file:${JSON.stringify({ filepath: 'C:\\\\Users\\\\test\\\\file.js' })}]]`
38+
expect(formatPlaceholder('file', objStr)).toBe(expected)
39+
})
40+
41+
it('should format a symbol placeholder with Unix path', () => {
42+
const objStr = JSON.stringify({ filepath: '/home/user/file.js', range: { start: 1, end: 5 } })
43+
expect(formatPlaceholder('symbol', objStr)).toBe(`[[symbol:${objStr}]]`)
44+
})
45+
46+
it('should handle empty object string', () => {
47+
expect(formatPlaceholder('file', '')).toBe('')
48+
})
49+
50+
it('should handle null or undefined', () => {
51+
expect(formatPlaceholder('file', null as any)).toBe('')
52+
expect(formatPlaceholder('symbol', undefined as any)).toBe('')
53+
})
54+
})
55+
56+
describe('formatObjectToMarkdownBlock', () => {
57+
it('should format object with JavaScript content', () => {
58+
const obj = { filepath: '/path/to/file.js' }
59+
const jsContent = `function test() {
60+
console.log("Hello World");
61+
return true;
62+
}`
63+
const result = formatObjectToMarkdownBlock('file', obj, jsContent)
64+
expect(result).toContain('```context label=file')
65+
expect(result).toContain(JSON.stringify(obj))
66+
expect(result).toContain(jsContent)
67+
})
68+
69+
it('should format object with Rust content', () => {
70+
const obj = { filepath: 'C:\\path\\to\\file.rs' }
71+
const rustContent = `fn main() {
72+
println!("Hello, world!");
73+
let x = 5;
74+
let y = 10;
75+
println!("x + y = {}", x + y);
76+
}`
77+
const result = formatObjectToMarkdownBlock('file', obj, rustContent)
78+
expect(result).toContain('```context label=file')
79+
expect(result).toContain(JSON.stringify(obj))
80+
expect(result).toContain(rustContent)
81+
})
82+
83+
it('should handle content with markdown code blocks', () => {
84+
const obj = { filepath: '/path/to/README.md' }
85+
const markdownContent = `# Title
86+
Some text here
87+
\`\`\`js
88+
const x = 5;
89+
\`\`\`
90+
More text`
91+
const result = formatObjectToMarkdownBlock('file', obj, markdownContent)
92+
expect(result).toContain('```context label=file')
93+
// Verify backticks are properly handled
94+
expect(result).not.toContain('```js')
95+
expect(result).toContain('` ` `js')
96+
})
97+
98+
it('should handle content ending with backtick', () => {
99+
const obj = { filepath: '/path/to/file.md' }
100+
const content = 'This is some text with a backtick at the end: `'
101+
const result = formatObjectToMarkdownBlock('file', obj, content)
102+
// Should append a space after the backtick
103+
expect(result.includes('backtick at the end: ` \n```')).toBeTruthy()
104+
})
105+
106+
it('should handle complex nested code blocks', () => {
107+
const obj = { filepath: '/path/to/doc.md' }
108+
const complexContent = `# Documentation
109+
\`\`\`html
110+
<div>
111+
<pre>
112+
\`\`\`typescript
113+
function test() {
114+
return true;
115+
}
116+
\`\`\`
117+
</pre>
118+
</div>
119+
\`\`\`
120+
`
121+
const result = formatObjectToMarkdownBlock('file', obj, complexContent)
122+
// Check that nested code blocks are handled properly
123+
expect(result).toContain('` ` `html')
124+
expect(result).toContain('` ` `typescript')
125+
})
126+
127+
it('should handle Windows paths in object properties', () => {
128+
const obj = { filepath: 'C:\\Users\\test\\file.txt' }
129+
const content = 'Simple text content'
130+
const result = formatObjectToMarkdownBlock('file', obj, content)
131+
expect(result).toContain(JSON.stringify(obj))
132+
expect(result).toContain(content)
133+
})
134+
135+
it('should handle error in object serialization', () => {
136+
// Create a circular reference that will cause JSON.stringify to fail
137+
const circularObj: any = { name: 'test' }
138+
circularObj.self = circularObj
139+
140+
const content = 'Some content'
141+
const result = formatObjectToMarkdownBlock('file', circularObj, content)
142+
expect(result).toBe('\n*Error formatting file*\n')
143+
})
144+
})
145+
146+
describe('customAstToString', () => {
147+
it('should properly stringify a simple markdown AST', () => {
148+
// 使用标准的remark处理器解析文本
149+
const markdownText = '# Title\n\nThis is a paragraph.\n\n* List item 1\n* List item 2'
150+
const ast = remark().parse(markdownText)
151+
152+
const result = customAstToString(ast)
153+
154+
// 验证最基本的结构被保留
155+
expect(result).toContain('# Title')
156+
expect(result).toContain('This is a paragraph')
157+
expect(result).toContain('* List item 1')
158+
expect(result).toContain('* List item 2')
159+
})
160+
161+
it('should preserve code blocks in the AST', () => {
162+
// 使用标准的remark处理器解析文本
163+
const markdownText = '```js\nconst x = 5;\n```'
164+
const ast = remark().parse(markdownText)
165+
166+
const result = customAstToString(ast)
167+
168+
expect(result).toContain('```js')
169+
expect(result).toContain('const x = 5;')
170+
expect(result).toContain('```')
171+
})
172+
173+
it('should maintain processContextCommand functionality', () => {
174+
const input = '```context label=file object={"filepath":"/path/to/file.js"}\nconst x = 5;\n```'
175+
const result = processContextCommand(input)
176+
177+
expect(result).toContain('[[file:')
178+
expect(result).toContain('{"filepath":"/path/to/file.js"}')
179+
})
180+
})

0 commit comments

Comments
 (0)