Skip to content

Commit edc1504

Browse files
lukastaegertlukastaegert
authored
Remove inlining logic in AST (de-)serializer (#5455)
* Do not inline attributes to reduce complexity * If the reference of a list is 0, treat it as an empty list * Extend zero logic to stateful lists * Inline reference position update into string conversions * Unify how flags are added * Lint imports
1 parent 1b85663 commit edc1504

14 files changed

+1015
-884
lines changed

‎rust/parse_ast/src/convert_ast/converter.rs

+373-249
Large diffs are not rendered by default.

‎rust/parse_ast/src/convert_ast/converter/ast_constants.rs

+223-165
Large diffs are not rendered by default.

‎rust/parse_ast/src/error_emit.rs

+10-5
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ use parking_lot::Mutex;
55
use swc_common::errors::{DiagnosticBuilder, Emitter, Handler, Level, HANDLER};
66
use swc_ecma_ast::Program;
77

8+
use crate::convert_ast::converter::ast_constants::PARSE_ERROR_MESSAGE_OFFSET;
89
use crate::convert_ast::converter::{
9-
ast_constants::{PARSE_ERROR_RESERVED_BYTES, TYPE_PARSE_ERROR_INLINED_MESSAGE},
10-
convert_string,
10+
ast_constants::{PARSE_ERROR_RESERVED_BYTES, TYPE_PARSE_ERROR},
11+
convert_string, update_reference_position,
1112
};
1213

1314
#[derive(Clone, Default)]
@@ -68,22 +69,26 @@ where
6869
}
6970

7071
fn create_error_buffer(wr: &Writer, code: &str) -> Vec<u8> {
71-
let mut buffer = TYPE_PARSE_ERROR_INLINED_MESSAGE.to_vec();
7272
let mut lock = wr.0.lock();
7373
let error_buffer = take(&mut *lock);
7474
let pos = u32::from_ne_bytes(error_buffer[0..4].try_into().unwrap());
7575
let mut utf_16_pos: u32 = 0;
76+
// convert utf-8 to utf-16 inline
7677
for (utf_8_pos, char) in code.char_indices() {
7778
if (utf_8_pos as u32) == pos {
7879
break;
7980
}
8081
utf_16_pos += char.len_utf16() as u32;
8182
}
83+
// type
84+
let mut buffer = TYPE_PARSE_ERROR.to_vec();
8285
// start
8386
buffer.extend_from_slice(&utf_16_pos.to_ne_bytes());
8487
// end
85-
buffer.resize(buffer.len() + PARSE_ERROR_RESERVED_BYTES, 0);
86-
// message
88+
let end_position = buffer.len();
89+
buffer.resize(end_position + PARSE_ERROR_RESERVED_BYTES, 0);
90+
// message, the string is already converted to a buffer via convert_string
91+
update_reference_position(&mut buffer, end_position + PARSE_ERROR_MESSAGE_OFFSET);
8792
buffer.extend_from_slice(&error_buffer[4..]);
8893
buffer
8994
}

‎rust/parse_ast/src/lib.rs

+12-8
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,22 @@
11
use std::panic::{catch_unwind, AssertUnwindSafe};
22

3-
use convert_ast::converter::ast_constants::{
4-
PANIC_ERROR_RESERVED_BYTES, TYPE_PANIC_ERROR_INLINED_MESSAGE,
5-
};
6-
use convert_ast::converter::{convert_string, AstConverter};
73
use swc_common::sync::Lrc;
84
use swc_common::{FileName, FilePathMapping, Globals, SourceMap, GLOBALS};
95
use swc_compiler_base::parse_js;
106
use swc_compiler_base::IsModule;
117
use swc_ecma_ast::EsVersion;
128
use swc_ecma_parser::{EsConfig, Syntax};
139

10+
use convert_ast::converter::ast_constants::{PANIC_ERROR_RESERVED_BYTES, TYPE_PANIC_ERROR};
11+
use convert_ast::converter::{convert_string, AstConverter};
12+
use error_emit::try_with_handler;
13+
1414
use crate::convert_ast::annotations::SequentialComments;
15+
use crate::convert_ast::converter::ast_constants::PANIC_ERROR_MESSAGE_OFFSET;
16+
use crate::convert_ast::converter::update_reference_position;
1517

1618
mod convert_ast;
1719

18-
use error_emit::try_with_handler;
19-
2020
mod error_emit;
2121

2222
pub fn parse_ast(code: String, allow_return_outside_function: bool) -> Vec<u8> {
@@ -63,9 +63,13 @@ pub fn parse_ast(code: String, allow_return_outside_function: bool) -> Vec<u8> {
6363
} else {
6464
"Unknown rust panic message"
6565
};
66-
let mut buffer = TYPE_PANIC_ERROR_INLINED_MESSAGE.to_vec();
66+
// type
67+
let mut buffer = TYPE_PANIC_ERROR.to_vec();
6768
// reserve for start and end even though they are unused
68-
buffer.resize(buffer.len() + 4 + PANIC_ERROR_RESERVED_BYTES, 0);
69+
let end_position = buffer.len() + 4;
70+
buffer.resize(end_position + PANIC_ERROR_RESERVED_BYTES, 0);
71+
// message
72+
update_reference_position(&mut buffer, end_position + PANIC_ERROR_MESSAGE_OFFSET);
6973
convert_string(&mut buffer, msg);
7074
buffer
7175
})

‎scripts/ast-types.js

+23-61
Original file line numberDiff line numberDiff line change
@@ -158,19 +158,22 @@ export const AST_NODES = {
158158
ClassBody: {
159159
fields: [['body', 'NodeList']],
160160
scriptedFields: {
161-
body: `const length = buffer[$position];
161+
body: ` const bodyPosition = $position;
162162
const body: (MethodDefinition | PropertyDefinition)[] = (node.body = []);
163-
for (let index = 0; index < length; index++) {
164-
const nodePosition = buffer[$position + 1 + index];
165-
body.push(
166-
convertNode(
167-
node,
168-
(buffer[nodePosition + 3] & 1) === 0 ? scope.instanceScope : scope,
169-
nodePosition,
170-
buffer,
171-
readString
172-
)
173-
);
163+
if (bodyPosition) {
164+
const length = buffer[bodyPosition];
165+
for (let index = 0; index < length; index++) {
166+
const nodePosition = buffer[bodyPosition + 1 + index];
167+
body.push(
168+
convertNode(
169+
node,
170+
(buffer[nodePosition + 3] & 1) === 0 ? scope.instanceScope : scope,
171+
nodePosition,
172+
buffer,
173+
readString
174+
)
175+
);
176+
}
174177
}`
175178
}
176179
},
@@ -638,56 +641,15 @@ export const AST_NODES = {
638641
}
639642
};
640643

641-
export const astNodeNamesWithFieldOrder = Object.entries(AST_NODES).map(([name, node]) => {
642-
/** @type {FieldWithType[]} */
643-
const fields =
644-
(node.hasSameFieldsAs ? AST_NODES[node.hasSameFieldsAs].fields : node.fields) || [];
645-
/** @type {FieldWithType[]} */
646-
const allFields = [];
647-
/** @type {FieldWithType[]} */
648-
const reservedFields = [];
649-
/** @type {FieldWithType|null|undefined} */
650-
let inlinedVariableField = undefined;
651-
for (const field of fields) {
652-
allFields.push(field);
653-
switch (field[1]) {
654-
case 'Annotations':
655-
case 'InvalidAnnotations':
656-
case 'String':
657-
case 'NodeList':
658-
case 'Node': {
659-
if (inlinedVariableField === undefined) {
660-
inlinedVariableField = field;
661-
} else {
662-
reservedFields.push(field);
663-
}
664-
break;
665-
}
666-
case 'OptionalNode': {
667-
// Optional nodes cannot be inlined, but they also cannot be parsed
668-
// out-of-order, so nothing is inlined as the inlined node is always
669-
// parsed first.
670-
if (inlinedVariableField === undefined) {
671-
inlinedVariableField = null;
672-
}
673-
reservedFields.push(field);
674-
break;
675-
}
676-
case 'OptionalString':
677-
case 'FixedString':
678-
case 'Float': {
679-
reservedFields.push(field);
680-
break;
681-
}
682-
default: {
683-
throw new Error(`Unknown field type ${field[0]}`);
684-
}
685-
}
686-
}
644+
/** @type { {name: string; fields: FieldWithType[]; node: NodeDescription; originalNode: NodeDescription;}[] } */
645+
export const astNodeNamesWithFieldOrder = Object.entries(AST_NODES).map(([name, originalNode]) => {
646+
const node = originalNode.hasSameFieldsAs
647+
? AST_NODES[originalNode.hasSameFieldsAs]
648+
: originalNode;
687649
return {
688-
allFields,
689-
inlinedVariableField,
650+
fields: node.fields || [],
690651
name,
691-
reservedFields
652+
node,
653+
originalNode
692654
};
693655
});

‎scripts/generate-buffer-parsers.js

+68-78
Original file line numberDiff line numberDiff line change
@@ -1,96 +1,84 @@
11
import { writeFile } from 'node:fs/promises';
2-
import { AST_NODES, astNodeNamesWithFieldOrder } from './ast-types.js';
3-
import { getNode } from './generate-buffer-to-ast.js';
2+
import { astNodeNamesWithFieldOrder } from './ast-types.js';
43
import { firstLetterLowercase, lintTsFile } from './helpers.js';
54

65
const bufferParsersFile = new URL('../src/ast/bufferParsers.ts', import.meta.url);
76

8-
const nodeTypes = astNodeNamesWithFieldOrder.map(({ name }) => getNode(name).astType || name);
7+
const nodeTypes = astNodeNamesWithFieldOrder.map(({ name, node }) => node.astType || name);
98

109
const nodeTypeImports = nodeTypes.map(name => `import ${name} from './nodes/${name}';`);
1110
const nodeTypeStrings = nodeTypes.map(name => `\t'${name}'`);
1211

13-
const jsConverters = astNodeNamesWithFieldOrder.map(
14-
({ name, inlinedVariableField, reservedFields, allFields }) => {
15-
const node = getNode(name);
16-
const readStringArgument = allFields.some(([, fieldType]) =>
17-
['Node', 'OptionalNode', 'NodeList', 'String', 'FixedString', 'OptionalString'].includes(
18-
fieldType
19-
)
12+
const jsConverters = astNodeNamesWithFieldOrder.map(({ name, fields, node, originalNode }) => {
13+
const readStringArgument = fields.some(([, fieldType]) =>
14+
['Node', 'OptionalNode', 'NodeList', 'String', 'FixedString', 'OptionalString'].includes(
15+
fieldType
2016
)
21-
? ', readString'
22-
: '';
23-
/** @type {string[]} */
24-
const definitions = [];
25-
let offset = 0;
26-
let needsBuffer = false;
27-
let needsScope = false;
28-
if (node.flags) {
29-
offset++;
30-
needsBuffer = true;
31-
definitions.push(
32-
'const flags = buffer[position];\n',
33-
...node.flags.map((flagName, index) => {
34-
let assignmentLeftHand = node.baseForAdditionalFields?.includes(flagName)
35-
? `const ${flagName} = `
36-
: '';
37-
if (!node.hiddenFields?.includes(flagName)) {
38-
assignmentLeftHand += `node.${flagName} = `;
39-
}
40-
return `${assignmentLeftHand}(flags & ${1 << index}) === ${1 << index};`;
41-
})
42-
);
43-
}
44-
for (const [index, field] of reservedFields.entries()) {
45-
const fieldDefinition = getFieldDefinition(field, name, offset + index, false);
46-
needsBuffer = true;
47-
needsScope ||= fieldDefinition.needsScope;
48-
definitions.push(`${fieldDefinition.definition}\n`);
49-
}
50-
offset += reservedFields.length;
51-
if (inlinedVariableField) {
52-
const fieldDefinition = getFieldDefinition(inlinedVariableField, name, offset, true);
53-
needsBuffer = true;
54-
needsScope ||= fieldDefinition.needsScope;
55-
definitions.push(`${fieldDefinition.definition}\n`);
56-
}
57-
for (const [fieldName, fieldValue] of Object.entries(node.additionalFields || {})) {
58-
definitions.push(`node.${fieldName} = ${fieldValue};\n`);
59-
}
60-
for (const [fieldName, fallbackName] of Object.entries(node.optionalFallback || {})) {
61-
needsScope = true;
62-
definitions.push(
63-
`node.${fieldName} = ${fieldName}Position === 0 ? node.${fallbackName} : convertNode(node, scope, ${fieldName}Position, buffer, readString);\n`
64-
);
65-
}
66-
if (needsScope) {
67-
definitions.unshift('const {scope} = node;');
68-
}
69-
/** @type {string[]} */
70-
const parameters = [];
71-
if (definitions.length > 0) {
72-
parameters.push(`node: ${node.astType || name}`);
73-
if (needsBuffer) {
74-
parameters.push(`position, buffer${readStringArgument}`);
75-
}
17+
)
18+
? ', readString'
19+
: '';
20+
/** @type {string[]} */
21+
const definitions = [];
22+
let offset = 0;
23+
let needsBuffer = false;
24+
let needsScope = false;
25+
if (node.flags) {
26+
offset++;
27+
needsBuffer = true;
28+
definitions.push(
29+
'const flags = buffer[position];\n',
30+
...node.flags.map((flagName, index) => {
31+
let assignmentLeftHand = node.baseForAdditionalFields?.includes(flagName)
32+
? `const ${flagName} = `
33+
: '';
34+
if (!node.hiddenFields?.includes(flagName)) {
35+
assignmentLeftHand += `node.${flagName} = `;
36+
}
37+
return `${assignmentLeftHand}(flags & ${1 << index}) === ${1 << index};`;
38+
})
39+
);
40+
}
41+
for (const [index, field] of fields.entries()) {
42+
const fieldDefinition = getFieldDefinition(field, node, originalNode, offset + index);
43+
needsBuffer = true;
44+
needsScope ||= fieldDefinition.needsScope;
45+
definitions.push(`${fieldDefinition.definition}\n`);
46+
}
47+
offset += fields.length;
48+
for (const [fieldName, fieldValue] of Object.entries(node.additionalFields || {})) {
49+
definitions.push(`node.${fieldName} = ${fieldValue};\n`);
50+
}
51+
for (const [fieldName, fallbackName] of Object.entries(node.optionalFallback || {})) {
52+
needsScope = true;
53+
definitions.push(
54+
`node.${fieldName} = ${fieldName}Position === 0 ? node.${fallbackName} : convertNode(node, scope, ${fieldName}Position, buffer, readString);\n`
55+
);
56+
}
57+
if (needsScope) {
58+
definitions.unshift('const {scope} = node;');
59+
}
60+
/** @type {string[]} */
61+
const parameters = [];
62+
if (definitions.length > 0) {
63+
parameters.push(`node: ${node.astType || name}`);
64+
if (needsBuffer) {
65+
parameters.push(`position, buffer${readStringArgument}`);
7666
}
77-
return `function ${firstLetterLowercase(name)} (${parameters.join(', ')}) {
78-
${definitions.join('')}}`;
7967
}
80-
);
68+
return `function ${firstLetterLowercase(name)} (${parameters.join(', ')}) {
69+
${definitions.join('')}}`;
70+
});
8171

8272
/**
83-
* @param {import('./ast-types.js').FieldWithType} field
84-
* @param {string} name
73+
* @param {import("./ast-types.js").FieldWithType} field
74+
* @param {import("./ast-types.js").NodeDescription} node
75+
* @param {import("./ast-types.js").NodeDescription} originalNode
8576
* @param {number} offset
86-
* @param {boolean} isInlined
8777
* @returns {{definition: string, needsScope: boolean}}
8878
*/
89-
function getFieldDefinition([fieldName, fieldType], name, offset, isInlined) {
90-
const originalNode = AST_NODES[name];
91-
const node = getNode(name);
79+
function getFieldDefinition([fieldName, fieldType], node, originalNode, offset) {
9280
const getPosition = offset > 0 ? `position + ${offset}` : 'position';
93-
const dataStart = isInlined ? getPosition : `buffer[${getPosition}]`;
81+
const dataStart = `buffer[${getPosition}]`;
9482
if (node.scriptedFields?.[fieldName]) {
9583
return {
9684
definition: node.scriptedFields?.[fieldName].replace(/\$position/g, dataStart),
@@ -114,7 +102,7 @@ function getFieldDefinition([fieldName, fieldType], name, offset, isInlined) {
114102
};
115103
}
116104
case 'OptionalNode': {
117-
let definition = `const ${fieldName}Position = buffer[${getPosition}];`;
105+
let definition = `const ${fieldName}Position = ${dataStart};`;
118106
let needsScope = false;
119107
if (!node.optionalFallback?.[fieldName]) {
120108
needsScope = true;
@@ -158,13 +146,13 @@ function getFieldDefinition([fieldName, fieldType], name, offset, isInlined) {
158146
}
159147
case 'OptionalString': {
160148
return {
161-
definition: `const ${fieldName}Position = buffer[${getPosition}];\n${assignmentLeftHand}${fieldName}Position === 0 ? undefined : convertString(${fieldName}Position, buffer, readString)${typeCastString};`,
149+
definition: `const ${fieldName}Position = ${dataStart};\n${assignmentLeftHand}${fieldName}Position === 0 ? undefined : convertString(${fieldName}Position, buffer, readString)${typeCastString};`,
162150
needsScope: false
163151
};
164152
}
165153
case 'FixedString': {
166154
return {
167-
definition: `${assignmentLeftHand}FIXED_STRINGS[buffer[${getPosition}]]${typeCastString};`,
155+
definition: `${assignmentLeftHand}FIXED_STRINGS[${dataStart}]${typeCastString};`,
168156
needsScope: false
169157
};
170158
}
@@ -186,6 +174,7 @@ const bufferParsers = `// This file is generated by scripts/generate-ast-convert
186174
import type * as estree from 'estree';
187175
import type { AstContext } from '../Module';
188176
import { convertAnnotations, convertString } from '../utils/astConverterHelpers';
177+
import { EMPTY_ARRAY } from '../utils/blank';
189178
import { convertNode as convertJsonNode } from '../utils/bufferToAst';
190179
import FIXED_STRINGS from '../utils/convert-ast-strings';
191180
import type { ReadString } from '../utils/getReadStringFunction';
@@ -242,6 +231,7 @@ function convertNode(parent: Node | { context: AstContext; type: string }, paren
242231
}
243232
244233
function convertNodeList(parent: Node | { context: AstContext; type: string }, parentScope: ChildScope, position: number, buffer: Uint32Array, readString: ReadString): any[] {
234+
if (position === 0) return EMPTY_ARRAY as never[];
245235
const length = buffer[position++];
246236
const list: any[] = [];
247237
for (let index = 0; index < length; index++) {

0 commit comments

Comments
 (0)