Skip to content

Commit fe71462

Browse files
HejsilHejsil
committed
std.zig.parser now accept both string and multiline string for strings
Related #909 Allows it to parse `std/special/compiler_rt/aullrem.zig`, `std/special/compiler_rt/aulldiv.zig` and `std/math/x86_64/sqrt.zig`
1 parent 28ea364 commit fe71462

File tree

2 files changed

+81
-54
lines changed

2 files changed

+81
-54
lines changed

‎std/zig/ast.zig

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1608,7 +1608,7 @@ pub const NodeThisLiteral = struct {
16081608
pub const NodeAsmOutput = struct {
16091609
base: Node,
16101610
symbolic_name: &NodeIdentifier,
1611-
constraint: &NodeStringLiteral,
1611+
constraint: &Node,
16121612
kind: Kind,
16131613

16141614
const Kind = union(enum) {
@@ -1622,7 +1622,7 @@ pub const NodeAsmOutput = struct {
16221622
if (i < 1) return &self.symbolic_name.base;
16231623
i -= 1;
16241624

1625-
if (i < 1) return &self.constraint.base;
1625+
if (i < 1) return self.constraint;
16261626
i -= 1;
16271627

16281628
switch (self.kind) {
@@ -1654,7 +1654,7 @@ pub const NodeAsmOutput = struct {
16541654
pub const NodeAsmInput = struct {
16551655
base: Node,
16561656
symbolic_name: &NodeIdentifier,
1657-
constraint: &NodeStringLiteral,
1657+
constraint: &Node,
16581658
expr: &Node,
16591659

16601660
pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node {
@@ -1663,7 +1663,7 @@ pub const NodeAsmInput = struct {
16631663
if (i < 1) return &self.symbolic_name.base;
16641664
i -= 1;
16651665

1666-
if (i < 1) return &self.constraint.base;
1666+
if (i < 1) return self.constraint;
16671667
i -= 1;
16681668

16691669
if (i < 1) return self.expr;
@@ -1685,11 +1685,11 @@ pub const NodeAsm = struct {
16851685
base: Node,
16861686
asm_token: Token,
16871687
is_volatile: bool,
1688-
template: Token,
1688+
template: &Node,
16891689
//tokens: ArrayList(AsmToken),
16901690
outputs: ArrayList(&NodeAsmOutput),
16911691
inputs: ArrayList(&NodeAsmInput),
1692-
cloppers: ArrayList(&NodeStringLiteral),
1692+
cloppers: ArrayList(&Node),
16931693
rparen: Token,
16941694

16951695
pub fn iterate(self: &NodeAsm, index: usize) ?&Node {
@@ -1701,7 +1701,7 @@ pub const NodeAsm = struct {
17011701
if (i < self.inputs.len) return &self.inputs.at(index).base;
17021702
i -= self.inputs.len;
17031703

1704-
if (i < self.cloppers.len) return &self.cloppers.at(index).base;
1704+
if (i < self.cloppers.len) return self.cloppers.at(index);
17051705
i -= self.cloppers.len;
17061706

17071707
return null;

‎std/zig/parser.zig

Lines changed: 74 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ pub const Parser = struct {
171171
Semicolon: &const &const ast.Node,
172172
AsmOutputItems: &ArrayList(&ast.NodeAsmOutput),
173173
AsmInputItems: &ArrayList(&ast.NodeAsmInput),
174-
AsmClopperItems: &ArrayList(&ast.NodeStringLiteral),
174+
AsmClopperItems: &ArrayList(&ast.Node),
175175
ExprListItemOrEnd: ExprListCtx,
176176
ExprListCommaOrEnd: ExprListCtx,
177177
FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer),
@@ -301,7 +301,11 @@ pub const Parser = struct {
301301
Token.Id.Keyword_test => {
302302
stack.append(State.TopLevel) catch unreachable;
303303

304-
const name_token = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
304+
const name_token = self.getNextToken();
305+
const name = (try self.parseStringLiteral(arena, name_token)) ?? {
306+
try self.parseError(&stack, name_token, "expected string literal, found {}", @tagName(name_token.id));
307+
continue;
308+
};
305309
const lbrace = (try self.expectToken(&stack, Token.Id.LBrace)) ?? continue;
306310

307311
const block = try self.createNode(arena, ast.NodeBlock,
@@ -317,7 +321,7 @@ pub const Parser = struct {
317321
ast.NodeTestDecl {
318322
.base = undefined,
319323
.test_token = token,
320-
.name = &(try self.createLiteral(arena, ast.NodeStringLiteral, name_token)).base,
324+
.name = name,
321325
.body_node = &block.base,
322326
}
323327
);
@@ -389,15 +393,12 @@ pub const Parser = struct {
389393
continue;
390394
},
391395
Token.Id.Keyword_extern => {
392-
const lib_name_token = self.getNextToken();
393396
const lib_name = blk: {
394-
if (lib_name_token.id == Token.Id.StringLiteral) {
395-
const res = try self.createLiteral(arena, ast.NodeStringLiteral, lib_name_token);
396-
break :blk &res.base;
397-
} else {
397+
const lib_name_token = self.getNextToken();
398+
break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? {
398399
self.putBackToken(lib_name_token);
399400
break :blk null;
400-
}
401+
};
401402
};
402403

403404
stack.append(State {
@@ -1504,10 +1505,6 @@ pub const Parser = struct {
15041505
dest_ptr.store(&(try self.createLiteral(arena, ast.NodeFloatLiteral, token)).base);
15051506
continue;
15061507
},
1507-
Token.Id.StringLiteral => {
1508-
dest_ptr.store(&(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base);
1509-
continue;
1510-
},
15111508
Token.Id.CharLiteral => {
15121509
dest_ptr.store(&(try self.createLiteral(arena, ast.NodeCharLiteral, token)).base);
15131510
continue;
@@ -1536,24 +1533,8 @@ pub const Parser = struct {
15361533
dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUnreachable, token)).base);
15371534
continue;
15381535
},
1539-
Token.Id.MultilineStringLiteralLine => {
1540-
const node = try self.createToDestNode(arena, dest_ptr, ast.NodeMultilineStringLiteral,
1541-
ast.NodeMultilineStringLiteral {
1542-
.base = undefined,
1543-
.tokens = ArrayList(Token).init(arena),
1544-
}
1545-
);
1546-
try node.tokens.append(token);
1547-
while (true) {
1548-
const multiline_str = self.getNextToken();
1549-
if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
1550-
self.putBackToken(multiline_str);
1551-
break;
1552-
}
1553-
1554-
try node.tokens.append(multiline_str);
1555-
}
1556-
continue;
1536+
Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => {
1537+
dest_ptr.store((try self.parseStringLiteral(arena, token)) ?? unreachable);
15571538
},
15581539
Token.Id.LParen => {
15591540
const node = try self.createToDestNode(arena, dest_ptr, ast.NodeGroupedExpression,
@@ -1781,7 +1762,12 @@ pub const Parser = struct {
17811762
break :blk true;
17821763
};
17831764
_ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
1784-
const template = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
1765+
1766+
const template_token = self.getNextToken();
1767+
const template = (try self.parseStringLiteral(arena, template_token)) ?? {
1768+
try self.parseError(&stack, template_token, "expected string literal, found {}", @tagName(template_token.id));
1769+
continue;
1770+
};
17851771
// TODO parse template
17861772

17871773
const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm,
@@ -1793,7 +1779,7 @@ pub const Parser = struct {
17931779
//.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena),
17941780
.outputs = ArrayList(&ast.NodeAsmOutput).init(arena),
17951781
.inputs = ArrayList(&ast.NodeAsmInput).init(arena),
1796-
.cloppers = ArrayList(&ast.NodeStringLiteral).init(arena),
1782+
.cloppers = ArrayList(&ast.Node).init(arena),
17971783
.rparen = undefined,
17981784
}
17991785
);
@@ -1881,7 +1867,12 @@ pub const Parser = struct {
18811867

18821868
const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;
18831869
_ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;
1884-
const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
1870+
1871+
const constraint_token = self.getNextToken();
1872+
const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? {
1873+
try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id));
1874+
continue;
1875+
};
18851876

18861877
_ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
18871878
try stack.append(State { .ExpectToken = Token.Id.RParen });
@@ -1890,7 +1881,7 @@ pub const Parser = struct {
18901881
ast.NodeAsmOutput {
18911882
.base = undefined,
18921883
.symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),
1893-
.constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint),
1884+
.constraint = constraint,
18941885
.kind = undefined,
18951886
}
18961887
);
@@ -1926,7 +1917,12 @@ pub const Parser = struct {
19261917

19271918
const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;
19281919
_ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;
1929-
const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;
1920+
1921+
const constraint_token = self.getNextToken();
1922+
const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? {
1923+
try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id));
1924+
continue;
1925+
};
19301926

19311927
_ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
19321928
try stack.append(State { .ExpectToken = Token.Id.RParen });
@@ -1935,7 +1931,7 @@ pub const Parser = struct {
19351931
ast.NodeAsmInput {
19361932
.base = undefined,
19371933
.symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),
1938-
.constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint),
1934+
.constraint = constraint,
19391935
.expr = undefined,
19401936
}
19411937
);
@@ -1944,13 +1940,13 @@ pub const Parser = struct {
19441940
},
19451941

19461942
State.AsmClopperItems => |items| {
1947-
const string = self.getNextToken();
1948-
if (string.id != Token.Id.StringLiteral) {
1949-
self.putBackToken(string);
1943+
const string_token = self.getNextToken();
1944+
const string = (try self.parseStringLiteral(arena, string_token)) ?? {
1945+
self.putBackToken(string_token);
19501946
continue;
1951-
}
1947+
};
1948+
try items.append(string);
19521949

1953-
try items.append(try self.createLiteral(arena, ast.NodeStringLiteral, string));
19541950
stack.append(State { .AsmClopperItems = items }) catch unreachable;
19551951
try stack.append(State { .IfToken = Token.Id.Comma });
19561952
},
@@ -2742,6 +2738,37 @@ pub const Parser = struct {
27422738
}
27432739
}
27442740

2741+
fn parseStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !?&ast.Node {
2742+
switch (token.id) {
2743+
Token.Id.StringLiteral => {
2744+
return &(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base;
2745+
},
2746+
Token.Id.MultilineStringLiteralLine => {
2747+
const node = try self.createNode(arena, ast.NodeMultilineStringLiteral,
2748+
ast.NodeMultilineStringLiteral {
2749+
.base = undefined,
2750+
.tokens = ArrayList(Token).init(arena),
2751+
}
2752+
);
2753+
try node.tokens.append(token);
2754+
while (true) {
2755+
const multiline_str = self.getNextToken();
2756+
if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
2757+
self.putBackToken(multiline_str);
2758+
break;
2759+
}
2760+
2761+
try node.tokens.append(multiline_str);
2762+
}
2763+
2764+
return &node.base;
2765+
},
2766+
// TODO: We shouldn't need a cast, but:
2767+
// zig: /home/jc/Documents/zig/src/ir.cpp:7962: TypeTableEntry* ir_resolve_peer_types(IrAnalyze*, AstNode*, IrInstruction**, size_t): Assertion `err_set_type != nullptr' failed.
2768+
else => return (?&ast.Node)(null),
2769+
}
2770+
}
2771+
27452772
fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool {
27462773
switch (token.id) {
27472774
Token.Id.Keyword_suspend => {
@@ -4085,16 +4112,14 @@ pub const Parser = struct {
40854112
try stream.write("volatile ");
40864113
}
40874114

4088-
try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template));
4089-
40904115
try stack.append(RenderState { .Indent = indent });
40914116
try stack.append(RenderState { .Text = ")" });
40924117
{
40934118
const cloppers = asm_node.cloppers.toSliceConst();
40944119
var i = cloppers.len;
40954120
while (i != 0) {
40964121
i -= 1;
4097-
try stack.append(RenderState { .Expression = &cloppers[i].base });
4122+
try stack.append(RenderState { .Expression = cloppers[i] });
40984123

40994124
if (i != 0) {
41004125
try stack.append(RenderState { .Text = ", " });
@@ -4163,14 +4188,16 @@ pub const Parser = struct {
41634188
try stack.append(RenderState.PrintIndent);
41644189
try stack.append(RenderState { .Indent = indent + indent_delta});
41654190
try stack.append(RenderState { .Text = "\n" });
4191+
try stack.append(RenderState { .Expression = asm_node.template });
4192+
try stack.append(RenderState { .Text = "(" });
41664193
},
41674194
ast.Node.Id.AsmInput => {
41684195
const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base);
41694196

41704197
try stack.append(RenderState { .Text = ")"});
41714198
try stack.append(RenderState { .Expression = asm_input.expr});
41724199
try stack.append(RenderState { .Text = " ("});
4173-
try stack.append(RenderState { .Expression = &asm_input.constraint.base});
4200+
try stack.append(RenderState { .Expression = asm_input.constraint });
41744201
try stack.append(RenderState { .Text = "] "});
41754202
try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base});
41764203
try stack.append(RenderState { .Text = "["});
@@ -4189,7 +4216,7 @@ pub const Parser = struct {
41894216
},
41904217
}
41914218
try stack.append(RenderState { .Text = " ("});
4192-
try stack.append(RenderState { .Expression = &asm_output.constraint.base});
4219+
try stack.append(RenderState { .Expression = asm_output.constraint });
41934220
try stack.append(RenderState { .Text = "] "});
41944221
try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base});
41954222
try stack.append(RenderState { .Text = "["});

0 commit comments

Comments
 (0)