Skip to content

Commit f5bf1a9

Browse files
edwin0chengedwin0cheng
committed
Fix builtin macros split exprs on comma
1 parent f682627 commit f5bf1a9

File tree

5 files changed

+144
-89
lines changed

5 files changed

+144
-89
lines changed

‎crates/hir_expand/src/builtin_macro.rs

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use crate::{
66

77
use base_db::{AnchoredPath, FileId};
88
use either::Either;
9-
use mbe::{parse_to_token_tree, ExpandResult};
9+
use mbe::{parse_exprs_with_sep, parse_to_token_tree, ExpandResult};
1010
use parser::FragmentKind;
1111
use syntax::ast::{self, AstToken};
1212

@@ -238,35 +238,21 @@ fn format_args_expand(
238238
// ])
239239
// ```,
240240
// which is still not really correct, but close enough for now
241-
let mut args = Vec::new();
242-
let mut current = Vec::new();
243-
for tt in tt.token_trees.iter().cloned() {
244-
match tt {
245-
tt::TokenTree::Leaf(tt::Leaf::Punct(p)) if p.char == ',' => {
246-
args.push(current);
247-
current = Vec::new();
248-
}
249-
_ => {
250-
current.push(tt);
251-
}
252-
}
253-
}
254-
if !current.is_empty() {
255-
args.push(current);
256-
}
241+
let mut args = parse_exprs_with_sep(tt, ',');
242+
257243
if args.is_empty() {
258244
return ExpandResult::only_err(mbe::ExpandError::NoMatchingRule);
259245
}
260246
for arg in &mut args {
261247
// Remove `key =`.
262-
if matches!(arg.get(1), Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p))) if p.char == '=' && p.spacing != tt::Spacing::Joint)
248+
if matches!(arg.token_trees.get(1), Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p))) if p.char == '=' && p.spacing != tt::Spacing::Joint)
263249
{
264-
arg.drain(..2);
250+
arg.token_trees.drain(..2);
265251
}
266252
}
267253
let _format_string = args.remove(0);
268254
let arg_tts = args.into_iter().flat_map(|arg| {
269-
quote! { std::fmt::ArgumentV1::new(&(##arg), std::fmt::Display::fmt), }
255+
quote! { std::fmt::ArgumentV1::new(&(#arg), std::fmt::Display::fmt), }
270256
}.token_trees).collect::<Vec<_>>();
271257
let expanded = quote! {
272258
std::fmt::Arguments::new_v1(&[], &[##arg_tts])
@@ -719,6 +705,25 @@ mod tests {
719705
);
720706
}
721707

708+
#[test]
709+
fn test_format_args_expand_with_comma_exprs() {
710+
let expanded = expand_builtin_macro(
711+
r#"
712+
#[rustc_builtin_macro]
713+
macro_rules! format_args {
714+
($fmt:expr) => ({ /* compiler built-in */ });
715+
($fmt:expr, $($args:tt)*) => ({ /* compiler built-in */ })
716+
}
717+
format_args!("{} {:?}", a::<A,B>(), b);
718+
"#,
719+
);
720+
721+
assert_eq!(
722+
expanded,
723+
r#"std::fmt::Arguments::new_v1(&[], &[std::fmt::ArgumentV1::new(&(a::<A,B>()),std::fmt::Display::fmt),std::fmt::ArgumentV1::new(&(b),std::fmt::Display::fmt),])"#
724+
);
725+
}
726+
722727
#[test]
723728
fn test_include_bytes_expand() {
724729
let expanded = expand_builtin_macro(

‎crates/mbe/src/expander/matcher.rs

Lines changed: 2 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,13 @@
33
use crate::{
44
expander::{Binding, Bindings, Fragment},
55
parser::{Op, RepeatKind, Separator},
6-
subtree_source::SubtreeTokenSource,
76
tt_iter::TtIter,
87
ExpandError, MetaTemplate,
98
};
109

1110
use super::ExpandResult;
12-
use parser::{FragmentKind::*, TreeSink};
13-
use syntax::{SmolStr, SyntaxKind};
14-
use tt::buffer::{Cursor, TokenBuffer};
11+
use parser::FragmentKind::*;
12+
use syntax::SmolStr;
1513

1614
impl Bindings {
1715
fn push_optional(&mut self, name: &SmolStr) {
@@ -409,68 +407,6 @@ impl<'a> TtIter<'a> {
409407
.into())
410408
}
411409

412-
fn expect_fragment(
413-
&mut self,
414-
fragment_kind: parser::FragmentKind,
415-
) -> ExpandResult<Option<tt::TokenTree>> {
416-
struct OffsetTokenSink<'a> {
417-
cursor: Cursor<'a>,
418-
error: bool,
419-
}
420-
421-
impl<'a> TreeSink for OffsetTokenSink<'a> {
422-
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
423-
if kind == SyntaxKind::LIFETIME_IDENT {
424-
n_tokens = 2;
425-
}
426-
for _ in 0..n_tokens {
427-
self.cursor = self.cursor.bump_subtree();
428-
}
429-
}
430-
fn start_node(&mut self, _kind: SyntaxKind) {}
431-
fn finish_node(&mut self) {}
432-
fn error(&mut self, _error: parser::ParseError) {
433-
self.error = true;
434-
}
435-
}
436-
437-
let buffer = TokenBuffer::from_tokens(&self.inner.as_slice());
438-
let mut src = SubtreeTokenSource::new(&buffer);
439-
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
440-
441-
parser::parse_fragment(&mut src, &mut sink, fragment_kind);
442-
443-
let mut err = None;
444-
if !sink.cursor.is_root() || sink.error {
445-
err = Some(err!("expected {:?}", fragment_kind));
446-
}
447-
448-
let mut curr = buffer.begin();
449-
let mut res = vec![];
450-
451-
if sink.cursor.is_root() {
452-
while curr != sink.cursor {
453-
if let Some(token) = curr.token_tree() {
454-
res.push(token);
455-
}
456-
curr = curr.bump();
457-
}
458-
}
459-
self.inner = self.inner.as_slice()[res.len()..].iter();
460-
if res.len() == 0 && err.is_none() {
461-
err = Some(err!("no tokens consumed"));
462-
}
463-
let res = match res.len() {
464-
1 => Some(res[0].cloned()),
465-
0 => None,
466-
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
467-
delimiter: None,
468-
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
469-
})),
470-
};
471-
ExpandResult { value: res, err }
472-
}
473-
474410
fn eat_vis(&mut self) -> Option<tt::TokenTree> {
475411
let mut fork = self.clone();
476412
match fork.expect_fragment(Visibility) {

‎crates/mbe/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ impl fmt::Display for ExpandError {
6565
}
6666

6767
pub use crate::syntax_bridge::{
68-
ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node,
69-
TokenMap,
68+
ast_to_token_tree, parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree,
69+
token_tree_to_syntax_node, TokenMap,
7070
};
7171

7272
/// This struct contains AST for a single `macro_rules` definition. What might

‎crates/mbe/src/syntax_bridge.rs

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ use syntax::{
1010
};
1111
use tt::buffer::{Cursor, TokenBuffer};
1212

13-
use crate::subtree_source::SubtreeTokenSource;
1413
use crate::ExpandError;
14+
use crate::{subtree_source::SubtreeTokenSource, tt_iter::TtIter};
1515

1616
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
1717
pub enum TokenTextRange {
@@ -112,6 +112,43 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
112112
Some((subtree, conv.id_alloc.map))
113113
}
114114

115+
/// Split token tree with seperate expr: $($e:expr)SEP*
116+
pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
117+
if tt.token_trees.is_empty() {
118+
return Vec::new();
119+
}
120+
121+
let mut iter = TtIter::new(tt);
122+
let mut res = Vec::new();
123+
124+
while iter.peek_n(0).is_some() {
125+
let expanded = iter.expect_fragment(FragmentKind::Expr);
126+
if expanded.err.is_some() {
127+
break;
128+
}
129+
130+
res.push(match expanded.value {
131+
None => break,
132+
Some(tt @ tt::TokenTree::Leaf(_)) => {
133+
tt::Subtree { delimiter: None, token_trees: vec![tt.into()] }
134+
}
135+
Some(tt::TokenTree::Subtree(tt)) => tt,
136+
});
137+
138+
let mut fork = iter.clone();
139+
if fork.expect_char(sep).is_err() {
140+
break;
141+
}
142+
iter = fork;
143+
}
144+
145+
if iter.peek_n(0).is_some() {
146+
res.push(tt::Subtree { delimiter: None, token_trees: iter.into_iter().cloned().collect() });
147+
}
148+
149+
res
150+
}
151+
115152
impl TokenMap {
116153
pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
117154
let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {

‎crates/mbe/src/tt_iter.rs

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
11
//! FIXME: write short doc here
22
3+
use crate::{subtree_source::SubtreeTokenSource, ExpandError, ExpandResult};
4+
5+
use parser::TreeSink;
6+
use syntax::SyntaxKind;
7+
use tt::buffer::{Cursor, TokenBuffer};
8+
9+
macro_rules! err {
10+
() => {
11+
ExpandError::BindingError(format!(""))
12+
};
13+
($($tt:tt)*) => {
14+
ExpandError::BindingError(format!($($tt)*))
15+
};
16+
}
17+
318
#[derive(Debug, Clone)]
419
pub(crate) struct TtIter<'a> {
520
pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
@@ -56,6 +71,68 @@ impl<'a> TtIter<'a> {
5671
}
5772
}
5873

74+
pub(crate) fn expect_fragment(
75+
&mut self,
76+
fragment_kind: parser::FragmentKind,
77+
) -> ExpandResult<Option<tt::TokenTree>> {
78+
struct OffsetTokenSink<'a> {
79+
cursor: Cursor<'a>,
80+
error: bool,
81+
}
82+
83+
impl<'a> TreeSink for OffsetTokenSink<'a> {
84+
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
85+
if kind == SyntaxKind::LIFETIME_IDENT {
86+
n_tokens = 2;
87+
}
88+
for _ in 0..n_tokens {
89+
self.cursor = self.cursor.bump_subtree();
90+
}
91+
}
92+
fn start_node(&mut self, _kind: SyntaxKind) {}
93+
fn finish_node(&mut self) {}
94+
fn error(&mut self, _error: parser::ParseError) {
95+
self.error = true;
96+
}
97+
}
98+
99+
let buffer = TokenBuffer::from_tokens(&self.inner.as_slice());
100+
let mut src = SubtreeTokenSource::new(&buffer);
101+
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
102+
103+
parser::parse_fragment(&mut src, &mut sink, fragment_kind);
104+
105+
let mut err = None;
106+
if !sink.cursor.is_root() || sink.error {
107+
err = Some(err!("expected {:?}", fragment_kind));
108+
}
109+
110+
let mut curr = buffer.begin();
111+
let mut res = vec![];
112+
113+
if sink.cursor.is_root() {
114+
while curr != sink.cursor {
115+
if let Some(token) = curr.token_tree() {
116+
res.push(token);
117+
}
118+
curr = curr.bump();
119+
}
120+
}
121+
self.inner = self.inner.as_slice()[res.len()..].iter();
122+
if res.len() == 0 && err.is_none() {
123+
err = Some(err!("no tokens consumed"));
124+
}
125+
let res = match res.len() {
126+
1 => Some(res[0].cloned()),
127+
0 => None,
128+
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
129+
delimiter: None,
130+
token_trees: res.into_iter().map(|it| it.cloned()).collect(),
131+
})),
132+
};
133+
ExpandResult { value: res, err }
134+
}
135+
59136
pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> {
60137
self.inner.as_slice().get(n)
61138
}

0 commit comments

Comments
 (0)