Skip to content

Commit c990e00

Browse files
c410-f3rc410-f3r
committed
Add support for literals
1 parent a0f01c3 commit c990e00

File tree

9 files changed

+272
-59
lines changed

9 files changed

+272
-59
lines changed

‎compiler/rustc_expand/src/mbe/metavar_expr.rs

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1-
use rustc_ast::token::{self, Delimiter, IdentIsRaw};
1+
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, Token, TokenKind};
22
use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
33
use rustc_ast::{LitIntType, LitKind};
44
use rustc_ast_pretty::pprust;
55
use rustc_errors::{Applicability, PResult};
66
use rustc_macros::{Decodable, Encodable};
77
use rustc_session::parse::ParseSess;
88
use rustc_span::symbol::Ident;
9-
use rustc_span::Span;
9+
use rustc_span::{Span, Symbol};
1010

1111
pub(crate) const RAW_IDENT_ERR: &str = "`${concat(..)}` currently does not support raw identifiers";
12+
pub(crate) const UNSUPPORTED_CONCAT_ELEM_ERR: &str = "expected identifier or string literal";
1213

1314
/// A meta-variable expression, for expansions based on properties of meta-variables.
1415
#[derive(Debug, PartialEq, Encodable, Decodable)]
@@ -51,11 +52,26 @@ impl MetaVarExpr {
5152
let mut result = Vec::new();
5253
loop {
5354
let is_var = try_eat_dollar(&mut iter);
54-
let element_ident = parse_ident(&mut iter, psess, outer_span)?;
55+
let token = parse_token(&mut iter, psess, outer_span)?;
5556
let element = if is_var {
56-
MetaVarExprConcatElem::Var(element_ident)
57+
MetaVarExprConcatElem::Var(parse_ident_from_token(psess, token)?)
58+
} else if let TokenKind::Literal(Lit {
59+
kind: token::LitKind::Str,
60+
symbol,
61+
suffix: None,
62+
}) = token.kind
63+
{
64+
MetaVarExprConcatElem::Literal(symbol)
5765
} else {
58-
MetaVarExprConcatElem::Ident(element_ident)
66+
match parse_ident_from_token(psess, token) {
67+
Err(err) => {
68+
err.cancel();
69+
return Err(psess
70+
.dcx()
71+
.struct_span_err(token.span, UNSUPPORTED_CONCAT_ELEM_ERR));
72+
}
73+
Ok(elem) => MetaVarExprConcatElem::Ident(elem),
74+
}
5975
};
6076
result.push(element);
6177
if iter.look_ahead(0).is_none() {
@@ -105,11 +121,13 @@ impl MetaVarExpr {
105121

106122
#[derive(Debug, Decodable, Encodable, PartialEq)]
107123
pub(crate) enum MetaVarExprConcatElem {
108-
/// There is NO preceding dollar sign, which means that this identifier should be interpreted
109-
/// as a literal.
124+
/// Identifier WITHOUT a preceding dollar sign, which means that this identifier should be
125+
/// interpreted as a literal.
110126
Ident(Ident),
111-
/// There is a preceding dollar sign, which means that this identifier should be expanded
112-
/// and interpreted as a variable.
127+
/// For example, a number or a string.
128+
Literal(Symbol),
129+
/// Identifier WITH a preceding dollar sign, which means that this identifier should be
130+
/// expanded and interpreted as a variable.
113131
Var(Ident),
114132
}
115133

@@ -158,7 +176,7 @@ fn parse_depth<'psess>(
158176
span: Span,
159177
) -> PResult<'psess, usize> {
160178
let Some(tt) = iter.next() else { return Ok(0) };
161-
let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else {
179+
let TokenTree::Token(Token { kind: TokenKind::Literal(lit), .. }, _) = tt else {
162180
return Err(psess
163181
.dcx()
164182
.struct_span_err(span, "meta-variable expression depth must be a literal"));
@@ -180,12 +198,14 @@ fn parse_ident<'psess>(
180198
psess: &'psess ParseSess,
181199
fallback_span: Span,
182200
) -> PResult<'psess, Ident> {
183-
let Some(tt) = iter.next() else {
184-
return Err(psess.dcx().struct_span_err(fallback_span, "expected identifier"));
185-
};
186-
let TokenTree::Token(token, _) = tt else {
187-
return Err(psess.dcx().struct_span_err(tt.span(), "expected identifier"));
188-
};
201+
let token = parse_token(iter, psess, fallback_span)?;
202+
parse_ident_from_token(psess, token)
203+
}
204+
205+
fn parse_ident_from_token<'psess>(
206+
psess: &'psess ParseSess,
207+
token: &Token,
208+
) -> PResult<'psess, Ident> {
189209
if let Some((elem, is_raw)) = token.ident() {
190210
if let IdentIsRaw::Yes = is_raw {
191211
return Err(psess.dcx().struct_span_err(elem.span, RAW_IDENT_ERR));
@@ -205,10 +225,24 @@ fn parse_ident<'psess>(
205225
Err(err)
206226
}
207227

228+
fn parse_token<'psess, 't>(
229+
iter: &mut RefTokenTreeCursor<'t>,
230+
psess: &'psess ParseSess,
231+
fallback_span: Span,
232+
) -> PResult<'psess, &'t Token> {
233+
let Some(tt) = iter.next() else {
234+
return Err(psess.dcx().struct_span_err(fallback_span, UNSUPPORTED_CONCAT_ELEM_ERR));
235+
};
236+
let TokenTree::Token(token, _) = tt else {
237+
return Err(psess.dcx().struct_span_err(tt.span(), UNSUPPORTED_CONCAT_ELEM_ERR));
238+
};
239+
Ok(token)
240+
}
241+
208242
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
209243
/// iterator is not modified and the result is `false`.
210244
fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
211-
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
245+
if let Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
212246
let _ = iter.next();
213247
return true;
214248
}
@@ -218,8 +252,7 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
218252
/// Tries to move the iterator forward returning `true` if there is a dollar sign. If not, then the
219253
/// iterator is not modified and the result is `false`.
220254
fn try_eat_dollar(iter: &mut RefTokenTreeCursor<'_>) -> bool {
221-
if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
222-
{
255+
if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
223256
let _ = iter.next();
224257
return true;
225258
}
@@ -232,8 +265,7 @@ fn eat_dollar<'psess>(
232265
psess: &'psess ParseSess,
233266
span: Span,
234267
) -> PResult<'psess, ()> {
235-
if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
236-
{
268+
if let Some(TokenTree::Token(Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0) {
237269
let _ = iter.next();
238270
return Ok(());
239271
}

‎compiler/rustc_expand/src/mbe/transcribe.rs

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,13 @@ use rustc_ast::token::{self, Delimiter, Token, TokenKind};
1111
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
1212
use rustc_data_structures::fx::FxHashMap;
1313
use rustc_errors::{pluralize, Diag, DiagCtxtHandle, PResult};
14+
use rustc_parse::lexer::nfc_normalize;
1415
use rustc_parse::parser::ParseNtResult;
1516
use rustc_session::parse::ParseSess;
17+
use rustc_session::parse::SymbolGallery;
1618
use rustc_span::hygiene::{LocalExpnId, Transparency};
1719
use rustc_span::symbol::{sym, Ident, MacroRulesNormalizedIdent};
18-
use rustc_span::{with_metavar_spans, Span, Symbol, SyntaxContext};
20+
use rustc_span::{with_metavar_spans, Span, SyntaxContext};
1921
use smallvec::{smallvec, SmallVec};
2022
use std::mem;
2123

@@ -312,7 +314,16 @@ pub(super) fn transcribe<'a>(
312314

313315
// Replace meta-variable expressions with the result of their expansion.
314316
mbe::TokenTree::MetaVarExpr(sp, expr) => {
315-
transcribe_metavar_expr(dcx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
317+
transcribe_metavar_expr(
318+
dcx,
319+
expr,
320+
interp,
321+
&mut marker,
322+
&repeats,
323+
&mut result,
324+
sp,
325+
&psess.symbol_gallery,
326+
)?;
316327
}
317328

318329
// If we are entering a new delimiter, we push its contents to the `stack` to be
@@ -669,6 +680,7 @@ fn transcribe_metavar_expr<'a>(
669680
repeats: &[(usize, usize)],
670681
result: &mut Vec<TokenTree>,
671682
sp: &DelimSpan,
683+
symbol_gallery: &SymbolGallery,
672684
) -> PResult<'a, ()> {
673685
let mut visited_span = || {
674686
let mut span = sp.entire();
@@ -680,16 +692,26 @@ fn transcribe_metavar_expr<'a>(
680692
let mut concatenated = String::new();
681693
for element in elements.into_iter() {
682694
let string = match element {
683-
MetaVarExprConcatElem::Ident(ident) => ident.to_string(),
684-
MetaVarExprConcatElem::Var(ident) => extract_ident(dcx, *ident, interp)?,
695+
MetaVarExprConcatElem::Ident(elem) => elem.to_string(),
696+
MetaVarExprConcatElem::Literal(elem) => elem.as_str().into(),
697+
MetaVarExprConcatElem::Var(elem) => extract_ident(dcx, *elem, interp)?,
685698
};
686699
concatenated.push_str(&string);
687700
}
701+
let symbol = nfc_normalize(&concatenated);
702+
let concatenated_span = visited_span();
703+
if !rustc_lexer::is_ident(symbol.as_str()) {
704+
return Err(dcx.struct_span_err(
705+
concatenated_span,
706+
"`${concat(..)}` is not generating a valid identifier",
707+
));
708+
}
709+
symbol_gallery.insert(symbol, concatenated_span);
688710
// The current implementation marks the span as coming from the macro regardless of
689711
// contexts of the concatenated identifiers but this behavior may change in the
690712
// future.
691713
result.push(TokenTree::Token(
692-
Token::from_ast_ident(Ident::new(Symbol::intern(&concatenated), visited_span())),
714+
Token::from_ast_ident(Ident::new(symbol, concatenated_span)),
693715
Spacing::Alone,
694716
));
695717
}

‎tests/ui/macros/macro-metavar-expr-concat/allowed-operations.rs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,16 @@ macro_rules! without_dollar_sign_is_an_ident {
3737
};
3838
}
3939

40+
macro_rules! literals {
41+
($ident:ident) => {{
42+
let ${concat(_a, "_b")}: () = ();
43+
let ${concat("_b", _a)}: () = ();
44+
45+
let ${concat($ident, "_b")}: () = ();
46+
let ${concat("_b", $ident)}: () = ();
47+
}};
48+
}
49+
4050
fn main() {
4151
create_things!(behold);
4252
behold_separated_idents_in_a_fn();
@@ -55,4 +65,6 @@ fn main() {
5565
without_dollar_sign_is_an_ident!(_123);
5666
assert_eq!(VARident, 1);
5767
assert_eq!(VAR_123, 2);
68+
69+
literals!(_hello);
5870
}

‎tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@ macro_rules! idents_11 {
2626
macro_rules! no_params {
2727
() => {
2828
let ${concat(r#abc, abc)}: () = ();
29-
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
29+
//~^ ERROR expected identifier or string literal
3030
//~| ERROR expected pattern, found `$`
3131

3232
let ${concat(abc, r#abc)}: () = ();
33-
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
33+
//~^ ERROR expected identifier or string literal
3434

3535
let ${concat(r#abc, r#abc)}: () = ();
36-
//~^ ERROR `${concat(..)}` currently does not support raw identifiers
36+
//~^ ERROR expected identifier or string literal
3737
};
3838
}
3939

‎tests/ui/macros/macro-metavar-expr-concat/raw-identifiers.stderr

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
error: `${concat(..)}` currently does not support raw identifiers
1+
error: expected identifier or string literal
22
--> $DIR/raw-identifiers.rs:28:22
33
|
44
LL | let ${concat(r#abc, abc)}: () = ();
55
| ^^^^^
66

7-
error: `${concat(..)}` currently does not support raw identifiers
7+
error: expected identifier or string literal
88
--> $DIR/raw-identifiers.rs:32:27
99
|
1010
LL | let ${concat(abc, r#abc)}: () = ();
1111
| ^^^^^
1212

13-
error: `${concat(..)}` currently does not support raw identifiers
13+
error: expected identifier or string literal
1414
--> $DIR/raw-identifiers.rs:35:22
1515
|
1616
LL | let ${concat(r#abc, r#abc)}: () = ();

‎tests/ui/macros/macro-metavar-expr-concat/syntax-errors.rs

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,6 @@ macro_rules! wrong_concat_declarations {
1111
${concat(aaaa,)}
1212
//~^ ERROR expected identifier
1313

14-
${concat(aaaa, 1)}
15-
//~^ ERROR expected identifier
16-
1714
${concat(_, aaaa)}
1815

1916
${concat(aaaa aaaa)}
@@ -30,9 +27,6 @@ macro_rules! wrong_concat_declarations {
3027

3128
${concat($ex, aaaa,)}
3229
//~^ ERROR expected identifier
33-
34-
${concat($ex, aaaa, 123)}
35-
//~^ ERROR expected identifier
3630
};
3731
}
3832

@@ -43,8 +37,80 @@ macro_rules! dollar_sign_without_referenced_ident {
4337
};
4438
}
4539

40+
macro_rules! starting_number {
41+
($ident:ident) => {{
42+
let ${concat("1", $ident)}: () = ();
43+
//~^ ERROR `${concat(..)}` is not generating a valid identifier
44+
}};
45+
}
46+
47+
macro_rules! starting_valid_unicode {
48+
($ident:ident) => {{
49+
let ${concat("Ý", $ident)}: () = ();
50+
}};
51+
}
52+
53+
macro_rules! starting_invalid_unicode {
54+
($ident:ident) => {{
55+
let ${concat("\u{00BD}", $ident)}: () = ();
56+
//~^ ERROR `${concat(..)}` is not generating a valid identifier
57+
}};
58+
}
59+
60+
macro_rules! ending_number {
61+
($ident:ident) => {{
62+
let ${concat($ident, "1")}: () = ();
63+
}};
64+
}
65+
66+
macro_rules! ending_valid_unicode {
67+
($ident:ident) => {{
68+
let ${concat($ident, "Ý")}: () = ();
69+
}};
70+
}
71+
72+
macro_rules! ending_invalid_unicode {
73+
($ident:ident) => {{
74+
let ${concat($ident, "\u{00BD}")}: () = ();
75+
//~^ ERROR `${concat(..)}` is not generating a valid identifier
76+
}};
77+
}
78+
79+
macro_rules! empty {
80+
() => {{
81+
let ${concat("", "")}: () = ();
82+
//~^ ERROR `${concat(..)}` is not generating a valid identifier
83+
}};
84+
}
85+
86+
macro_rules! unsupported_literals {
87+
($ident:ident) => {{
88+
let ${concat(_a, 'b')}: () = ();
89+
//~^ ERROR expected identifier or string literal
90+
//~| ERROR expected pattern
91+
let ${concat(_a, 1)}: () = ();
92+
//~^ ERROR expected identifier or string literal
93+
94+
let ${concat($ident, 'b')}: () = ();
95+
//~^ ERROR expected identifier or string literal
96+
let ${concat($ident, 1)}: () = ();
97+
//~^ ERROR expected identifier or string literal
98+
}};
99+
}
100+
46101
fn main() {
47102
wrong_concat_declarations!(1);
48103

49104
dollar_sign_without_referenced_ident!(VAR);
105+
106+
starting_number!(_abc);
107+
starting_valid_unicode!(_abc);
108+
starting_invalid_unicode!(_abc);
109+
110+
ending_number!(_abc);
111+
ending_valid_unicode!(_abc);
112+
ending_invalid_unicode!(_abc);
113+
unsupported_literals!(_abc);
114+
115+
empty!();
50116
}

0 commit comments

Comments
 (0)