Remove NtPat.

The one notable test change is `tests/ui/macros/trace_faulty_macros.rs`.
This commit removes the complicated `Interpolated` handling in
`expected_expression_found` that results in a longer error message. But
I think the new, shorter message is actually an improvement.

The original complaint was in #71039, when the error message started
with "error: expected expression, found `1 + 1`". That was confusing
because `1 + 1` is an expression. Other than that, the reporter said
"the whole error message is not too bad if you ignore the first line".

Subsequently, extra complexity and wording was added to the error
message. But I don't think the extra wording actually helps all that
much. In particular, it still says of the `1+1` that "this is expected
to be expression". This repeats the problem from the original complaint!

This commit removes the extra complexity, reverting to a simpler error
message. This is primarily because the traversal is a pain without
`Interpolated` tokens. Nonetheless, I think the error message is
*improved*. It now starts with "expected expression, found `pat`
metavariable", which is much clearer and the real problem. It also
doesn't say anything specific about `1+1`, which is good, because the
`1+1` isn't really relevant to the error -- it's the `$e:pat` that's
important.

Author: nnethercote

compiler/rustc_ast/src/ast_traits.rs

@@ -202,7 +202,6 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtItem(item) => item.tokens(),
             Nonterminal::NtStmt(stmt) => stmt.tokens(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
-            Nonterminal::NtPat(pat) => pat.tokens(),
             Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
             Nonterminal::NtPath(path) => path.tokens(),
             Nonterminal::NtBlock(block) => block.tokens(),
@@ -213,7 +212,6 @@ impl HasTokens for Nonterminal {
             Nonterminal::NtItem(item) => item.tokens_mut(),
             Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
-            Nonterminal::NtPat(pat) => pat.tokens_mut(),
             Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
             Nonterminal::NtPath(path) => path.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),

compiler/rustc_ast/src/mut_visit.rs

@@ -905,7 +905,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
                 vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item")
             })
         }),
-        token::NtPat(pat) => vis.visit_pat(pat),
         token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {

compiler/rustc_ast/src/token.rs

@@ -659,7 +659,6 @@ impl Token {
                     | NtExpr(..)
                     | NtLiteral(..)
                     | NtMeta(..)
-                    | NtPat(..)
                     | NtPath(..)
                 ),
             OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
@@ -1075,7 +1074,6 @@ pub enum Nonterminal {
     NtItem(P<ast::Item>),
     NtBlock(P<ast::Block>),
     NtStmt(P<ast::Stmt>),
-    NtPat(P<ast::Pat>),
     NtExpr(P<ast::Expr>),
     NtLiteral(P<ast::Expr>),
     /// Stuff inside brackets for attributes
@@ -1172,7 +1170,6 @@ impl Nonterminal {
             NtItem(item) => item.span,
             NtBlock(block) => block.span,
             NtStmt(stmt) => stmt.span,
-            NtPat(pat) => pat.span,
             NtExpr(expr) | NtLiteral(expr) => expr.span,
             NtMeta(attr_item) => attr_item.span(),
             NtPath(path) => path.span,
@@ -1184,7 +1181,6 @@ impl Nonterminal {
             NtItem(..) => "item",
             NtBlock(..) => "block",
             NtStmt(..) => "statement",
-            NtPat(..) => "pattern",
             NtExpr(..) => "expression",
             NtLiteral(..) => "literal",
             NtMeta(..) => "attribute",
@@ -1209,7 +1205,6 @@ impl fmt::Debug for Nonterminal {
             NtItem(..) => f.pad("NtItem(..)"),
             NtBlock(..) => f.pad("NtBlock(..)"),
             NtStmt(..) => f.pad("NtStmt(..)"),
-            NtPat(..) => f.pad("NtPat(..)"),
             NtExpr(..) => f.pad("NtExpr(..)"),
             NtLiteral(..) => f.pad("NtLiteral(..)"),
             NtMeta(..) => f.pad("NtMeta(..)"),

compiler/rustc_ast/src/tokenstream.rs

@@ -468,7 +468,6 @@ impl TokenStream {
                 TokenStream::token_alone(token::Semi, stmt.span)
             }
             Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
-            Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
             Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
             Nonterminal::NtPath(path) => TokenStream::from_ast(path),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -279,9 +279,9 @@ pub(super) fn transcribe<'a>(
                 if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
                     // We wrap the tokens in invisible delimiters, unless they are already wrapped
                     // in invisible delimiters with the same `MetaVarKind`. Because some proc
-                    // macros can't multiple layers of invisible delimiters of the same
+                    // macros can't handle multiple layers of invisible delimiters of the same
                     // `MetaVarKind`. This loses some span info, though it hopefully won't matter.
-                    let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
+                    let mut mk_delimited = |mk_span, mv_kind, mut stream: TokenStream| {
                         if stream.len() == 1 {
                             let tree = stream.iter().next().unwrap();
                             if let TokenTree::Delimited(_, _, delim, inner) = tree
@@ -295,6 +295,7 @@ pub(super) fn transcribe<'a>(
                         // Emit as a token stream within `Delimiter::Invisible` to maintain
                         // parsing priorities.
                         marker.visit_span(&mut sp);
+                        with_metavar_spans(|mspans| mspans.insert(mk_span, sp));
                         // Both the open delim and close delim get the same span, which covers the
                         // `$foo` in the decl macro RHS.
                         TokenTree::Delimited(
@@ -322,12 +323,21 @@ pub(super) fn transcribe<'a>(
                             let kind = token::NtLifetime(*ident, *is_raw);
                             TokenTree::token_alone(kind, sp)
                         }
+                        MatchedSingle(ParseNtResult::Pat(pat, pat_kind)) => mk_delimited(
+                            pat.span,
+                            MetaVarKind::Pat(*pat_kind),
+                            TokenStream::from_ast(pat),
+                        ),
                         MatchedSingle(ParseNtResult::Ty(ty)) => {
                             let is_path = matches!(&ty.kind, TyKind::Path(None, _path));
-                            mk_delimited(MetaVarKind::Ty { is_path }, TokenStream::from_ast(ty))
+                            mk_delimited(
+                                ty.span,
+                                MetaVarKind::Ty { is_path },
+                                TokenStream::from_ast(ty),
+                            )
                         }
                         MatchedSingle(ParseNtResult::Vis(vis)) => {
-                            mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
+                            mk_delimited(vis.span, MetaVarKind::Vis, TokenStream::from_ast(vis))
                         }
                         MatchedSingle(ParseNtResult::Nt(nt)) => {
                             // Other variables are emitted into the output stream as groups with

compiler/rustc_parse/src/parser/diagnostics.rs

@@ -1,17 +1,15 @@
 use std::mem::take;
 use std::ops::{Deref, DerefMut};
-use std::sync::Arc;
 
 use ast::token::IdentIsRaw;
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Lit, LitKind, Token, TokenKind};
-use rustc_ast::tokenstream::AttrTokenTree;
 use rustc_ast::util::parser::AssocOp;
 use rustc_ast::{
     AngleBracketedArg, AngleBracketedArgs, AnonConst, AttrVec, BinOpKind, BindingMode, Block,
-    BlockCheckMode, Expr, ExprKind, GenericArg, Generics, HasTokens, Item, ItemKind, Param, Pat,
-    PatKind, Path, PathSegment, QSelf, Recovered, Ty, TyKind,
+    BlockCheckMode, Expr, ExprKind, GenericArg, Generics, Item, ItemKind, Param, Pat, PatKind,
+    Path, PathSegment, QSelf, Recovered, Ty, TyKind,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashSet;
@@ -2400,52 +2398,6 @@ impl<'a> Parser<'a> {
             err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
         }
         err.span_label(span, "expected expression");
-
-        // Walk the chain of macro expansions for the current token to point at how the original
-        // code was interpreted. This helps the user realize when a macro argument of one type is
-        // later reinterpreted as a different type, like `$x:expr` being reinterpreted as `$x:pat`
-        // in a subsequent macro invocation (#71039).
-        let mut tok = self.token.clone();
-        let mut labels = vec![];
-        while let TokenKind::Interpolated(nt) = &tok.kind {
-            let tokens = nt.tokens();
-            labels.push(Arc::clone(nt));
-            if let Some(tokens) = tokens
-                && let tokens = tokens.to_attr_token_stream()
-                && let tokens = tokens.0.deref()
-                && let [AttrTokenTree::Token(token, _)] = &tokens[..]
-            {
-                tok = token.clone();
-            } else {
-                break;
-            }
-        }
-        let mut iter = labels.into_iter().peekable();
-        let mut show_link = false;
-        while let Some(nt) = iter.next() {
-            let descr = nt.descr();
-            if let Some(next) = iter.peek() {
-                let next_descr = next.descr();
-                if next_descr != descr {
-                    err.span_label(next.use_span(), format!("this is expected to be {next_descr}"));
-                    err.span_label(
-                        nt.use_span(),
-                        format!(
-                            "this is interpreted as {}, but it is expected to be {}",
-                            next_descr, descr,
-                        ),
-                    );
-                    show_link = true;
-                }
-            }
-        }
-        if show_link {
-            err.note(
-                "when forwarding a matched fragment to another macro-by-example, matchers in the \
-                 second macro will see an opaque AST of the fragment type, not the underlying \
-                 tokens",
-            );
-        }
         err
     }
 

compiler/rustc_parse/src/parser/item.rs

@@ -4,7 +4,7 @@ use std::mem;
 use ast::token::IdentIsRaw;
 use rustc_ast::ast::*;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter, TokenKind};
+use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_ast::util::case::Case;
 use rustc_ast::{self as ast};
@@ -3071,8 +3071,10 @@ impl<'a> Parser<'a> {
 
     fn is_named_param(&self) -> bool {
         let offset = match &self.token.kind {
-            token::Interpolated(nt) => match &**nt {
-                token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
+            token::OpenDelim(Delimiter::Invisible(origin)) => match origin {
+                InvisibleOrigin::MetaVar(MetaVarKind::Pat(_)) => {
+                    return self.check_noexpect_past_close_delim(&token::Colon);
+                }
                 _ => 0,
             },
             token::BinOp(token::And) | token::AndAnd => 1,

compiler/rustc_parse/src/parser/mod.rs

@@ -24,7 +24,8 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
 use path::PathStyle;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{
-    self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, Token, TokenKind,
+    self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, NtPatKind, Token,
+    TokenKind,
 };
 use rustc_ast::tokenstream::{AttrsTarget, Spacing, TokenStream, TokenTree};
 use rustc_ast::util::case::Case;
@@ -1745,6 +1746,7 @@ pub enum ParseNtResult {
     Tt(TokenTree),
     Ident(Ident, IdentIsRaw),
     Lifetime(Ident, IdentIsRaw),
+    Pat(P<ast::Pat>, NtPatKind),
     Ty(P<ast::Ty>),
     Vis(P<ast::Visibility>),
 

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -49,7 +49,6 @@ impl<'a> Parser<'a> {
         fn nt_may_be_ident(nt: &Nonterminal) -> bool {
             match nt {
                 NtStmt(_)
-                | NtPat(_)
                 | NtExpr(_)
                 | NtLiteral(_) // `true`, `false`
                 | NtMeta(_)
@@ -99,7 +98,7 @@ impl<'a> Parser<'a> {
                 token::NtLifetime(..) => true,
                 token::Interpolated(nt) => match &**nt {
                     NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
-                    NtItem(_) | NtPat(_) | NtMeta(_) | NtPath(_) => false,
+                    NtItem(_) | NtMeta(_) | NtPath(_) => false,
                 },
                 token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
                     MetaVarKind::Block
@@ -170,15 +169,18 @@ impl<'a> Parser<'a> {
                 }
             },
             NonterminalKind::Pat(pat_kind) => {
-                NtPat(self.collect_tokens_no_attrs(|this| match pat_kind {
-                    PatParam { .. } => this.parse_pat_no_top_alt(None, None),
-                    PatWithOr => this.parse_pat_no_top_guard(
-                        None,
-                        RecoverComma::No,
-                        RecoverColon::No,
-                        CommaRecoveryMode::EitherTupleOrPipe,
-                    ),
-                })?)
+                return Ok(ParseNtResult::Pat(
+                    self.collect_tokens_no_attrs(|this| match pat_kind {
+                        PatParam { .. } => this.parse_pat_no_top_alt(None, None),
+                        PatWithOr => this.parse_pat_no_top_guard(
+                            None,
+                            RecoverComma::No,
+                            RecoverColon::No,
+                            CommaRecoveryMode::EitherTupleOrPipe,
+                        ),
+                    })?,
+                    pat_kind,
+                ));
             }
             NonterminalKind::Expr(_) => NtExpr(self.parse_expr_force_collect()?),
             NonterminalKind::Literal => {

compiler/rustc_parse/src/parser/pat.rs

@@ -2,7 +2,8 @@ use std::ops::Bound;
 
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw, Token};
+use rustc_ast::token::NtPatKind::*;
+use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw, MetaVarKind, Token};
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_ast::visit::{self, Visitor};
 use rustc_ast::{
@@ -30,7 +31,7 @@ use crate::errors::{
     UnexpectedVertVertInPattern, WrapInParens,
 };
 use crate::parser::expr::{DestructuredFloat, could_be_unclosed_char_literal};
-use crate::{exp, maybe_recover_from_interpolated_ty_qpath, maybe_whole};
+use crate::{exp, maybe_recover_from_interpolated_ty_qpath};
 
 #[derive(PartialEq, Copy, Clone)]
 pub enum Expected {
@@ -689,6 +690,27 @@ impl<'a> Parser<'a> {
         PatVisitor { parser: self, stmt, arm: None, field: None }.visit_stmt(stmt);
     }
 
+    fn eat_metavar_pat(&mut self) -> Option<P<Pat>> {
+        // Must try both kinds of pattern nonterminals.
+        if let Some(pat) = self.eat_metavar_seq_with_matcher(
+            |mv_kind| matches!(mv_kind, MetaVarKind::Pat(PatParam { .. })),
+            |this| this.parse_pat_no_top_alt(None, None),
+        ) {
+            Some(pat)
+        } else if let Some(pat) = self.eat_metavar_seq(MetaVarKind::Pat(PatWithOr), |this| {
+            this.parse_pat_no_top_guard(
+                None,
+                RecoverComma::No,
+                RecoverColon::No,
+                CommaRecoveryMode::EitherTupleOrPipe,
+            )
+        }) {
+            Some(pat)
+        } else {
+            None
+        }
+    }
+
     /// Parses a pattern, with a setting whether modern range patterns (e.g., `a..=b`, `a..b` are
     /// allowed).
     fn parse_pat_with_range_pat(
@@ -698,7 +720,10 @@ impl<'a> Parser<'a> {
         syntax_loc: Option<PatternLocation>,
     ) -> PResult<'a, P<Pat>> {
         maybe_recover_from_interpolated_ty_qpath!(self, true);
-        maybe_whole!(self, NtPat, |pat| pat);
+
+        if let Some(pat) = self.eat_metavar_pat() {
+            return Ok(pat);
+        }
 
         let mut lo = self.token.span;
 
@@ -1043,10 +1068,8 @@ impl<'a> Parser<'a> {
         self.recover_additional_muts();
 
         // Make sure we don't allow e.g. `let mut $p;` where `$p:pat`.
-        if let token::Interpolated(nt) = &self.token.kind {
-            if let token::NtPat(..) = &**nt {
-                self.expected_ident_found_err().emit();
-            }
+        if let Some(MetaVarKind::Pat(_)) = self.token.is_metavar_seq() {
+            self.expected_ident_found_err().emit();
         }
 
         // Parse the pattern we hope to be an identifier.

tests/ui/macros/trace_faulty_macros.rs

@@ -46,7 +46,7 @@ macro_rules! test {
     (let $p:pat = $e:expr) => {test!(($p,$e))};
     // this should be expr
     //           vvv
-    (($p:pat, $e:pat)) => {let $p = $e;}; //~ ERROR expected expression, found pattern `1+1`
+    (($p:pat, $e:pat)) => {let $p = $e;}; //~ ERROR expected expression, found `pat` metavariable
 }
 
 fn foo() {

tests/ui/macros/trace_faulty_macros.stderr

@@ -50,7 +50,7 @@ LL |     my_recursive_macro!();
    = note: expanding `my_recursive_macro! {  }`
    = note: to `my_recursive_macro! ();`
 
-error: expected expression, found pattern `A { a : a, b : 0, c : _, .. }`
+error: expected expression, found `pat` metavariable
   --> $DIR/trace_faulty_macros.rs:16:9
    |
 LL |         $a
@@ -69,22 +69,15 @@ LL | #[derive(Debug)]
 LL | fn use_derive_macro_as_attr() {}
    | -------------------------------- not a `struct`, `enum` or `union`
 
-error: expected expression, found pattern `1+1`
+error: expected expression, found `pat` metavariable
   --> $DIR/trace_faulty_macros.rs:49:37
    |
-LL |     (let $p:pat = $e:expr) => {test!(($p,$e))};
-   |                                          -- this is interpreted as expression, but it is expected to be pattern
-...
 LL |     (($p:pat, $e:pat)) => {let $p = $e;};
    |                                     ^^ expected expression
 ...
 LL |     test!(let x = 1+1);
-   |     ------------------
-   |     |             |
-   |     |             this is expected to be expression
-   |     in this macro invocation
+   |     ------------------ in this macro invocation
    |
-   = note: when forwarding a matched fragment to another macro-by-example, matchers in the second macro will see an opaque AST of the fragment type, not the underlying tokens
    = note: this error originates in the macro `test` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: trace_macro

tests/ui/parser/issues/issue-65122-mac-invoc-in-mut-patterns.rs

@@ -12,7 +12,7 @@ macro_rules! mac2 {
     ($eval:pat) => {
         let mut $eval = ();
         //~^ ERROR `mut` must be followed by a named binding
-        //~| ERROR expected identifier, found `does_not_exist!()`
+        //~| ERROR expected identifier, found metavariable
     };
 }