NFA parser for mbe matcher

crates/mbe/Cargo.toml

@@ -18,6 +18,7 @@ syntax = { path = "../syntax", version = "0.0.0" }
 parser = { path = "../parser", version = "0.0.0" }
 tt = { path = "../tt", version = "0.0.0" }
 test_utils = { path = "../test_utils", version = "0.0.0" }
+stdx = { path = "../stdx", version = "0.0.0" }
 
 [dev-dependencies]
 profile = { path = "../profile", version = "0.0.0" }

crates/mbe/src/benchmark.rs

@@ -40,18 +40,12 @@ fn benchmark_expand_macro_rules() {
             .into_iter()
             .map(|(id, tt)| {
                 let res = rules[&id].expand(&tt);
-                if res.err.is_some() {
-                    // FIXME:
-                    // Currently `invocation_fixtures` will generate some correct invocations but
-                    // cannot be expanded by mbe. We ignore errors here.
-                    // See: https://github.com/rust-analyzer/rust-analyzer/issues/4777
-                    eprintln!("err from {} {:?}", id, res.err);
-                }
+                assert!(res.err.is_none());
                 res.value.token_trees.len()
             })
             .sum()
     };
-    assert_eq!(hash, 66995);
+    assert_eq!(hash, 69413);
 }
 
 fn macro_rules_fixtures() -> FxHashMap<String, MacroRules> {
@@ -77,7 +71,7 @@ fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree> {
         .collect()
 }
 
-// Generate random invocation fixtures from rules
+/// Generate random invocation fixtures from rules
 fn invocation_fixtures(rules: &FxHashMap<String, MacroRules>) -> Vec<(String, tt::Subtree)> {
     let mut seed = 123456789;
     let mut res = Vec::new();
@@ -86,11 +80,31 @@ fn invocation_fixtures(rules: &FxHashMap<String, MacroRules>) -> Vec<(String, tt
         for rule in &it.rules {
             // Generate twice
             for _ in 0..2 {
-                let mut subtree = tt::Subtree::default();
-                for op in rule.lhs.iter() {
-                    collect_from_op(op, &mut subtree, &mut seed);
+                // The input are generated by filling the `Op` randomly.
+                // However, there are some cases generated are ambiguous for expanding, for example:
+                // ```rust
+                // macro_rules! m {
+                //    ($($t:ident),* as $ty:ident) => {}
+                // }
+                // m!(as u32);  // error: local ambiguity: multiple parsing options: built-in NTs ident ('t') or 1 other option.
+                // ```
+                //
+                // So we just skip any error cases and try again
+                let mut try_cnt = 0;
+                loop {
+                    let mut subtree = tt::Subtree::default();
+                    for op in rule.lhs.iter() {
+                        collect_from_op(op, &mut subtree, &mut seed);
+                    }
+                    if it.expand(&subtree).err.is_none() {
+                        res.push((name.clone(), subtree));
+                        break;
+                    }
+                    try_cnt += 1;
+                    if try_cnt > 100 {
+                        panic!("invocaton fixture {} cannot be generated.\n", name);
+                    }
                 }
-                res.push((name.clone(), subtree));
             }
         }
     }

crates/mbe/src/expander.rs

@@ -5,7 +5,7 @@
 mod matcher;
 mod transcriber;
 
-use rustc_hash::FxHashMap;
+use smallvec::SmallVec;
 use syntax::SmolStr;
 
 use crate::{ExpandError, ExpandResult};
@@ -28,10 +28,10 @@ pub(crate) fn expand_rules(
                 return ExpandResult::ok(value);
             }
         }
-        // Use the rule if we matched more tokens, or had fewer errors
+        // Use the rule if we matched more tokens, or bound variables count
         if let Some((prev_match, _)) = &match_ {
-            if (new_match.unmatched_tts, new_match.err_count)
-                < (prev_match.unmatched_tts, prev_match.err_count)
+            if (new_match.unmatched_tts, -(new_match.bound_count as i32))
+                < (prev_match.unmatched_tts, -(prev_match.bound_count as i32))
             {
                 match_ = Some((new_match, rule));
             }
@@ -94,19 +94,19 @@ pub(crate) fn expand_rules(
 /// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
 /// `tt::TokenTree`, where the index to select a particular `TokenTree` among
 /// many is not a plain `usize`, but an `&[usize]`.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 struct Bindings {
-    inner: FxHashMap<SmolStr, Binding>,
+    inner: SmallVec<[(SmolStr, Binding); 4]>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 enum Binding {
     Fragment(Fragment),
     Nested(Vec<Binding>),
     Empty,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 enum Fragment {
     /// token fragments are just copy-pasted into the output
     Tokens(tt::TokenTree),