Merge #7827

7827: Fix proc macro TokenStream::from_str token ids r=vlad20012 a=vlad20012

To be honest, I don't know what it changes from a user perspective.

Internally, this fixes spans (token ids) of a `TokenStream` parsed from a string:

```rust
#[proc_macro_derive(FooDerive)]
pub fn foo_derive(item: TokenStream) -> TokenStream {
    "fn foo() {}".parse().unwrap()
}
```

Previously, `TokenStream` was constructed from tokens with incremental ids (that conflicted with call-site tokens). Now they are `-1`.

Co-authored-by: vlad20012 <[email protected]>

Author: bors[bot]

crates/proc_macro_srv/src/rustc_server.rs

@@ -14,7 +14,6 @@ use std::collections::HashMap;
 use std::hash::Hash;
 use std::iter::FromIterator;
 use std::ops::Bound;
-use std::str::FromStr;
 use std::{ascii, vec::IntoIter};
 
 type Group = tt::Subtree;
@@ -278,6 +277,42 @@ impl server::FreeFunctions for Rustc {
     }
 }
 
+fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
+    tt::Subtree {
+        delimiter: subtree.delimiter.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
+        token_trees: subtree
+            .token_trees
+            .into_iter()
+            .map(|t| token_tree_replace_token_ids_with_unspecified(t))
+            .collect(),
+    }
+}
+
+fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
+    match tt {
+        tt::TokenTree::Leaf(leaf) => {
+            tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
+        }
+        tt::TokenTree::Subtree(subtree) => {
+            tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
+        }
+    }
+}
+
+fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
+    match leaf {
+        tt::Leaf::Literal(lit) => {
+            tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
+        }
+        tt::Leaf::Punct(punct) => {
+            tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
+        }
+        tt::Leaf::Ident(ident) => {
+            tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
+        }
+    }
+}
+
 impl server::TokenStream for Rustc {
     fn new(&mut self) -> Self::TokenStream {
         Self::TokenStream::new()
@@ -287,7 +322,8 @@ impl server::TokenStream for Rustc {
         stream.is_empty()
     }
     fn from_str(&mut self, src: &str) -> Self::TokenStream {
-        Self::TokenStream::from_str(src).expect("cannot parse string")
+        let (subtree, _) = mbe::parse_to_token_tree(src).expect("cannot parse string");
+        TokenStream::with_subtree(subtree_replace_token_ids_with_unspecified(subtree))
     }
     fn to_string(&mut self, stream: &Self::TokenStream) -> String {
         stream.to_string()