Skip to content

Commit da24953

Browse files
borsbors
committed
Auto merge of rust-lang#14061 - Veykril:generic-tt, r=Veykril
Make tt generic over the span data This also fixes up our delimiter representation in tt, it is no longer optional (we use invisible delims in the same way as before, that is still incorrectly) and we now store two spans instead of one. These changes should help with adjusting our token map. Though this will probably break proc-macros in some ways, will need to test that for now.
2 parents d805c74 + 41a46a7 commit da24953

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+804
-567
lines changed

‎crates/base-db/src/fixture.rs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use rustc_hash::FxHashMap;
66
use test_utils::{
77
extract_range_or_offset, Fixture, RangeOrOffset, CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
88
};
9-
use tt::Subtree;
9+
use tt::token_id::Subtree;
1010
use vfs::{file_set::FileSet, VfsPath};
1111

1212
use crate::{
@@ -495,16 +495,15 @@ impl ProcMacroExpander for MirrorProcMacroExpander {
495495
_: &Env,
496496
) -> Result<Subtree, ProcMacroExpansionError> {
497497
fn traverse(input: &Subtree) -> Subtree {
498-
let mut res = Subtree::default();
499-
res.delimiter = input.delimiter;
498+
let mut token_trees = vec![];
500499
for tt in input.token_trees.iter().rev() {
501500
let tt = match tt {
502501
tt::TokenTree::Leaf(leaf) => tt::TokenTree::Leaf(leaf.clone()),
503502
tt::TokenTree::Subtree(sub) => tt::TokenTree::Subtree(traverse(sub)),
504503
};
505-
res.token_trees.push(tt);
504+
token_trees.push(tt);
506505
}
507-
res
506+
Subtree { delimiter: input.delimiter, token_trees }
508507
}
509508
Ok(traverse(input))
510509
}

‎crates/base-db/src/input.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use cfg::CfgOptions;
1212
use rustc_hash::FxHashMap;
1313
use stdx::hash::{NoHashHashMap, NoHashHashSet};
1414
use syntax::SmolStr;
15-
use tt::Subtree;
15+
use tt::token_id::Subtree;
1616
use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
1717

1818
/// Files are grouped into source roots. A source root is a directory on the

‎crates/cfg/src/cfg_expr.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ impl From<CfgAtom> for CfgExpr {
6666
}
6767

6868
impl CfgExpr {
69-
pub fn parse(tt: &tt::Subtree) -> CfgExpr {
69+
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
7070
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
7171
}
7272
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -85,7 +85,7 @@ impl CfgExpr {
8585
}
8686
}
8787

88-
fn next_cfg_expr(it: &mut SliceIter<'_, tt::TokenTree>) -> Option<CfgExpr> {
88+
fn next_cfg_expr<S>(it: &mut SliceIter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
8989
let name = match it.next() {
9090
None => return None,
9191
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.text.clone(),

‎crates/hir-def/src/adt.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
use std::sync::Arc;
44

5+
use crate::tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
56
use base_db::CrateId;
67
use either::Either;
78
use hir_expand::{
@@ -12,7 +13,6 @@ use intern::Interned;
1213
use la_arena::{Arena, ArenaMap};
1314
use rustc_abi::{Integer, IntegerType};
1415
use syntax::ast::{self, HasName, HasVisibility};
15-
use tt::{Delimiter, DelimiterKind, Leaf, Subtree, TokenTree};
1616

1717
use crate::{
1818
body::{CfgExpander, LowerCtx},
@@ -82,7 +82,7 @@ fn repr_from_value(
8282

8383
fn parse_repr_tt(tt: &Subtree) -> Option<ReprOptions> {
8484
match tt.delimiter {
85-
Some(Delimiter { kind: DelimiterKind::Parenthesis, .. }) => {}
85+
Delimiter { kind: DelimiterKind::Parenthesis, .. } => {}
8686
_ => return None,
8787
}
8888

‎crates/hir-def/src/attr.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ use syntax::{
1616
ast::{self, HasAttrs, IsString},
1717
AstPtr, AstToken, SmolStr, TextRange, TextSize,
1818
};
19-
use tt::Subtree;
2019

2120
use crate::{
2221
db::DefDatabase,
@@ -234,7 +233,7 @@ impl Attrs {
234233

235234
pub fn has_doc_hidden(&self) -> bool {
236235
self.by_key("doc").tt_values().any(|tt| {
237-
tt.delimiter_kind() == Some(DelimiterKind::Parenthesis) &&
236+
tt.delimiter.kind == DelimiterKind::Parenthesis &&
238237
matches!(&*tt.token_trees, [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] if ident.text == "hidden")
239238
})
240239
}
@@ -628,7 +627,7 @@ pub struct AttrQuery<'attr> {
628627
}
629628

630629
impl<'attr> AttrQuery<'attr> {
631-
pub fn tt_values(self) -> impl Iterator<Item = &'attr Subtree> {
630+
pub fn tt_values(self) -> impl Iterator<Item = &'attr crate::tt::Subtree> {
632631
self.attrs().filter_map(|attr| attr.token_tree_value())
633632
}
634633

‎crates/hir-def/src/data.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ impl FunctionData {
142142
}
143143
}
144144

145-
fn parse_rustc_legacy_const_generics(tt: &tt::Subtree) -> Box<[u32]> {
145+
fn parse_rustc_legacy_const_generics(tt: &crate::tt::Subtree) -> Box<[u32]> {
146146
let mut indices = Vec::new();
147147
for args in tt.token_trees.chunks(2) {
148148
match &args[0] {

‎crates/hir-def/src/lib.rs

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ use nameres::DefMap;
7979
use stdx::impl_from;
8080
use syntax::ast;
8181

82+
use ::tt::token_id as tt;
83+
8284
use crate::{
8385
adt::VariantData,
8486
builtin_type::BuiltinType,
@@ -973,15 +975,19 @@ fn attr_macro_as_call_id(
973975
def: MacroDefId,
974976
is_derive: bool,
975977
) -> MacroCallId {
976-
let mut arg = match macro_attr.input.as_deref() {
977-
Some(AttrInput::TokenTree(tt, map)) => (tt.clone(), map.clone()),
978-
_ => Default::default(),
978+
let arg = match macro_attr.input.as_deref() {
979+
Some(AttrInput::TokenTree(tt, map)) => (
980+
{
981+
let mut tt = tt.clone();
982+
tt.delimiter = tt::Delimiter::UNSPECIFIED;
983+
tt
984+
},
985+
map.clone(),
986+
),
987+
_ => (tt::Subtree::empty(), Default::default()),
979988
};
980989

981-
// The parentheses are always disposed here.
982-
arg.0.delimiter = None;
983-
984-
let res = def.as_lazy_macro(
990+
def.as_lazy_macro(
985991
db.upcast(),
986992
krate,
987993
MacroCallKind::Attr {
@@ -990,8 +996,7 @@ fn attr_macro_as_call_id(
990996
invoc_attr_index: macro_attr.id,
991997
is_derive,
992998
},
993-
);
994-
res
999+
)
9951000
}
9961001
intern::impl_internable!(
9971002
crate::type_ref::TypeRef,

‎crates/hir-def/src/macro_expansion_tests.rs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ use syntax::{
3030
SyntaxKind::{self, COMMENT, EOF, IDENT, LIFETIME_IDENT},
3131
SyntaxNode, TextRange, T,
3232
};
33-
use tt::{Subtree, TokenId};
33+
use tt::token_id::{Subtree, TokenId};
3434

3535
use crate::{
3636
db::DefDatabase, macro_id_to_def_id, nameres::ModuleSource, resolver::HasResolver,
@@ -253,9 +253,9 @@ fn extract_id_ranges(ranges: &mut Vec<(TextRange, TokenId)>, map: &TokenMap, tre
253253
tree.token_trees.iter().for_each(|tree| match tree {
254254
tt::TokenTree::Leaf(leaf) => {
255255
let id = match leaf {
256-
tt::Leaf::Literal(it) => it.id,
257-
tt::Leaf::Punct(it) => it.id,
258-
tt::Leaf::Ident(it) => it.id,
256+
tt::Leaf::Literal(it) => it.span,
257+
tt::Leaf::Punct(it) => it.span,
258+
tt::Leaf::Ident(it) => it.span,
259259
};
260260
ranges.extend(map.ranges_by_token(id, SyntaxKind::ERROR).map(|range| (range, id)));
261261
}

‎crates/hir-def/src/nameres/collector.rs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ use crate::{
4646
},
4747
path::{ImportAlias, ModPath, PathKind},
4848
per_ns::PerNs,
49+
tt,
4950
visibility::{RawVisibility, Visibility},
5051
AdtId, AstId, AstIdWithPath, ConstLoc, EnumLoc, EnumVariantId, ExternBlockLoc, FunctionId,
5152
FunctionLoc, ImplLoc, Intern, ItemContainerId, LocalModuleId, Macro2Id, Macro2Loc,
@@ -83,7 +84,8 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
8384
.enumerate()
8485
.map(|(idx, it)| {
8586
// FIXME: a hacky way to create a Name from string.
86-
let name = tt::Ident { text: it.name.clone(), id: tt::TokenId::unspecified() };
87+
let name =
88+
tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
8789
(
8890
name.as_name(),
8991
ProcMacroExpander::new(def_map.krate, base_db::ProcMacroId(idx as u32)),
@@ -451,7 +453,10 @@ impl DefCollector<'_> {
451453
directive.module_id,
452454
MacroCallKind::Attr {
453455
ast_id: ast_id.ast_id,
454-
attr_args: Default::default(),
456+
attr_args: std::sync::Arc::new((
457+
tt::Subtree::empty(),
458+
Default::default(),
459+
)),
455460
invoc_attr_index: attr.id,
456461
is_derive: false,
457462
},
@@ -1947,7 +1952,8 @@ impl ModCollector<'_, '_> {
19471952
let name = match attrs.by_key("rustc_builtin_macro").string_value() {
19481953
Some(it) => {
19491954
// FIXME: a hacky way to create a Name from string.
1950-
name = tt::Ident { text: it.clone(), id: tt::TokenId::unspecified() }.as_name();
1955+
name =
1956+
tt::Ident { text: it.clone(), span: tt::TokenId::unspecified() }.as_name();
19511957
&name
19521958
}
19531959
None => {

‎crates/hir-def/src/nameres/proc_macro.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
//! Nameres-specific procedural macro data and helpers.
22
33
use hir_expand::name::{AsName, Name};
4-
use tt::{Leaf, TokenTree};
54

65
use crate::attr::Attrs;
6+
use crate::tt::{Leaf, TokenTree};
77

88
#[derive(Debug, PartialEq, Eq)]
99
pub struct ProcMacroDef {

‎crates/hir-expand/src/attrs.rs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ use intern::Interned;
88
use mbe::{syntax_node_to_token_tree, DelimiterKind, Punct};
99
use smallvec::{smallvec, SmallVec};
1010
use syntax::{ast, match_ast, AstNode, SmolStr, SyntaxNode};
11-
use tt::Subtree;
1211

1312
use crate::{
1413
db::AstDatabase,
1514
hygiene::Hygiene,
1615
mod_path::{ModPath, PathKind},
1716
name::AsName,
17+
tt::{self, Subtree},
1818
InFile,
1919
};
2020

@@ -117,7 +117,10 @@ impl RawAttrs {
117117
let index = attr.id;
118118
let attrs =
119119
parts.enumerate().take(1 << AttrId::CFG_ATTR_BITS).filter_map(|(idx, attr)| {
120-
let tree = Subtree { delimiter: None, token_trees: attr.to_vec() };
120+
let tree = Subtree {
121+
delimiter: tt::Delimiter::unspecified(),
122+
token_trees: attr.to_vec(),
123+
};
121124
// FIXME hygiene
122125
let hygiene = Hygiene::new_unhygienic();
123126
Attr::from_tt(db, &tree, &hygiene, index.with_cfg_attr(idx))
@@ -266,7 +269,7 @@ impl Attr {
266269
pub fn parse_path_comma_token_tree(&self) -> Option<impl Iterator<Item = ModPath> + '_> {
267270
let args = self.token_tree_value()?;
268271

269-
if args.delimiter_kind() != Some(DelimiterKind::Parenthesis) {
272+
if args.delimiter.kind != DelimiterKind::Parenthesis {
270273
return None;
271274
}
272275
let paths = args

‎crates/hir-expand/src/builtin_attr_macro.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
//! Builtin attributes.
22
3-
use crate::{db::AstDatabase, name, ExpandResult, MacroCallId, MacroCallKind};
3+
use crate::{db::AstDatabase, name, tt, ExpandResult, MacroCallId, MacroCallKind};
44

55
macro_rules! register_builtin {
66
( $(($name:ident, $variant:ident) => $expand:ident),* ) => {
@@ -97,7 +97,7 @@ fn derive_attr_expand(
9797
let loc = db.lookup_intern_macro_call(id);
9898
let derives = match &loc.kind {
9999
MacroCallKind::Attr { attr_args, is_derive: true, .. } => &attr_args.0,
100-
_ => return ExpandResult::ok(Default::default()),
100+
_ => return ExpandResult::ok(tt::Subtree::empty()),
101101
};
102102
pseudo_derive_attr_expansion(tt, derives)
103103
}
@@ -110,7 +110,7 @@ pub fn pseudo_derive_attr_expansion(
110110
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
111111
char,
112112
spacing: tt::Spacing::Alone,
113-
id: tt::TokenId::unspecified(),
113+
span: tt::TokenId::unspecified(),
114114
}))
115115
};
116116

‎crates/hir-expand/src/builtin_derive_macro.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
use base_db::{CrateOrigin, LangCrateOrigin};
44
use tracing::debug;
55

6+
use crate::tt::{self, TokenId};
67
use syntax::{
78
ast::{self, AstNode, HasGenericParams, HasModuleItem, HasName},
89
match_ast,
910
};
10-
use tt::TokenId;
1111

1212
use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
1313

@@ -92,7 +92,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
9292
})?;
9393
let name_token_id =
9494
token_map.token_by_range(name.syntax().text_range()).unwrap_or_else(TokenId::unspecified);
95-
let name_token = tt::Ident { id: name_token_id, text: name.text().into() };
95+
let name_token = tt::Ident { span: name_token_id, text: name.text().into() };
9696
let param_types = params
9797
.into_iter()
9898
.flat_map(|param_list| param_list.type_or_const_params())
@@ -101,7 +101,7 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
101101
let ty = param
102102
.ty()
103103
.map(|ty| mbe::syntax_node_to_token_tree(ty.syntax()).0)
104-
.unwrap_or_default();
104+
.unwrap_or_else(tt::Subtree::empty);
105105
Some(ty)
106106
} else {
107107
None
@@ -114,15 +114,15 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, ExpandError> {
114114
fn expand_simple_derive(tt: &tt::Subtree, trait_path: tt::Subtree) -> ExpandResult<tt::Subtree> {
115115
let info = match parse_adt(tt) {
116116
Ok(info) => info,
117-
Err(e) => return ExpandResult::only_err(e),
117+
Err(e) => return ExpandResult::with_err(tt::Subtree::empty(), e),
118118
};
119119
let (params, args): (Vec<_>, Vec<_>) = info
120120
.param_types
121121
.into_iter()
122122
.enumerate()
123123
.map(|(idx, param_ty)| {
124124
let ident = tt::Leaf::Ident(tt::Ident {
125-
id: tt::TokenId::unspecified(),
125+
span: tt::TokenId::unspecified(),
126126
text: format!("T{idx}").into(),
127127
});
128128
let ident_ = ident.clone();

0 commit comments

Comments
 (0)