Merge #10872

10872: ide_db: build symbol index from crate def map r=Veykril a=jhgg

fixes #4842, #10764

Is this looking correct? 👀 

- [x] build the symbol index based upon the CrateDefMap for the given crate in `crate_symbols`
   - [x] make it multi threaded again, and figure out how to cache each moduleid's symbol index in salsa.
   - [x] NavigationTarget for names in macros is wrong, need to figure out how to compute a text range in the original file id?
   - [x] cleanup some duped code
   - [x] collect macros from `ItemScope.declared_macros()` into symbol index.
        - [x] store declared macros in `ItemScope` so we can figure out where macros were defined for the index.  
   - [x] do something about `SymbolIndex::for_files` - ideally it should use the new module symbol index stuff. 
       - [x] delete `source_file_to_file_symbols` & co...
           - [x] figure out what to do about `library_symbols` 
           - [x] maybe... speed up the new `library_symbols` - the new impl is probably much slower, and definitely much less parallel. **deciding to do nothing here, we can optimize later if necerssary.** 
   - [x] fix failing test: `navigation_target::tests::test_nav_for_symbol` - notably the crate def map doesn't seem to find declarations inside function. 
       - [x] now a bunch of other tests are failing around auto_import & qualify_path handlers. :(
           - [x] need to assoc items in traits and impls
 

Co-authored-by: Jake Heinz <[email protected]>

Author: bors[bot]

crates/base_db/src/fixture.rs

@@ -19,8 +19,8 @@ use crate::{
 pub const WORKSPACE: SourceRootId = SourceRootId(0);
 
 pub trait WithFixture: Default + SourceDatabaseExt + 'static {
-    fn with_single_file(text: &str) -> (Self, FileId) {
-        let fixture = ChangeFixture::parse(text);
+    fn with_single_file(ra_fixture: &str) -> (Self, FileId) {
+        let fixture = ChangeFixture::parse(ra_fixture);
         let mut db = Self::default();
         fixture.change.apply(&mut db);
         assert_eq!(fixture.files.len(), 1);

crates/base_db/src/lib.rs

@@ -102,7 +102,7 @@ fn source_root_crates(db: &dyn SourceDatabaseExt, id: SourceRootId) -> Arc<FxHas
             let root_file = graph[krate].root_file_id;
             db.file_source_root(root_file) == id
         })
-        .collect::<FxHashSet<_>>();
+        .collect();
     Arc::new(res)
 }
 

crates/hir/src/lib.rs

@@ -40,18 +40,14 @@ use hir_def::{
     adt::{ReprKind, VariantData},
     body::{BodyDiagnostic, SyntheticSyntax},
     expr::{BindingAnnotation, LabelId, Pat, PatId},
-    item_tree::ItemTreeNode,
     lang_item::LangItemTarget,
     nameres,
     per_ns::PerNs,
     resolver::{HasResolver, Resolver},
-    src::HasSource as _,
-    AdtId, AssocContainerId, AssocItemId, AssocItemLoc, AttrDefId, ConstId, ConstParamId,
-    DefWithBodyId, EnumId, FunctionId, GenericDefId, HasModule, ImplId, LifetimeParamId,
-    LocalEnumVariantId, LocalFieldId, Lookup, ModuleId, StaticId, StructId, TraitId, TypeAliasId,
-    TypeParamId, UnionId,
+    AttrDefId, ConstId, ConstParamId, EnumId, FunctionId, GenericDefId, HasModule, LifetimeParamId,
+    LocalEnumVariantId, LocalFieldId, StaticId, StructId, TypeAliasId, TypeParamId, UnionId,
 };
-use hir_expand::{name::name, MacroCallKind, MacroDefId, MacroDefKind};
+use hir_expand::{name::name, MacroCallKind, MacroDefKind};
 use hir_ty::{
     autoderef,
     consteval::ConstExt,
@@ -109,14 +105,28 @@ pub use {
         attr::{Attr, Attrs, AttrsWithOwner, Documentation},
         find_path::PrefixKind,
         import_map,
-        nameres::ModuleSource,
+        item_scope::ItemScope,
+        item_tree::ItemTreeNode,
+        nameres::{DefMap, ModuleData, ModuleOrigin, ModuleSource},
         path::{ModPath, PathKind},
+        src::HasSource as DefHasSource, // xx: I don't like this shadowing of HasSource... :(
         type_ref::{Mutability, TypeRef},
         visibility::Visibility,
+        AdtId,
+        AssocContainerId,
+        AssocItemId,
+        AssocItemLoc,
+        DefWithBodyId,
+        ImplId,
+        ItemLoc,
+        Lookup,
+        ModuleDefId,
+        ModuleId,
+        TraitId,
     },
     hir_expand::{
         name::{known, Name},
-        ExpandResult, HirFileId, InFile, MacroFile, Origin,
+        ExpandResult, HirFileId, InFile, MacroDefId, MacroFile, Origin,
     },
     hir_ty::display::HirDisplay,
 };

crates/hir/src/semantics.rs

@@ -143,6 +143,10 @@ impl<'db, DB: HirDatabase> Semantics<'db, DB> {
         self.imp.parse(file_id)
     }
 
+    pub fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode> {
+        self.imp.parse_or_expand(file_id)
+    }
+
     pub fn expand(&self, macro_call: &ast::MacroCall) -> Option<SyntaxNode> {
         self.imp.expand(macro_call)
     }
@@ -416,6 +420,12 @@ impl<'db> SemanticsImpl<'db> {
         tree
     }
 
+    fn parse_or_expand(&self, file_id: HirFileId) -> Option<SyntaxNode> {
+        let node = self.db.parse_or_expand(file_id)?;
+        self.cache(node.clone(), file_id);
+        Some(node)
+    }
+
     fn expand(&self, macro_call: &ast::MacroCall) -> Option<SyntaxNode> {
         let sa = self.analyze(macro_call.syntax());
         let file_id = sa.expand(self.db, InFile::new(sa.file_id, macro_call))?;

crates/hir_def/src/item_scope.rs

@@ -44,6 +44,8 @@ pub struct ItemScope {
     /// The defs declared in this scope. Each def has a single scope where it is
     /// declared.
     declarations: Vec<ModuleDefId>,
+    macro_declarations: Vec<MacroDefId>,
+
     impls: Vec<ImplId>,
     unnamed_consts: Vec<ConstId>,
     /// Traits imported via `use Trait as _;`.
@@ -101,6 +103,10 @@ impl ItemScope {
         self.declarations.iter().copied()
     }
 
+    pub fn macro_declarations(&self) -> impl Iterator<Item = MacroDefId> + '_ {
+        self.macro_declarations.iter().copied()
+    }
+
     pub fn impls(&self) -> impl Iterator<Item = ImplId> + ExactSizeIterator + '_ {
         self.impls.iter().copied()
     }
@@ -163,6 +169,10 @@ impl ItemScope {
         self.declarations.push(def)
     }
 
+    pub(crate) fn declare_macro(&mut self, def: MacroDefId) {
+        self.macro_declarations.push(def);
+    }
+
     pub(crate) fn get_legacy_macro(&self, name: &Name) -> Option<MacroDefId> {
         self.legacy_macros.get(name).copied()
     }
@@ -336,7 +346,8 @@ impl ItemScope {
             values,
             macros,
             unresolved,
-            declarations: defs,
+            declarations,
+            macro_declarations,
             impls,
             unnamed_consts,
             unnamed_trait_imports,
@@ -348,7 +359,8 @@ impl ItemScope {
         values.shrink_to_fit();
         macros.shrink_to_fit();
         unresolved.shrink_to_fit();
-        defs.shrink_to_fit();
+        declarations.shrink_to_fit();
+        macro_declarations.shrink_to_fit();
         impls.shrink_to_fit();
         unnamed_consts.shrink_to_fit();
         unnamed_trait_imports.shrink_to_fit();

crates/hir_def/src/nameres.rs

@@ -148,7 +148,7 @@ pub enum ModuleOrigin {
 }
 
 impl ModuleOrigin {
-    fn declaration(&self) -> Option<AstId<ast::Module>> {
+    pub fn declaration(&self) -> Option<AstId<ast::Module>> {
         match self {
             ModuleOrigin::File { declaration: module, .. }
             | ModuleOrigin::Inline { definition: module, .. } => Some(*module),

crates/hir_def/src/nameres/collector.rs

@@ -594,6 +594,7 @@ impl DefCollector<'_> {
     ) {
         // Textual scoping
         self.define_legacy_macro(module_id, name.clone(), macro_);
+        self.def_map.modules[module_id].scope.declare_macro(macro_);
 
         // Module scoping
         // In Rust, `#[macro_export]` macros are unconditionally visible at the
@@ -632,6 +633,7 @@ impl DefCollector<'_> {
     ) {
         let vis =
             self.def_map.resolve_visibility(self.db, module_id, vis).unwrap_or(Visibility::Public);
+        self.def_map.modules[module_id].scope.declare_macro(macro_);
         self.update(module_id, &[(Some(name), PerNs::macros(macro_, vis))], vis, ImportType::Named);
     }
 
@@ -640,6 +642,7 @@ impl DefCollector<'_> {
     /// A proc macro is similar to normal macro scope, but it would not visible in legacy textual scoped.
     /// And unconditionally exported.
     fn define_proc_macro(&mut self, name: Name, macro_: MacroDefId) {
+        self.def_map.modules[self.def_map.root].scope.declare_macro(macro_);
         self.update(
             self.def_map.root,
             &[(Some(name), PerNs::macros(macro_, Visibility::Public))],

crates/ide/src/lib.rs

@@ -367,8 +367,8 @@ impl Analysis {
     pub fn symbol_search(&self, query: Query) -> Cancellable<Vec<NavigationTarget>> {
         self.with_db(|db| {
             symbol_index::world_symbols(db, query)
-                .into_iter()
-                .map(|s| s.to_nav(db))
+                .into_iter() // xx: should we make this a par iter?
+                .filter_map(|s| s.try_to_nav(db))
                 .collect::<Vec<_>>()
         })
     }

crates/ide/src/navigation_target.rs

@@ -167,10 +167,13 @@ impl NavigationTarget {
     }
 }
 
-impl ToNav for FileSymbol {
-    fn to_nav(&self, db: &RootDatabase) -> NavigationTarget {
-        NavigationTarget {
-            file_id: self.file_id,
+impl TryToNav for FileSymbol {
+    fn try_to_nav(&self, db: &RootDatabase) -> Option<NavigationTarget> {
+        let full_range = self.loc.original_range(db)?;
+        let name_range = self.loc.original_name_range(db)?;
+
+        Some(NavigationTarget {
+            file_id: full_range.file_id,
             name: self.name.clone(),
             kind: Some(match self.kind {
                 FileSymbolKind::Function => SymbolKind::Function,
@@ -184,12 +187,12 @@ impl ToNav for FileSymbol {
                 FileSymbolKind::Macro => SymbolKind::Macro,
                 FileSymbolKind::Union => SymbolKind::Union,
             }),
-            full_range: self.range,
-            focus_range: self.name_range,
+            full_range: full_range.range,
+            focus_range: Some(name_range.range),
             container_name: self.container_name.clone(),
             description: description_from_symbol(db, self),
             docs: None,
-        }
+        })
     }
 }
 
@@ -517,8 +520,7 @@ impl TryToNav for hir::ConstParam {
 /// e.g. `struct Name`, `enum Name`, `fn Name`
 pub(crate) fn description_from_symbol(db: &RootDatabase, symbol: &FileSymbol) -> Option<String> {
     let sema = Semantics::new(db);
-    let parse = sema.parse(symbol.file_id);
-    let node = symbol.ptr.to_node(parse.syntax());
+    let node = symbol.loc.syntax(&sema)?;
 
     match_ast! {
         match node {

crates/ide/src/status.rs

@@ -11,7 +11,6 @@ use ide_db::{
 };
 use itertools::Itertools;
 use profile::{memory_usage, Bytes};
-use rustc_hash::FxHashMap;
 use std::env;
 use stdx::format_to;
 use syntax::{ast, Parse, SyntaxNode};
@@ -149,20 +148,16 @@ impl fmt::Display for LibrarySymbolsStats {
     }
 }
 
-impl FromIterator<TableEntry<(), Arc<FxHashMap<SourceRootId, SymbolIndex>>>>
-    for LibrarySymbolsStats
-{
+impl FromIterator<TableEntry<SourceRootId, Arc<SymbolIndex>>> for LibrarySymbolsStats {
     fn from_iter<T>(iter: T) -> LibrarySymbolsStats
     where
-        T: IntoIterator<Item = TableEntry<(), Arc<FxHashMap<SourceRootId, SymbolIndex>>>>,
+        T: IntoIterator<Item = TableEntry<SourceRootId, Arc<SymbolIndex>>>,
     {
         let mut res = LibrarySymbolsStats::default();
         for entry in iter {
-            let value = entry.value.unwrap();
-            for symbols in value.values() {
-                res.total += symbols.len();
-                res.size += symbols.memory_size();
-            }
+            let symbols = entry.value.unwrap();
+            res.total += symbols.len();
+            res.size += symbols.memory_size();
         }
         res
     }

crates/ide_db/src/apply_change.rs

@@ -137,7 +137,7 @@ impl RootDatabase {
             hir::db::InternTypeParamIdQuery
 
             // SymbolsDatabase
-            crate::symbol_index::FileSymbolsQuery
+            crate::symbol_index::ModuleSymbolsQuery
             crate::symbol_index::LibrarySymbolsQuery
             crate::symbol_index::LocalRootsQuery
             crate::symbol_index::LibraryRootsQuery

crates/ide_db/src/items_locator.rs

@@ -133,9 +133,8 @@ fn get_name_definition(
     import_candidate: &FileSymbol,
 ) -> Option<Definition> {
     let _p = profile::span("get_name_definition");
-    let file_id = import_candidate.file_id;
 
-    let candidate_node = import_candidate.ptr.to_node(sema.parse(file_id).syntax());
+    let candidate_node = import_candidate.loc.syntax(sema)?;
     let candidate_name_node = if candidate_node.kind() != NAME {
         candidate_node.children().find(|it| it.kind() == NAME)?
     } else {