Merge pull request #19084 from Veykril/push-muworpzpzqup

Split cache priming into distinct phases

Author: Veykril

crates/base-db/src/input.rs

@@ -129,9 +129,9 @@ impl fmt::Display for CrateName {
 }
 
 impl ops::Deref for CrateName {
-    type Target = str;
-    fn deref(&self) -> &str {
-        self.0.as_str()
+    type Target = Symbol;
+    fn deref(&self) -> &Symbol {
+        &self.0
     }
 }
 
@@ -230,8 +230,8 @@ impl fmt::Display for CrateDisplayName {
 }
 
 impl ops::Deref for CrateDisplayName {
-    type Target = str;
-    fn deref(&self) -> &str {
+    type Target = Symbol;
+    fn deref(&self) -> &Symbol {
         &self.crate_name
     }
 }

crates/hir-def/src/import_map.rs

@@ -519,7 +519,7 @@ mod tests {
                 crate_graph[krate]
                     .display_name
                     .as_ref()
-                    .is_some_and(|it| &**it.crate_name() == crate_name)
+                    .is_some_and(|it| it.crate_name().as_str() == crate_name)
             })
             .expect("could not find crate");
 

crates/hir-def/src/nameres.rs

@@ -337,7 +337,7 @@ impl DefMap {
     pub(crate) fn crate_def_map_query(db: &dyn DefDatabase, crate_id: CrateId) -> Arc<DefMap> {
         let crate_graph = db.crate_graph();
         let krate = &crate_graph[crate_id];
-        let name = krate.display_name.as_deref().unwrap_or_default();
+        let name = krate.display_name.as_deref().map(Symbol::as_str).unwrap_or_default();
         let _p = tracing::info_span!("crate_def_map_query", ?name).entered();
 
         let module_data = ModuleData::new(

crates/hir-expand/src/name.rs

@@ -262,6 +262,6 @@ impl AsName for ast::FieldKind {
 
 impl AsName for base_db::Dependency {
     fn as_name(&self) -> Name {
-        Name::new_root(&self.name)
+        Name::new_symbol_root((*self.name).clone())
     }
 }

crates/hir-expand/src/prettify_macro_expansion_.rs

@@ -41,9 +41,9 @@ pub fn prettify_macro_expansion(
                 } else if let Some(dep) =
                     target_crate.dependencies.iter().find(|dep| dep.crate_id == macro_def_crate)
                 {
-                    make::tokens::ident(&dep.name)
+                    make::tokens::ident(dep.name.as_str())
                 } else if let Some(crate_name) = &crate_graph[macro_def_crate].display_name {
-                    make::tokens::ident(crate_name.crate_name())
+                    make::tokens::ident(crate_name.crate_name().as_str())
                 } else {
                     return dollar_crate.clone();
                 }

crates/ide-db/src/prime_caches.rs

@@ -6,12 +6,13 @@ mod topologic_sort;
 
 use std::time::Duration;
 
-use hir::db::DefDatabase;
+use hir::{db::DefDatabase, Symbol};
+use itertools::Itertools;
 
 use crate::{
     base_db::{
         ra_salsa::{Database, ParallelDatabase, Snapshot},
-        Cancelled, CrateId, SourceDatabase, SourceRootDatabase,
+        Cancelled, CrateId, SourceDatabase,
     },
     symbol_index::SymbolsDatabase,
     FxIndexMap, RootDatabase,
@@ -21,11 +22,12 @@ use crate::{
 #[derive(Debug)]
 pub struct ParallelPrimeCachesProgress {
     /// the crates that we are currently priming.
-    pub crates_currently_indexing: Vec<String>,
+    pub crates_currently_indexing: Vec<Symbol>,
     /// the total number of crates we want to prime.
     pub crates_total: usize,
     /// the total number of crates that have finished priming
     pub crates_done: usize,
+    pub work_type: &'static str,
 }
 
 pub fn parallel_prime_caches(
@@ -47,41 +49,32 @@ pub fn parallel_prime_caches(
     };
 
     enum ParallelPrimeCacheWorkerProgress {
-        BeginCrate { crate_id: CrateId, crate_name: String },
+        BeginCrate { crate_id: CrateId, crate_name: Symbol },
         EndCrate { crate_id: CrateId },
     }
 
+    // We split off def map computation from other work,
+    // as the def map is the relevant one. Once the defmaps are computed
+    // the project is ready to go, the other indices are just nice to have for some IDE features.
+    #[derive(PartialOrd, Ord, PartialEq, Eq, Copy, Clone)]
+    enum PrimingPhase {
+        DefMap,
+        ImportMap,
+        CrateSymbols,
+    }
+
     let (work_sender, progress_receiver) = {
         let (progress_sender, progress_receiver) = crossbeam_channel::unbounded();
         let (work_sender, work_receiver) = crossbeam_channel::unbounded();
-        let graph = graph.clone();
-        let local_roots = db.local_roots();
         let prime_caches_worker = move |db: Snapshot<RootDatabase>| {
-            while let Ok((crate_id, crate_name)) = work_receiver.recv() {
+            while let Ok((crate_id, crate_name, kind)) = work_receiver.recv() {
                 progress_sender
                     .send(ParallelPrimeCacheWorkerProgress::BeginCrate { crate_id, crate_name })?;
 
-                // Compute the DefMap and possibly ImportMap
-                let file_id = graph[crate_id].root_file_id;
-                let root_id = db.file_source_root(file_id);
-                if db.source_root(root_id).is_library {
-                    db.crate_def_map(crate_id);
-                } else {
-                    // This also computes the DefMap
-                    db.import_map(crate_id);
-                }
-
-                // Compute the symbol search index.
-                // This primes the cache for `ide_db::symbol_index::world_symbols()`.
-                //
-                // We do this for workspace crates only (members of local_roots), because doing it
-                // for all dependencies could be *very* unnecessarily slow in a large project.
-                //
-                // FIXME: We should do it unconditionally if the configuration is set to default to
-                // searching dependencies (rust-analyzer.workspace.symbol.search.scope), but we
-                // would need to pipe that configuration information down here.
-                if local_roots.contains(&root_id) {
-                    db.crate_symbols(crate_id.into());
+                match kind {
+                    PrimingPhase::DefMap => _ = db.crate_def_map(crate_id),
+                    PrimingPhase::ImportMap => _ = db.import_map(crate_id),
+                    PrimingPhase::CrateSymbols => _ = db.crate_symbols(crate_id.into()),
                 }
 
                 progress_sender.send(ParallelPrimeCacheWorkerProgress::EndCrate { crate_id })?;
@@ -112,16 +105,34 @@ pub fn parallel_prime_caches(
     let mut crates_currently_indexing =
         FxIndexMap::with_capacity_and_hasher(num_worker_threads, Default::default());
 
+    let mut additional_phases = vec![];
+
     while crates_done < crates_total {
         db.unwind_if_cancelled();
 
         for crate_id in &mut crates_to_prime {
-            work_sender
-                .send((
-                    crate_id,
-                    graph[crate_id].display_name.as_deref().unwrap_or_default().to_owned(),
-                ))
-                .ok();
+            let krate = &graph[crate_id];
+            let name = krate
+                .display_name
+                .as_deref()
+                .cloned()
+                .unwrap_or_else(|| Symbol::integer(crate_id.into_raw().into_u32() as usize));
+            if krate.origin.is_lang() {
+                additional_phases.push((crate_id, name.clone(), PrimingPhase::ImportMap));
+            } else if krate.origin.is_local() {
+                // Compute the symbol search index.
+                // This primes the cache for `ide_db::symbol_index::world_symbols()`.
+                //
+                // We do this for workspace crates only (members of local_roots), because doing it
+                // for all dependencies could be *very* unnecessarily slow in a large project.
+                //
+                // FIXME: We should do it unconditionally if the configuration is set to default to
+                // searching dependencies (rust-analyzer.workspace.symbol.search.scope), but we
+                // would need to pipe that configuration information down here.
+                additional_phases.push((crate_id, name.clone(), PrimingPhase::CrateSymbols));
+            }
+
+            work_sender.send((crate_id, name, PrimingPhase::DefMap)).ok();
         }
 
         // recv_timeout is somewhat a hack, we need a way to from this thread check to see if the current salsa revision
@@ -153,6 +164,50 @@ pub fn parallel_prime_caches(
             crates_currently_indexing: crates_currently_indexing.values().cloned().collect(),
             crates_done,
             crates_total,
+            work_type: "Indexing",
+        };
+
+        cb(progress);
+    }
+
+    let mut crates_done = 0;
+    let crates_total = additional_phases.len();
+    for w in additional_phases.into_iter().sorted_by_key(|&(_, _, phase)| phase) {
+        work_sender.send(w).ok();
+    }
+
+    while crates_done < crates_total {
+        db.unwind_if_cancelled();
+
+        // recv_timeout is somewhat a hack, we need a way to from this thread check to see if the current salsa revision
+        // is cancelled on a regular basis. workers will only exit if they are processing a task that is cancelled, or
+        // if this thread exits, and closes the work channel.
+        let worker_progress = match progress_receiver.recv_timeout(Duration::from_millis(10)) {
+            Ok(p) => p,
+            Err(crossbeam_channel::RecvTimeoutError::Timeout) => {
+                continue;
+            }
+            Err(crossbeam_channel::RecvTimeoutError::Disconnected) => {
+                // our workers may have died from a cancelled task, so we'll check and re-raise here.
+                db.unwind_if_cancelled();
+                break;
+            }
+        };
+        match worker_progress {
+            ParallelPrimeCacheWorkerProgress::BeginCrate { crate_id, crate_name } => {
+                crates_currently_indexing.insert(crate_id, crate_name);
+            }
+            ParallelPrimeCacheWorkerProgress::EndCrate { crate_id } => {
+                crates_currently_indexing.swap_remove(&crate_id);
+                crates_done += 1;
+            }
+        };
+
+        let progress = ParallelPrimeCachesProgress {
+            crates_currently_indexing: crates_currently_indexing.values().cloned().collect(),
+            crates_done,
+            crates_total,
+            work_type: "Populating symbols",
         };
 
         cb(progress);

crates/ide/src/view_crate_graph.rs

@@ -84,7 +84,8 @@ impl<'a> dot::Labeller<'a, CrateId, Edge<'a>> for DotCrateGraph {
     }
 
     fn node_label(&'a self, n: &CrateId) -> LabelText<'a> {
-        let name = self.graph[*n].display_name.as_ref().map_or("(unnamed crate)", |name| name);
+        let name =
+            self.graph[*n].display_name.as_ref().map_or("(unnamed crate)", |name| name.as_str());
         LabelText::LabelStr(name.into())
     }
 }

crates/intern/src/symbol/symbols.rs

@@ -481,6 +481,7 @@ define_symbols! {
     u64,
     u8,
     unadjusted,
+    unknown,
     Unknown,
     unpin,
     unreachable_2015,

crates/project-model/src/project_json.rs

@@ -508,5 +508,5 @@ fn serialize_crate_name<S>(name: &CrateName, se: S) -> Result<S::Ok, S::Error>
 where
     S: serde::Serializer,
 {
-    se.serialize_str(name)
+    se.serialize_str(name.as_str())
 }

crates/project-model/src/tests.rs

@@ -230,7 +230,7 @@ fn rust_project_is_proc_macro_has_proc_macro_dep() {
     let crate_data = &crate_graph[crate_id];
     // Assert that the project crate with `is_proc_macro` has a dependency
     // on the proc_macro sysroot crate.
-    crate_data.dependencies.iter().find(|&dep| dep.name.deref() == "proc_macro").unwrap();
+    crate_data.dependencies.iter().find(|&dep| *dep.name.deref() == sym::proc_macro).unwrap();
 }
 
 #[test]

crates/rust-analyzer/src/cli/diagnostics.rs

@@ -4,7 +4,7 @@
 use project_model::{CargoConfig, RustLibSource};
 use rustc_hash::FxHashSet;
 
-use hir::{db::HirDatabase, Crate, HirFileIdExt, Module};
+use hir::{db::HirDatabase, sym, Crate, HirFileIdExt, Module};
 use ide::{AnalysisHost, AssistResolveStrategy, Diagnostic, DiagnosticsConfig, Severity};
 use ide_db::{base_db::SourceRootDatabase, LineIndexDatabase};
 use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
@@ -60,7 +60,7 @@ impl flags::Diagnostics {
             let file_id = module.definition_source_file_id(db).original_file(db);
             if !visited_files.contains(&file_id) {
                 let crate_name =
-                    module.krate().display_name(db).as_deref().unwrap_or("unknown").to_owned();
+                    module.krate().display_name(db).as_deref().unwrap_or(&sym::unknown).to_owned();
                 println!(
                     "processing crate: {crate_name}, module: {}",
                     _vfs.file_path(file_id.into())

crates/rust-analyzer/src/cli/unresolved_references.rs

@@ -1,5 +1,5 @@
 //! Reports references in code that the IDE layer cannot resolve.
-use hir::{db::HirDatabase, AnyDiagnostic, Crate, HirFileIdExt as _, Module, Semantics};
+use hir::{db::HirDatabase, sym, AnyDiagnostic, Crate, HirFileIdExt as _, Module, Semantics};
 use ide::{AnalysisHost, RootDatabase, TextRange};
 use ide_db::{
     base_db::{SourceDatabase, SourceRootDatabase},
@@ -66,7 +66,7 @@ impl flags::UnresolvedReferences {
             let file_id = module.definition_source_file_id(db).original_file(db);
             if !visited_files.contains(&file_id) {
                 let crate_name =
-                    module.krate().display_name(db).as_deref().unwrap_or("unknown").to_owned();
+                    module.krate().display_name(db).as_deref().unwrap_or(&sym::unknown).to_owned();
                 let file_path = vfs.file_path(file_id.into());
                 eprintln!("processing crate: {crate_name}, module: {file_path}",);
 

crates/rust-analyzer/src/main_loop.rs

@@ -325,26 +325,30 @@ impl GlobalState {
                 }
 
                 for progress in prime_caches_progress {
-                    let (state, message, fraction);
+                    let (state, message, fraction, title);
                     match progress {
                         PrimeCachesProgress::Begin => {
                             state = Progress::Begin;
                             message = None;
                             fraction = 0.0;
+                            title = "Indexing";
                         }
                         PrimeCachesProgress::Report(report) => {
                             state = Progress::Report;
+                            title = report.work_type;
 
-                            message = match &report.crates_currently_indexing[..] {
+                            message = match &*report.crates_currently_indexing {
                                 [crate_name] => Some(format!(
-                                    "{}/{} ({crate_name})",
-                                    report.crates_done, report.crates_total
+                                    "{}/{} ({})",
+                                    report.crates_done,
+                                    report.crates_total,
+                                    crate_name.as_str(),
                                 )),
                                 [crate_name, rest @ ..] => Some(format!(
                                     "{}/{} ({} + {} more)",
                                     report.crates_done,
                                     report.crates_total,
-                                    crate_name,
+                                    crate_name.as_str(),
                                     rest.len()
                                 )),
                                 _ => None,
@@ -356,6 +360,7 @@ impl GlobalState {
                             state = Progress::End;
                             message = None;
                             fraction = 1.0;
+                            title = "Indexing";
 
                             self.prime_caches_queue.op_completed(());
                             if cancelled {
@@ -365,7 +370,13 @@ impl GlobalState {
                         }
                     };
 
-                    self.report_progress("Indexing", state, message, Some(fraction), None);
+                    self.report_progress(
+                        title,
+                        state,
+                        message,
+                        Some(fraction),
+                        Some("rustAnalyzer/cachePriming".to_owned()),
+                    );
                 }
             }
             Event::Vfs(message) => {

crates/test-fixture/src/lib.rs

@@ -258,15 +258,7 @@ impl ChangeFixture {
                 let to_id = crates[&to];
                 let sysroot = crate_graph[to_id].origin.is_lang();
                 crate_graph
-                    .add_dep(
-                        from_id,
-                        Dependency::with_prelude(
-                            CrateName::new(&to).unwrap(),
-                            to_id,
-                            prelude,
-                            sysroot,
-                        ),
-                    )
+                    .add_dep(from_id, Dependency::with_prelude(to.clone(), to_id, prelude, sysroot))
                     .unwrap();
             }
         }