feature: move linked_projects discovery to the rust-analyzer server

Author: davidbarsky

Cargo.lock

@@ -24,6 +24,7 @@ command-group.workspace = true
 paths.workspace = true
 stdx.workspace = true
 toolchain.workspace = true
+project-model.workspace = true
 
 [lints]
 workspace = true

crates/flycheck/Cargo.toml

@@ -0,0 +1,76 @@
+//! A `cargo-metadata`-equivalent for non-Cargo build systems.
+use std::{io, process::Command};
+
+use crossbeam_channel::Sender;
+use paths::AbsPathBuf;
+use project_model::ProjectJsonData;
+use serde::{Deserialize, Serialize};
+
+use crate::command::{CommandHandle, ParseFromLine};
+
+/// A command wrapper for getting a `rust-project.json`.
+///
+/// This is analogous to `cargo-metadata`, but for non-Cargo build systems.
+pub struct JsonWorkspace {
+    command: Vec<String>,
+    sender: Sender<DiscoverProjectMessage>,
+}
+
+impl JsonWorkspace {
+    /// Create a new [JsonWorkspace].
+    pub fn new(sender: Sender<DiscoverProjectMessage>, command: Vec<String>) -> Self {
+        Self { sender, command }
+    }
+
+    /// Spawn the command inside [JsonWorkspace] and report progress, if any.
+    pub fn spawn(&self, files: Vec<AbsPathBuf>) -> io::Result<JsonWorkspaceHandle> {
+        let command = &self.command[0];
+        let args = &self.command[1..];
+        let first_file: &AbsPathBuf = &files[0];
+
+        let mut cmd = Command::new(command);
+        cmd.args(args);
+        cmd.arg(first_file);
+
+        Ok(JsonWorkspaceHandle { _handle: CommandHandle::spawn(cmd, self.sender.clone())? })
+    }
+}
+
+/// A handle to a spawned [JsonWorkspace].
+#[derive(Debug)]
+pub struct JsonWorkspaceHandle {
+    _handle: CommandHandle<DiscoverProjectMessage>,
+}
+
+/// An enum containing either progress messages or the materialized rust-project.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+#[serde(tag = "type")]
+pub enum DiscoverProjectMessage {
+    Progress { message: String },
+    Finished { project_json: Vec<ProjectJsonData> },
+}
+
+impl ParseFromLine for DiscoverProjectMessage {
+    fn from_line(line: &str, _error: &mut String) -> Option<Self> {
+        let value = serde_json::from_str::<serde_json::Value>(&line).unwrap();
+
+        // let mut deserializer = serde_json::Deserializer::from_str(line);
+        // deserializer.disable_recursion_limit();
+
+        if let Ok(project) = serde_json::from_value::<ProjectJsonData>(value.clone()) {
+            return Some(DiscoverProjectMessage::Finished { project_json: vec![project] });
+        }
+
+        if let Some(message) = value.pointer("/fields/message") {
+            return Some(DiscoverProjectMessage::Progress {
+                message: message.as_str().unwrap().to_owned(),
+            });
+        }
+
+        None
+    }
+
+    fn from_eof() -> Option<Self> {
+        None
+    }
+}

crates/flycheck/src/json_workspace.rs

@@ -22,9 +22,11 @@ pub use cargo_metadata::diagnostic::{
 use toolchain::Tool;
 
 mod command;
+mod json_workspace;
 mod test_runner;
 
 use command::{CommandHandle, ParseFromLine};
+pub use json_workspace::{DiscoverProjectMessage, JsonWorkspace, JsonWorkspaceHandle};
 pub use test_runner::{CargoTestHandle, CargoTestMessage, TestState};
 
 #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)]
@@ -227,7 +229,7 @@ enum Event {
     CheckEvent(Option<CargoCheckMessage>),
 }
 
-const SAVED_FILE_PLACEHOLDER: &str = "$saved_file";
+pub const SAVED_FILE_PLACEHOLDER: &str = "$saved_file";
 
 impl FlycheckActor {
     fn new(

crates/flycheck/src/lib.rs

@@ -17,7 +17,7 @@ use itertools::Itertools;
 use proc_macro_api::{MacroDylib, ProcMacroServer};
 use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace};
 use span::Span;
-use tracing::{instrument, Level};
+use tracing::Level;
 use vfs::{file_set::FileSetConfig, loader::Handle, AbsPath, AbsPathBuf, VfsPath};
 
 pub struct LoadCargoConfig {
@@ -51,7 +51,6 @@ pub fn load_workspace_at(
     load_workspace(workspace, &cargo_config.extra_env, load_config)
 }
 
-#[instrument(skip_all)]
 pub fn load_workspace(
     ws: ProjectWorkspace,
     extra_env: &FxHashMap<String, String>,

crates/load-cargo/src/lib.rs

@@ -14,6 +14,7 @@ doctest = false
 [dependencies]
 anyhow.workspace = true
 cargo_metadata.workspace = true
+crossbeam-channel.workspace = true
 rustc-hash.workspace = true
 semver.workspace = true
 serde_json.workspace = true

crates/project-model/Cargo.toml

@@ -175,7 +175,7 @@ impl ProjectJson {
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ProjectJsonData {
-    sysroot: Option<Utf8PathBuf>,
+    pub sysroot: Option<Utf8PathBuf>,
     sysroot_src: Option<Utf8PathBuf>,
     crates: Vec<CrateData>,
 }

crates/project-model/src/project_json.rs

@@ -30,6 +30,7 @@ use crate::{
     utf8_stdout, CargoConfig, CargoWorkspace, InvocationStrategy, ManifestPath, Package,
     ProjectJson, ProjectManifest, Sysroot, TargetData, TargetKind, WorkspaceBuildScripts,
 };
+use tracing::{debug, error, info, Level};
 
 pub type FileLoader<'a> = &'a mut dyn for<'b> FnMut(&'b AbsPath) -> Option<FileId>;
 
@@ -249,7 +250,7 @@ impl ProjectWorkspace {
                 };
 
                 let rustc =  rustc_dir.and_then(|rustc_dir| {
-                    tracing::info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source");
+                    info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source");
                     match CargoWorkspace::fetch_metadata(
                         &rustc_dir,
                         cargo_toml.parent(),
@@ -705,7 +706,7 @@ impl ProjectWorkspace {
         load: FileLoader<'_>,
         extra_env: &FxHashMap<String, String>,
     ) -> (CrateGraph, ProcMacroPaths) {
-        let _p = tracing::span!(tracing::Level::INFO, "ProjectWorkspace::to_crate_graph").entered();
+        let _p = tracing::span!(tracing::Level::INFO, "to_crate_graph").entered();
 
         let Self { kind, sysroot, cfg_overrides, rustc_cfg, .. } = self;
         let ((mut crate_graph, proc_macros), sysroot) = match kind {
@@ -762,9 +763,9 @@ impl ProjectWorkspace {
         };
 
         if matches!(sysroot.mode(), SysrootMode::Stitched(_)) && crate_graph.patch_cfg_if() {
-            tracing::debug!("Patched std to depend on cfg-if")
+            debug!("Patched std to depend on cfg-if")
         } else {
-            tracing::debug!("Did not patch std to depend on cfg-if")
+            debug!("Did not patch std to depend on cfg-if")
         }
         (crate_graph, proc_macros)
     }
@@ -909,6 +910,11 @@ fn project_json_to_crate_graph(
                         CrateOrigin::Local { repo: None, name: None }
                     },
                 );
+                debug!(
+                    ?crate_graph_crate_id,
+                    crate = display_name.as_ref().map(|name| name.canonical_name()),
+                    "added root to crate graph"
+                );
                 if *is_proc_macro {
                     if let Some(path) = proc_macro_dylib_path.clone() {
                         let node = Ok((
@@ -923,6 +929,7 @@ fn project_json_to_crate_graph(
         )
         .collect();
 
+    debug!(map = ?idx_to_crate_id);
     for (from_idx, krate) in project.crates() {
         if let Some(&from) = idx_to_crate_id.get(&from_idx) {
             public_deps.add_to_crate_graph(crate_graph, from);
@@ -949,7 +956,7 @@ fn cargo_to_crate_graph(
     override_cfg: &CfgOverrides,
     build_scripts: &WorkspaceBuildScripts,
 ) -> (CrateGraph, ProcMacroPaths) {
-    let _p = tracing::span!(tracing::Level::INFO, "cargo_to_crate_graph").entered();
+    let _p = tracing::span!(Level::INFO, "cargo_to_crate_graph").entered();
     let mut res = (CrateGraph::default(), ProcMacroPaths::default());
     let crate_graph = &mut res.0;
     let proc_macros = &mut res.1;
@@ -1134,7 +1141,7 @@ fn detached_file_to_crate_graph(
     sysroot: &Sysroot,
     override_cfg: &CfgOverrides,
 ) -> (CrateGraph, ProcMacroPaths) {
-    let _p = tracing::span!(tracing::Level::INFO, "detached_file_to_crate_graph").entered();
+    let _p = tracing::span!(Level::INFO, "detached_file_to_crate_graph").entered();
     let mut crate_graph = CrateGraph::default();
     let (public_deps, _libproc_macro) =
         sysroot_to_crate_graph(&mut crate_graph, sysroot, rustc_cfg.clone(), load);
@@ -1148,7 +1155,7 @@ fn detached_file_to_crate_graph(
     let file_id = match load(detached_file) {
         Some(file_id) => file_id,
         None => {
-            tracing::error!("Failed to load detached file {:?}", detached_file);
+            error!("Failed to load detached file {:?}", detached_file);
             return (crate_graph, FxHashMap::default());
         }
     };
@@ -1345,7 +1352,7 @@ fn add_target_crate_root(
     crate_id
 }
 
-#[derive(Default)]
+#[derive(Default, Debug)]
 struct SysrootPublicDeps {
     deps: Vec<(CrateName, CrateId, bool)>,
 }

crates/project-model/src/workspace.rs

@@ -172,6 +172,7 @@ fn run_server() -> anyhow::Result<()> {
             return Err(e.into());
         }
     };
+
     tracing::info!("InitializeParams: {}", initialize_params);
     let lsp_types::InitializeParams {
         root_uri,
@@ -221,6 +222,7 @@ fn run_server() -> anyhow::Result<()> {
         .unwrap_or_else(|| vec![root_path.clone()]);
     let mut config =
         Config::new(root_path, capabilities, workspace_roots, visual_studio_code_version);
+
     if let Some(json) = initialization_options {
         if let Err(e) = config.update(json) {
             use lsp_types::{
@@ -255,8 +257,10 @@ fn run_server() -> anyhow::Result<()> {
         return Err(e.into());
     }
 
-    if !config.has_linked_projects() && config.detached_files().is_empty() {
-        config.rediscover_workspaces();
+    if config.discover_command().is_none() {
+        if !config.has_linked_projects() && config.detached_files().is_empty() {
+            config.rediscover_workspaces();
+        }
     }
 
     // If the io_threads have an error, there's usually an error on the main

crates/rust-analyzer/src/bin/main.rs

@@ -272,6 +272,11 @@ config_data! {
         /// The warnings will be indicated by a blue squiggly underline in code
         /// and a blue icon in the `Problems Panel`.
         diagnostics_warningsAsInfo: Vec<String> = vec![],
+        /// Enables automatic discovery of projects using the discoverCommand.
+        ///
+        /// Setting this command will result in rust-analyzer starting indexing
+        /// only once a Rust file has been opened.
+        discoverCommand: Option<Vec<String>> = None,
         /// These directories will be ignored by rust-analyzer. They are
         /// relative to the workspace root, and globs are not supported. You may
         /// also need to add the folders to Code's `files.watcherExclude`.
@@ -654,7 +659,7 @@ config_data! {
 
 #[derive(Debug, Clone)]
 pub struct Config {
-    discovered_projects: Vec<ProjectManifest>,
+    pub(crate) discovered_projects: Vec<ProjectManifest>,
     /// The workspace roots as registered by the LSP client
     workspace_roots: Vec<AbsPathBuf>,
     caps: lsp_types::ClientCapabilities,
@@ -929,6 +934,18 @@ impl Config {
         self.workspace_roots.extend(paths);
     }
 
+    pub fn add_linked_projects(&mut self, projects: Vec<ProjectJsonData>) {
+        let linked_projects = &mut self.client_config.global.linkedProjects;
+
+        let mut new_projects: Vec<_> =
+            projects.into_iter().map(ManifestOrProjectJson::ProjectJson).collect();
+
+        match linked_projects {
+            Some(projects) => projects.append(&mut new_projects),
+            None => *linked_projects = Some(new_projects),
+        }
+    }
+
     pub fn update(&mut self, mut json: serde_json::Value) -> Result<(), ConfigError> {
         tracing::info!("updating config from JSON: {:#}", json);
         if json.is_null() || json.as_object().map_or(false, |it| it.is_empty()) {
@@ -1297,6 +1314,7 @@ impl Config {
     pub fn has_linked_projects(&self) -> bool {
         !self.linkedProjects().is_empty()
     }
+
     pub fn linked_manifests(&self) -> impl Iterator<Item = &Utf8Path> + '_ {
         self.linkedProjects().iter().filter_map(|it| match it {
             ManifestOrProjectJson::Manifest(p) => Some(&**p),
@@ -1306,6 +1324,11 @@ impl Config {
     pub fn has_linked_project_jsons(&self) -> bool {
         self.linkedProjects().iter().any(|it| matches!(it, ManifestOrProjectJson::ProjectJson(_)))
     }
+
+    pub fn discover_command(&self) -> Option<Vec<String>> {
+        self.discoverCommand().clone()
+    }
+
     pub fn linked_or_discovered_projects(&self) -> Vec<LinkedProject> {
         match self.linkedProjects().as_slice() {
             [] => {
@@ -1571,7 +1594,7 @@ impl Config {
     }
 
     pub fn cargo_autoreload_config(&self) -> bool {
-        self.cargo_autoreload().to_owned()
+        self.cargo_autoreload().to_owned() && self.discoverCommand().is_none()
     }
 
     pub fn run_build_scripts(&self) -> bool {
@@ -3026,7 +3049,7 @@ fn doc_comment_to_string(doc: &[&str]) -> String {
 
 #[cfg(test)]
 mod tests {
-    use std::fs;
+    use std::{env::temp_dir, fs};
 
     use test_utils::{ensure_file_contents, project_root};
 
@@ -3186,6 +3209,34 @@ mod tests {
         );
     }
 
+    #[test]
+    fn linked_targets_updates() {
+        let mut config = Config::new(
+            AbsPathBuf::try_from(project_root()).unwrap(),
+            Default::default(),
+            vec![],
+            None,
+        );
+        assert!(config.linkedProjects().is_empty());
+        let project_json = serde_json::json!({
+            "sysroot_src": null,
+            "crates": [
+                {
+                    "display_name": "hello_world",
+                    "root_module": "$ROOT$src/lib.rs",
+                    "edition": "2018",
+                    "deps": [],
+                    "is_workspace_member": true
+                }
+            ]
+        });
+
+        let project_json = serde_json::from_value(project_json).expect("unable to deserialize");
+        config.add_linked_projects(vec![project_json]);
+
+        assert!(!config.linkedProjects().is_empty());
+    }
+
     #[test]
     fn cargo_target_dir_relative_dir() {
         let mut config = Config::new(