Auto merge of rust-lang#136779 - jieyouxu:rahhh, r=<try>

[DO NOT MERGE] `./x test rust-analyzer`

I somehow made `./x test rust-analyzer` work on my machine[^machine], **but at what cost?**

Not intended for merge but only as a reference. If we do want to land this, I'll need to tidy this up.

Notes:

- I abused a bunch of cargo features  `in-rust-tree`. It probably doesn't need to be, and simply `--cfg` might work. I was trying to get the main rust-analyzer tests to build *at all*. Anything building is already a miracle.
- I had to slap a bunch of the following to all the r-a crates to get the tests to build at all. I don't 100% understand why, but otherwise I get a whole ton of ``expected `rustc_lexer` to be available in rlib format`` build failures.

```rs
#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
#[cfg(all(feature = "in-rust-tree", test))]
extern crate rustc_driver as _;
```

- Skipped one config test that was fixed on r-a master but not synced here in r-l/r yet.

[^machine]: `x86_64-unknown-linux-gnu`, haven't bothered trying this on msvc yet.

try-job: aarch64-gnu
try-job: x86_64-apple-1
try-job: aarch64-apple
try-job: i686-mingw-1
try-job: x86_64-mingw-1
try-job: i686-msvc-1
try-job: x86_64-msvc-1

Author: bors

src/bootstrap/src/core/build_steps/compile.rs

@@ -9,6 +9,8 @@ use std::path::{Path, PathBuf};
 use std::{env, fs, iter};
 
 use clap_complete::shells;
+#[cfg(feature = "tracing")]
+use tracing::instrument;
 
 use crate::core::build_steps::compile::run_cargo;
 use crate::core::build_steps::doc::DocumentationFormat;
@@ -28,7 +30,7 @@ use crate::utils::helpers::{
     linker_flags, t, target_supports_cranelift_backend, up_to_date,
 };
 use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests};
-use crate::{CLang, DocTests, GitRepo, Mode, PathSet, envify};
+use crate::{CLang, DocTests, GitRepo, Mode, PathSet, debug, envify};
 
 const ADB_TEST_DIR: &str = "/data/local/tmp/work";
 
@@ -354,50 +356,136 @@ impl Step for RustAnalyzer {
     const DEFAULT: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/rust-analyzer")
+        run.path("src/tools/rust-analyzer").alias("rust-analyzer")
     }
 
     fn make_run(run: RunConfig<'_>) {
         run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target });
     }
 
-    /// Runs `cargo test` for rust-analyzer
+    #[cfg_attr(
+        feature = "tracing",
+        instrument(
+            level = "debug",
+            name = "RustAnalyzer::run",
+            skip_all,
+            fields(stage = self.stage, host = ?self.host),
+        ),
+    )]
+    fn run(self, builder: &Builder<'_>) {
+        debug!(stage = self.stage, host = ?self.host, "ensuring compiler");
+        let compiler = builder.compiler(self.stage, self.host);
+
+        debug!(stage = self.stage, host = ?self.host, "ensuring std");
+        builder.ensure(compile::Rustc::new(compiler, self.host));
+
+        let mut cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolRustc,
+            self.host,
+            Kind::Test,
+            "src/tools/rust-analyzer",
+            SourceType::InTree,
+            &["in-rust-tree".to_owned()],
+        );
+        cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES);
+
+        // RA's test suite tries to write to the source directory, that can't work in Rust CI.
+        cargo.env("SKIP_SLOW_TESTS", "1");
+
+        // NOTE: unlike `proc-macro-srv` step, we must **not** set `CARGO_WORKSPACE_DIR` because I
+        // believe `src/tools/rust-analyzer/.cargo/config.toml`'s relative `CARGO_WORKSPACE_DIR`
+        // takes effect,
+
+        cargo.add_rustc_lib_path(builder);
+        run_cargo_test(
+            cargo,
+            &[
+                // FIXME: may need a fix from https://github.com/rust-lang/rust-analyzer/pull/19124.
+                "--skip=config::tests::cargo_target_dir_subdir",
+            ],
+            &[],
+            "rust-analyzer",
+            "rust-analyzer",
+            self.host,
+            builder,
+        );
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct RustAnalyzerProcMacroSrv {
+    stage: u32,
+    host: TargetSelection,
+}
+
+impl Step for RustAnalyzerProcMacroSrv {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-analyzer/crates/proc-macro-srv").alias("rust-analyzer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target });
+    }
+
+    #[cfg_attr(
+        feature = "tracing",
+        instrument(
+            level = "debug",
+            name = "RustAnalyzerProcMacroSrv::run",
+            skip_all,
+            fields(stage = self.stage, host = ?self.host),
+        ),
+    )]
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let host = self.host;
+
+        debug!(stage, ?host, "ensuring compiler");
         let compiler = builder.compiler(stage, host);
 
         // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite,
         // but we do need the standard library to be present.
+        debug!(stage, ?compiler, "ensuring std");
         builder.ensure(compile::Rustc::new(compiler, host));
 
-        let workspace_path = "src/tools/rust-analyzer";
-        // until the whole RA test suite runs on `i686`, we only run
-        // `proc-macro-srv` tests
-        let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv";
+        debug!("running `cargo test` on `src/tools/rust-analyzer/crates/proc-macro-srv`");
+
         let mut cargo = tool::prepare_tool_cargo(
             builder,
             compiler,
             Mode::ToolRustc,
             host,
             Kind::Test,
-            crate_path,
+            "src/tools/rust-analyzer/crates/proc-macro-srv",
             SourceType::InTree,
             &["in-rust-tree".to_owned()],
         );
         cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES);
 
-        let dir = builder.src.join(workspace_path);
-        // needed by rust-analyzer to find its own text fixtures, cf.
-        // https://github.com/rust-analyzer/expect-test/issues/33
+        let dir = builder.src.join("src/tools/rust-analyzer");
+        // Needed by rust-analyzer to find its own text fixtures, cf.
+        // https://github.com/rust-analyzer/expect-test/issues/33.
         cargo.env("CARGO_WORKSPACE_DIR", &dir);
 
-        // RA's test suite tries to write to the source directory, that can't
-        // work in Rust CI
+        // RA's test suite tries to write to the source directory, that can't work in Rust CI.
         cargo.env("SKIP_SLOW_TESTS", "1");
 
         cargo.add_rustc_lib_path(builder);
-        run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", host, builder);
+        run_cargo_test(
+            cargo,
+            &[],
+            &[],
+            "rust-analyzer/crates/proc-macro-srv",
+            "rust-analyzer proc-macro-srv",
+            host,
+            builder,
+        );
     }
 }
 
@@ -2618,14 +2706,14 @@ fn prepare_cargo_test(
         cargo.arg("--quiet");
     }
 
-    // The tests are going to run with the *target* libraries, so we need to
-    // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent.
+    // The tests are going to run with the *target* libraries, so we need to ensure that those
+    // libraries show up in the LD_LIBRARY_PATH equivalent.
     //
-    // Note that to run the compiler we need to run with the *host* libraries,
-    // but our wrapper scripts arrange for that to be the case anyway.
+    // Note that to run the compiler we need to run with the *host* libraries, but our wrapper
+    // scripts arrange for that to be the case anyway.
     //
-    // We skip everything on Miri as then this overwrites the libdir set up
-    // by `Cargo::new` and that actually makes things go wrong.
+    // We skip everything on Miri as then this overwrites the libdir set up by `Cargo::new` and that
+    // actually makes things go wrong.
     if builder.kind != Kind::Miri {
         let mut dylib_path = dylib_path();
         dylib_path.insert(0, PathBuf::from(&*builder.sysroot_target_libdir(compiler, target)));

src/bootstrap/src/core/build_steps/test.rs

@@ -21,7 +21,7 @@ use crate::core::config::{DryRun, TargetSelection};
 use crate::utils::cache::Cache;
 use crate::utils::exec::{BootstrapCommand, command};
 use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t};
-use crate::{Build, Crate};
+use crate::{Build, Crate, trace};
 
 mod cargo;
 
@@ -971,6 +971,7 @@ impl<'a> Builder<'a> {
                 test::Cargotest,
                 test::Cargo,
                 test::RustAnalyzer,
+                test::RustAnalyzerProcMacroSrv,
                 test::ErrorIndex,
                 test::Distcheck,
                 test::Nomicon,
@@ -1215,7 +1216,7 @@ impl<'a> Builder<'a> {
     /// `Compiler` since all `Compiler` instances are meant to be obtained through this function,
     /// since it ensures that they are valid (i.e., built and assembled).
     pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler {
-        self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } })
+        self.ensure(compile::Assemble { output_compiler: Compiler { stage, host } })
     }
 
     /// Similar to `compiler`, except handles the full-bootstrap option to
@@ -1331,6 +1332,8 @@ impl<'a> Builder<'a> {
             return;
         }
 
+        trace!(rustc_lib_paths = ?self.rustc_lib_paths(compiler));
+
         add_dylib_path(self.rustc_lib_paths(compiler), cmd);
     }
 

src/bootstrap/src/core/builder/mod.rs

@@ -350,9 +350,9 @@ mod defaults {
         assert_eq!(
             first(cache.all::<compile::Assemble>()),
             &[
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
-                compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 0 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: b, stage: 1 } },
             ]
         );
         assert_eq!(
@@ -630,10 +630,10 @@ mod dist {
         assert_eq!(
             first(cache.all::<compile::Assemble>()),
             &[
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
-                compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 0 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 2 } },
+                compile::Assemble { output_compiler: Compiler { host: b, stage: 2 } },
             ]
         );
     }
@@ -711,9 +711,9 @@ mod dist {
         assert_eq!(
             first(builder.cache.all::<compile::Assemble>()),
             &[
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } },
-                compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 0 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 1 } },
+                compile::Assemble { output_compiler: Compiler { host: a, stage: 2 } },
             ]
         );
         assert_eq!(

src/bootstrap/src/core/builder/tests.rs

@@ -30,5 +30,8 @@ vfs.workspace = true
 span.workspace = true
 intern.workspace = true
 
+[features]
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/base-db/Cargo.toml

@@ -1,4 +1,9 @@
 //! base_db defines basic database traits. The concrete DB is defined by ide.
+
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 // FIXME: Rename this crate, base db is non descriptive
 mod change;
 mod input;

src/tools/rust-analyzer/crates/base-db/src/lib.rs

@@ -33,5 +33,9 @@ derive_arbitrary = "1.3.2"
 syntax-bridge.workspace = true
 syntax.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/cfg/Cargo.toml

@@ -1,5 +1,9 @@
 //! cfg defines conditional compiling options, `cfg` attribute parser and evaluator
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod cfg_expr;
 mod dnf;
 #[cfg(test)]

src/tools/rust-analyzer/crates/cfg/src/lib.rs

@@ -54,6 +54,7 @@ expect-test.workspace = true
 test-utils.workspace = true
 test-fixture.workspace = true
 syntax-bridge.workspace = true
+
 [features]
 in-rust-tree = ["hir-expand/in-rust-tree"]
 

src/tools/rust-analyzer/crates/hir-def/Cargo.toml

@@ -8,6 +8,8 @@
 //! actually true.
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 #[cfg(feature = "in-rust-tree")]
 extern crate rustc_parse_format;

src/tools/rust-analyzer/crates/hir-def/src/lib.rs

@@ -3,8 +3,13 @@
 //! Specifically, it implements a concept of `MacroFile` -- a file whose syntax
 //! tree originates not from the text of some `FileId`, but from some macro
 //! expansion.
+
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 
+#[cfg_attr(all(feature = "in-rust-tree", test), allow(unused_extern_crates))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver;
+
 pub mod attrs;
 pub mod builtin;
 pub mod change;

src/tools/rust-analyzer/crates/hir-expand/src/lib.rs

@@ -2,6 +2,8 @@
 //! information and various assists.
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 #[cfg(feature = "in-rust-tree")]
 extern crate rustc_index;

src/tools/rust-analyzer/crates/hir-ty/src/lib.rs

@@ -17,9 +17,12 @@
 //! from the ide with completions, hovers, etc. It is a (soft, internal) boundary:
 //! <https://www.tedinski.com/2018/02/06/system-boundaries.html>.
 
-#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 #![recursion_limit = "512"]
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod attrs;
 mod from_id;
 mod has_source;

src/tools/rust-analyzer/crates/hir/src/lib.rs

@@ -59,6 +59,8 @@
 //! <https://rust-analyzer.github.io/blog/2020/09/28/how-to-make-a-light-bulb.html>
 
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
 
 mod assist_config;
 mod assist_context;

src/tools/rust-analyzer/crates/ide-assists/src/lib.rs

@@ -36,5 +36,9 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true

src/tools/rust-analyzer/crates/ide-completion/Cargo.toml

@@ -1,5 +1,9 @@
 //! `completions` crate provides utilities for generating completions of user input.
 
+#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(all(feature = "in-rust-tree", test))]
+extern crate rustc_driver as _;
+
 mod completions;
 mod config;
 mod context;

src/tools/rust-analyzer/crates/ide-completion/src/lib.rs

@@ -49,5 +49,9 @@ expect-test = "1.4.0"
 test-utils.workspace = true
 test-fixture.workspace = true
 
+[features]
+default = []
+in-rust-tree = []
+
 [lints]
 workspace = true