Add support for -Zembed-metadata

src/cargo/core/compiler/build_context/target_info.rs

@@ -570,9 +570,10 @@ impl TargetInfo {
         mode: CompileMode,
         target_kind: &TargetKind,
         target_triple: &str,
+        gctx: &GlobalContext,
     ) -> CargoResult<(Vec<FileType>, Vec<CrateType>)> {
         match mode {
-            CompileMode::Build => self.calc_rustc_outputs(target_kind, target_triple),
+            CompileMode::Build => self.calc_rustc_outputs(target_kind, target_triple, gctx),
             CompileMode::Test | CompileMode::Bench => {
                 match self.file_types(&CrateType::Bin, FileFlavor::Normal, target_triple)? {
                     Some(fts) => Ok((fts, Vec::new())),
@@ -593,6 +594,7 @@ impl TargetInfo {
         &self,
         target_kind: &TargetKind,
         target_triple: &str,
+        gctx: &GlobalContext,
     ) -> CargoResult<(Vec<FileType>, Vec<CrateType>)> {
         let mut unsupported = Vec::new();
         let mut result = Vec::new();
@@ -613,9 +615,18 @@ impl TargetInfo {
                 }
             }
         }
-        if !result.is_empty() && !crate_types.iter().any(|ct| ct.requires_upstream_objects()) {
-            // Only add rmeta if pipelining.
-            result.push(FileType::new_rmeta());
+        if !result.is_empty() {
+            if gctx.cli_unstable().no_embed_metadata
+                && crate_types
+                    .iter()
+                    .any(|ct| ct.benefits_from_no_embed_metadata())
+            {
+                // Add .rmeta when we apply -Zembed-metadata=no to the unit.
+                result.push(FileType::new_rmeta());
+            } else if !crate_types.iter().any(|ct| ct.requires_upstream_objects()) {
+                // Only add rmeta if pipelining
+                result.push(FileType::new_rmeta());
+            }
         }
         Ok((result, unsupported))
     }

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -363,6 +363,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                 CompileMode::Build,
                 &TargetKind::Bin,
                 bcx.target_data.short_name(&kind),
+                bcx.gctx,
             )
             .expect("target must support `bin`");
 
@@ -540,7 +541,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         let info = bcx.target_data.info(unit.kind);
         let triple = bcx.target_data.short_name(&unit.kind);
         let (file_types, unsupported) =
-            info.rustc_outputs(unit.mode, unit.target.kind(), triple)?;
+            info.rustc_outputs(unit.mode, unit.target.kind(), triple, bcx.gctx)?;
         if file_types.is_empty() {
             if !unsupported.is_empty() {
                 let unsupported_strs: Vec<_> = unsupported.iter().map(|ct| ct.as_str()).collect();

src/cargo/core/compiler/crate_type.rs

@@ -76,6 +76,38 @@ impl CrateType {
         // Everything else, however, is some form of "linkable output" or
         // something that requires upstream object files.
     }
+
+    /// Returns whether production of this crate type could benefit from splitting metadata
+    /// into a .rmeta file.
+    ///
+    /// See also [`TargetKind::benefits_from_no_embed_metadata`].
+    ///
+    /// [`TargetKind::benefits_from_no_embed_metadata`]: crate::core::manifest::TargetKind::benefits_from_no_embed_metadata
+    pub fn benefits_from_no_embed_metadata(&self) -> bool {
+        match self {
+            // rlib/libs generate .rmeta files for pipelined compilation.
+            // If we also include metadata inside of them, we waste disk space, since the metadata
+            // will be located both in the lib/rlib and the .rmeta file.
+            CrateType::Lib |
+            CrateType::Rlib |
+            // Dylibs do not have to contain metadata when they are used as a runtime dependency.
+            // If we split the metadata into a separate .rmeta file, the dylib file (that
+            // can be shipped as a runtime dependency) can be smaller.
+            CrateType::Dylib => true,
+            // Proc macros contain metadata that specifies what macro functions are available in
+            // it, but the metadata is typically very small. The metadata of proc macros is also
+            // self-contained (unlike rlibs/dylibs), so let's not unnecessarily split it into
+            // multiple files.
+            CrateType::ProcMacro |
+            // cdylib and staticlib produce artifacts that are used through the C ABI and do not
+            // contain Rust-specific metadata.
+            CrateType::Cdylib |
+            CrateType::Staticlib |
+            // Binaries also do not contain metadata
+            CrateType::Bin |
+            CrateType::Other(_) => false
+        }
+    }
 }
 
 impl fmt::Display for CrateType {

src/cargo/core/compiler/mod.rs

@@ -1131,13 +1131,31 @@ fn build_base_args(
 
     if unit.mode.is_check() {
         cmd.arg("--emit=dep-info,metadata");
-    } else if !unit.requires_upstream_objects() {
-        // Always produce metadata files for rlib outputs. Metadata may be used
-        // in this session for a pipelined compilation, or it may be used in a
-        // future Cargo session as part of a pipelined compile.
-        cmd.arg("--emit=dep-info,metadata,link");
+    } else if build_runner.bcx.gctx.cli_unstable().no_embed_metadata {
+        // Nightly rustc supports the -Zembed-metadata=no flag, which tells it to avoid including
+        // full metadata in rlib/dylib artifacts, to save space on disk. In this case, metadata
+        // will only be stored in .rmeta files.
+        // When we use this flag, we should also pass --emit=metadata to all artifacts that
+        // contain useful metadata (rlib/dylib/proc macros), so that a .rmeta file is actually
+        // generated. If we didn't do this, the full metadata would not get written anywhere.
+        // However, we do not want to pass --emit=metadata to artifacts that never produce useful
+        // metadata, such as binaries, because that would just unnecessarily create empty .rmeta
+        // files on disk.
+        if unit.benefits_from_no_embed_metadata() {
+            cmd.arg("--emit=dep-info,metadata,link");
+            cmd.args(&["-Z", "embed-metadata=no"]);
+        } else {
+            cmd.arg("--emit=dep-info,link");
+        }
     } else {
-        cmd.arg("--emit=dep-info,link");
+        // If we don't use -Zembed-metadata=no, we emit .rmeta files only for rlib outputs.
+        // This metadata may be used in this session for a pipelined compilation, or it may
+        // be used in a future Cargo session as part of a pipelined compile.
+        if !unit.requires_upstream_objects() {
+            cmd.arg("--emit=dep-info,metadata,link");
+        } else {
+            cmd.arg("--emit=dep-info,link");
+        }
     }
 
     let prefer_dynamic = (unit.target.for_host() && !unit.target.is_custom_build())
@@ -1636,6 +1654,8 @@ pub fn extern_args(
     let mut result = Vec::new();
     let deps = build_runner.unit_deps(unit);
 
+    let no_embed_metadata = build_runner.bcx.gctx.cli_unstable().no_embed_metadata;
+
     // Closure to add one dependency to `result`.
     let mut link_to =
         |dep: &UnitDep, extern_crate_name: InternedString, noprelude: bool| -> CargoResult<()> {
@@ -1685,6 +1705,12 @@ pub fn extern_args(
                     if output.flavor == FileFlavor::Linkable {
                         pass(&output.path);
                     }
+                    // If we use -Zembed-metadata=no, we also need to pass the path to the
+                    // corresponding .rmeta file to the linkable artifact, because the
+                    // normal dependency (rlib) doesn't contain the full metadata.
+                    else if no_embed_metadata && output.flavor == FileFlavor::Rmeta {
+                        pass(&output.path);
+                    }
                 }
             }
             Ok(())

src/cargo/core/compiler/unit.rs

@@ -125,6 +125,13 @@ impl UnitInner {
         self.mode.is_any_test() || self.target.kind().requires_upstream_objects()
     }
 
+    /// Returns whether compilation of this unit could benefit from splitting metadata
+    /// into a .rmeta file.
+    pub fn benefits_from_no_embed_metadata(&self) -> bool {
+        matches!(self.mode, CompileMode::Build)
+            && self.target.kind().benefits_from_no_embed_metadata()
+    }
+
     /// Returns whether or not this is a "local" package.
     ///
     /// A "local" package is one that the user can likely edit, or otherwise

src/cargo/core/features.rs

@@ -783,6 +783,7 @@ unstable_cli_options!(
     msrv_policy: bool = ("Enable rust-version aware policy within cargo"),
     mtime_on_use: bool = ("Configure Cargo to update the mtime of used files"),
     next_lockfile_bump: bool,
+    no_embed_metadata: bool = ("Avoid embedding metadata in library artifacts"),
     no_index_update: bool = ("Do not update the registry index even if the cache is outdated"),
     package_workspace: bool = ("Handle intra-workspace dependencies when packaging"),
     panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"),
@@ -1294,6 +1295,7 @@ impl CliUnstable {
             "msrv-policy" => self.msrv_policy = parse_empty(k, v)?,
             // can also be set in .cargo/config or with and ENV
             "mtime-on-use" => self.mtime_on_use = parse_empty(k, v)?,
+            "no-embed-metadata" => self.no_embed_metadata = parse_empty(k, v)?,
             "no-index-update" => self.no_index_update = parse_empty(k, v)?,
             "package-workspace" => self.package_workspace = parse_empty(k, v)?,
             "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?,

src/cargo/core/manifest.rs

@@ -279,6 +279,17 @@ impl TargetKind {
         }
     }
 
+    /// Returns whether production of this artifact could benefit from splitting metadata
+    /// into a .rmeta file.
+    pub fn benefits_from_no_embed_metadata(&self) -> bool {
+        match self {
+            TargetKind::Lib(kinds) | TargetKind::ExampleLib(kinds) => {
+                kinds.iter().any(|k| k.benefits_from_no_embed_metadata())
+            }
+            _ => false,
+        }
+    }
+
     /// Returns the arguments suitable for `--crate-type` to pass to rustc.
     pub fn rustc_crate_types(&self) -> Vec<CrateType> {
         match self {

src/cargo/ops/cargo_clean.rs

@@ -230,7 +230,7 @@ fn clean_specs(
 
                     let (file_types, _unsupported) = target_data
                         .info(*compile_kind)
-                        .rustc_outputs(mode, target.kind(), triple)?;
+                        .rustc_outputs(mode, target.kind(), triple, clean_ctx.gctx)?;
                     let (dir, uplift_dir) = match target.kind() {
                         TargetKind::ExampleBin | TargetKind::ExampleLib(..) => {
                             (layout.build_examples(), Some(layout.examples()))

src/doc/src/reference/unstable.md

@@ -91,6 +91,7 @@ Each new feature described below should explain how to use it.
     * [checksum-freshness](#checksum-freshness) --- When passed, the decision as to whether a crate needs to be rebuilt is made using file checksums instead of the file mtime.
     * [panic-abort-tests](#panic-abort-tests) --- Allows running tests with the "abort" panic strategy.
     * [host-config](#host-config) --- Allows setting `[target]`-like configuration settings for host build targets.
+    * [no-embed-metadata](#no-embed-metadata) --- Passes `-Zembed-metadata=no` to the compiler, which avoid embedding metadata into rlib and dylib artifacts, to save disk space.
     * [target-applies-to-host](#target-applies-to-host) --- Alters whether certain flags will be passed to host build targets.
     * [gc](#gc) --- Global cache garbage collection.
     * [open-namespaces](#open-namespaces) --- Allow multiple packages to participate in the same API namespace
@@ -1914,6 +1915,23 @@ The `-Z rustdoc-depinfo` flag leverages rustdoc's dep-info files to determine
 whether documentations are required to re-generate. This can be combined with
 `-Z checksum-freshness` to detect checksum changes rather than file mtime.
 
+## no-embed-metadata
+* Original Pull Request: [#15378](https://github.com/rust-lang/cargo/pull/15378)
+* Tracking Issue: [#15495](https://github.com/rust-lang/cargo/issues/15495)
+
+The default behavior of Rust is to embed crate metadata into `rlib` and `dylib` artifacts.
+Since Cargo also passes `--emit=metadata` to these intermediate artifacts to enable pipelined
+compilation, this means that a lot of metadata ends up being duplicated on disk, which wastes
+disk space in the target directory.
+
+This feature tells Cargo to pass the `-Zembed-metadata=no` flag to the compiler, which instructs
+it not to embed metadata within rlib and dylib artifacts. In this case, the metadata will only
+be stored in `.rmeta` files.
+
+```console
+cargo +nightly -Zno-embed-metadata build
+```
+
 # Stabilized and removed features
 
 ## Compile progress

tests/testsuite/build.rs

@@ -6749,3 +6749,91 @@ fn renamed_uplifted_artifact_remains_unmodified_after_rebuild() {
     let not_the_same = !same_file::is_same_file(bin, renamed_bin).unwrap();
     assert!(not_the_same, "renamed uplifted artifact must be unmodified");
 }
+
+#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")]
+fn embed_metadata() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+
+                name = "foo"
+                version = "0.5.0"
+                edition = "2015"
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[]))
+        .file("bar/Cargo.toml", &basic_lib_manifest("bar"))
+        .file(
+            "bar/src/bar.rs",
+            r#"
+                pub fn gimme() -> &'static str {
+                    "test passed"
+                }
+            "#,
+        )
+        .build();
+
+    p.cargo("build -Z no-embed-metadata")
+        .masquerade_as_nightly_cargo(&["-Z no-embed-metadata"])
+        .arg("-v")
+        .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`")
+        .with_stderr_contains(
+            "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar-[HASH].rmeta[..]`",
+        )
+        .run();
+}
+
+// Make sure that cargo passes --extern=<dep>.rmeta even if <dep>
+// is compiled as a dylib.
+#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")]
+fn embed_metadata_dylib_dep() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.5.0"
+                edition = "2015"
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[]))
+        .file(
+            "bar/Cargo.toml",
+            r#"
+                [package]
+                name = "bar"
+                version = "0.5.0"
+                edition = "2015"
+
+                [lib]
+                crate-type = ["dylib"]
+            "#,
+        )
+        .file(
+            "bar/src/lib.rs",
+            r#"
+                pub fn gimme() -> &'static str {
+                    "test passed"
+                }
+            "#,
+        )
+        .build();
+
+    p.cargo("build -Z no-embed-metadata")
+        .masquerade_as_nightly_cargo(&["-Z no-embed-metadata"])
+        .arg("-v")
+        .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`")
+        .with_stderr_contains(
+            "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar.rmeta[..]`",
+        )
+        .run();
+}