Add -Zno-embed-metadata unstable flag

Author: Kobzol

src/cargo/core/compiler/build_context/target_info.rs

@@ -613,9 +613,19 @@ impl TargetInfo {
                 }
             }
         }
-        if !result.is_empty() && !crate_types.iter().any(|ct| ct.requires_upstream_objects()) {
-            // Only add rmeta if pipelining.
-            result.push(FileType::new_rmeta());
+        if !result.is_empty() {
+            if !crate_types.iter().any(|ct| ct.requires_upstream_objects()) {
+                // Only add rmeta if pipelining...
+                result.push(FileType::new_rmeta());
+            } else if crate_types
+                .iter()
+                .any(|ct| ct.benefits_from_no_embed_metadata())
+            {
+                // ...or when we apply -Zembed-metadata=no to the unit.
+                // TODO: should we thread through the information if we use
+                // embed-metadata?
+                result.push(FileType::new_rmeta());
+            }
         }
         Ok((result, unsupported))
     }

src/cargo/core/compiler/crate_type.rs

@@ -76,6 +76,38 @@ impl CrateType {
         // Everything else, however, is some form of "linkable output" or
         // something that requires upstream object files.
     }
+
+    /// Returns whether production of this crate type could benefit from splitting metadata
+    /// into a .rmeta file.
+    ///
+    /// See also [`TargetKind::benefits_from_no_embed_metadata`].
+    ///
+    /// [`TargetKind::benefits_from_no_embed_metadata`]: crate::core::manifest::TargetKind::benefits_from_no_embed_metadata
+    pub fn benefits_from_no_embed_metadata(&self) -> bool {
+        match self {
+            // rlib/libs generate .rmeta files for pipelined compilation.
+            // If we also include metadata inside of them, we waste disk space, since the metadata
+            // will be located both in the lib/rlib and the .rmeta file.
+            CrateType::Lib |
+            CrateType::Rlib |
+            // Dylibs do not have to contain metadata when they are used as a runtime dependency.
+            // If we split the metadata into a separate .rmeta file, the dylib file (that
+            // can be shipped as a runtime dependency) can be smaller.
+            CrateType::Dylib => true,
+            // Proc macros contain metadata that specifies what macro functions are available in
+            // it, but the metadata is typically very small. The metadata of proc macros is also
+            // self-contained (unlike rlibs/dylibs), so let's not unnecessarily split it into
+            // multiple files.
+            CrateType::ProcMacro |
+            // cdylib and staticlib produce artifacts that are used through the C ABI and do not
+            // contain Rust-specific metadata.
+            CrateType::Cdylib |
+            CrateType::Staticlib |
+            // Binaries also do not contain metadata
+            CrateType::Bin |
+            CrateType::Other(_) => false
+        }
+    }
 }
 
 impl fmt::Display for CrateType {

src/cargo/core/compiler/mod.rs

@@ -1131,13 +1131,31 @@ fn build_base_args(
 
     if unit.mode.is_check() {
         cmd.arg("--emit=dep-info,metadata");
-    } else if !unit.requires_upstream_objects() {
-        // Always produce metadata files for rlib outputs. Metadata may be used
-        // in this session for a pipelined compilation, or it may be used in a
-        // future Cargo session as part of a pipelined compile.
-        cmd.arg("--emit=dep-info,metadata,link");
+    } else if build_runner.bcx.gctx.cli_unstable().no_embed_metadata {
+        // Nightly rustc supports the -Zembed-metadata=no flag, which tells it to avoid including
+        // full metadata in rlib/dylib artifacts, to save space on disk. In this case, metadata
+        // will only be stored in .rmeta files.
+        // When we use this flag, we should also pass --emit=metadata to all artifacts that
+        // contain useful metadata (rlib/dylib/proc macros), so that a .rmeta file is actually
+        // generated. If we didn't do this, the full metadata would not get written anywhere.
+        // However, we do not want to pass --emit=metadata to artifacts that never produce useful
+        // metadata, such as binaries, because that would just unnecessarily create empty .rmeta
+        // files on disk.
+        if unit.benefits_from_no_embed_metadata() {
+            cmd.arg("--emit=dep-info,metadata,link");
+            cmd.args(&["-Z", "embed-metadata=no"]);
+        } else {
+            cmd.arg("--emit=dep-info,link");
+        }
     } else {
-        cmd.arg("--emit=dep-info,link");
+        // If we don't use -Zembed-metadata=no, we emit .rmeta files only for rlib outputs.
+        // This metadata may be used in this session for a pipelined compilation, or it may
+        // be used in a future Cargo session as part of a pipelined compile.
+        if !unit.requires_upstream_objects() {
+            cmd.arg("--emit=dep-info,metadata,link");
+        } else {
+            cmd.arg("--emit=dep-info,link");
+        }
     }
 
     let prefer_dynamic = (unit.target.for_host() && !unit.target.is_custom_build())
@@ -1636,6 +1654,8 @@ pub fn extern_args(
     let mut result = Vec::new();
     let deps = build_runner.unit_deps(unit);
 
+    let no_embed_metadata = build_runner.bcx.gctx.cli_unstable().no_embed_metadata;
+
     // Closure to add one dependency to `result`.
     let mut link_to =
         |dep: &UnitDep, extern_crate_name: InternedString, noprelude: bool| -> CargoResult<()> {
@@ -1685,6 +1705,12 @@ pub fn extern_args(
                     if output.flavor == FileFlavor::Linkable {
                         pass(&output.path);
                     }
+                    // If we use -Zembed-metadata=no, we also need to pass the path to the
+                    // corresponding .rmeta file to the linkable artifact, because the
+                    // normal dependency (rlib) doesn't contain the full metadata.
+                    else if no_embed_metadata && output.flavor == FileFlavor::Rmeta {
+                        pass(&output.path);
+                    }
                 }
             }
             Ok(())

src/cargo/core/compiler/unit.rs

@@ -125,6 +125,13 @@ impl UnitInner {
         self.mode.is_any_test() || self.target.kind().requires_upstream_objects()
     }
 
+    /// Returns whether compilation of this unit could benefit from splitting metadata
+    /// into a .rmeta file.
+    pub fn benefits_from_no_embed_metadata(&self) -> bool {
+        matches!(self.mode, CompileMode::Build)
+            && self.target.kind().benefits_from_no_embed_metadata()
+    }
+
     /// Returns whether or not this is a "local" package.
     ///
     /// A "local" package is one that the user can likely edit, or otherwise

src/cargo/core/features.rs

@@ -783,6 +783,7 @@ unstable_cli_options!(
     msrv_policy: bool = ("Enable rust-version aware policy within cargo"),
     mtime_on_use: bool = ("Configure Cargo to update the mtime of used files"),
     next_lockfile_bump: bool,
+    no_embed_metadata: bool = ("Avoid embedding metadata in library artifacts"),
     no_index_update: bool = ("Do not update the registry index even if the cache is outdated"),
     package_workspace: bool = ("Handle intra-workspace dependencies when packaging"),
     panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"),
@@ -1294,6 +1295,7 @@ impl CliUnstable {
             "msrv-policy" => self.msrv_policy = parse_empty(k, v)?,
             // can also be set in .cargo/config or with and ENV
             "mtime-on-use" => self.mtime_on_use = parse_empty(k, v)?,
+            "no-embed-metadata" => self.no_embed_metadata = parse_empty(k, v)?,
             "no-index-update" => self.no_index_update = parse_empty(k, v)?,
             "package-workspace" => self.package_workspace = parse_empty(k, v)?,
             "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?,

src/cargo/core/manifest.rs

@@ -279,6 +279,17 @@ impl TargetKind {
         }
     }
 
+    /// Returns whether production of this artifact could benefit from splitting metadata
+    /// into a .rmeta file.
+    pub fn benefits_from_no_embed_metadata(&self) -> bool {
+        match self {
+            TargetKind::Lib(kinds) | TargetKind::ExampleLib(kinds) => {
+                kinds.iter().any(|k| k.benefits_from_no_embed_metadata())
+            }
+            _ => false,
+        }
+    }
+
     /// Returns the arguments suitable for `--crate-type` to pass to rustc.
     pub fn rustc_crate_types(&self) -> Vec<CrateType> {
         match self {

tests/testsuite/build.rs

@@ -6749,3 +6749,91 @@ fn renamed_uplifted_artifact_remains_unmodified_after_rebuild() {
     let not_the_same = !same_file::is_same_file(bin, renamed_bin).unwrap();
     assert!(not_the_same, "renamed uplifted artifact must be unmodified");
 }
+
+#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")]
+fn embed_metadata() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+
+                name = "foo"
+                version = "0.5.0"
+                edition = "2015"
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[]))
+        .file("bar/Cargo.toml", &basic_lib_manifest("bar"))
+        .file(
+            "bar/src/bar.rs",
+            r#"
+                pub fn gimme() -> &'static str {
+                    "test passed"
+                }
+            "#,
+        )
+        .build();
+
+    p.cargo("build -Z no-embed-metadata")
+        .masquerade_as_nightly_cargo(&["-Z no-embed-metadata"])
+        .arg("-v")
+        .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`")
+        .with_stderr_contains(
+            "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar-[HASH].rmeta[..]`",
+        )
+        .run();
+}
+
+// Make sure that cargo passes --extern=<dep>.rmeta even if <dep>
+// is compiled as a dylib.
+#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")]
+fn embed_metadata_dylib_dep() {
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.5.0"
+                edition = "2015"
+
+                [dependencies.bar]
+                path = "bar"
+            "#,
+        )
+        .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[]))
+        .file(
+            "bar/Cargo.toml",
+            r#"
+                [package]
+                name = "bar"
+                version = "0.5.0"
+                edition = "2015"
+
+                [lib]
+                crate-type = ["dylib"]
+            "#,
+        )
+        .file(
+            "bar/src/lib.rs",
+            r#"
+                pub fn gimme() -> &'static str {
+                    "test passed"
+                }
+            "#,
+        )
+        .build();
+
+    p.cargo("build -Z no-embed-metadata")
+        .masquerade_as_nightly_cargo(&["-Z no-embed-metadata"])
+        .arg("-v")
+        .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`")
+        .with_stderr_contains(
+            "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar.rmeta[..]`",
+        )
+        .run();
+}