diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index 209d88a02d2..c2fcad6b7b3 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -570,9 +570,10 @@ impl TargetInfo { mode: CompileMode, target_kind: &TargetKind, target_triple: &str, + gctx: &GlobalContext, ) -> CargoResult<(Vec, Vec)> { match mode { - CompileMode::Build => self.calc_rustc_outputs(target_kind, target_triple), + CompileMode::Build => self.calc_rustc_outputs(target_kind, target_triple, gctx), CompileMode::Test | CompileMode::Bench => { match self.file_types(&CrateType::Bin, FileFlavor::Normal, target_triple)? { Some(fts) => Ok((fts, Vec::new())), @@ -593,6 +594,7 @@ impl TargetInfo { &self, target_kind: &TargetKind, target_triple: &str, + gctx: &GlobalContext, ) -> CargoResult<(Vec, Vec)> { let mut unsupported = Vec::new(); let mut result = Vec::new(); @@ -613,9 +615,18 @@ impl TargetInfo { } } } - if !result.is_empty() && !crate_types.iter().any(|ct| ct.requires_upstream_objects()) { - // Only add rmeta if pipelining. - result.push(FileType::new_rmeta()); + if !result.is_empty() { + if gctx.cli_unstable().no_embed_metadata + && crate_types + .iter() + .any(|ct| ct.benefits_from_no_embed_metadata()) + { + // Add .rmeta when we apply -Zembed-metadata=no to the unit. + result.push(FileType::new_rmeta()); + } else if !crate_types.iter().any(|ct| ct.requires_upstream_objects()) { + // Only add rmeta if pipelining + result.push(FileType::new_rmeta()); + } } Ok((result, unsupported)) } diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/cargo/core/compiler/build_runner/compilation_files.rs index c691a19ad0b..b6f44155977 100644 --- a/src/cargo/core/compiler/build_runner/compilation_files.rs +++ b/src/cargo/core/compiler/build_runner/compilation_files.rs @@ -363,6 +363,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> { CompileMode::Build, &TargetKind::Bin, bcx.target_data.short_name(&kind), + bcx.gctx, ) .expect("target must support `bin`"); @@ -540,7 +541,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> { let info = bcx.target_data.info(unit.kind); let triple = bcx.target_data.short_name(&unit.kind); let (file_types, unsupported) = - info.rustc_outputs(unit.mode, unit.target.kind(), triple)?; + info.rustc_outputs(unit.mode, unit.target.kind(), triple, bcx.gctx)?; if file_types.is_empty() { if !unsupported.is_empty() { let unsupported_strs: Vec<_> = unsupported.iter().map(|ct| ct.as_str()).collect(); diff --git a/src/cargo/core/compiler/crate_type.rs b/src/cargo/core/compiler/crate_type.rs index a36ef6c0f83..531df0f8c5e 100644 --- a/src/cargo/core/compiler/crate_type.rs +++ b/src/cargo/core/compiler/crate_type.rs @@ -76,6 +76,38 @@ impl CrateType { // Everything else, however, is some form of "linkable output" or // something that requires upstream object files. } + + /// Returns whether production of this crate type could benefit from splitting metadata + /// into a .rmeta file. + /// + /// See also [`TargetKind::benefits_from_no_embed_metadata`]. + /// + /// [`TargetKind::benefits_from_no_embed_metadata`]: crate::core::manifest::TargetKind::benefits_from_no_embed_metadata + pub fn benefits_from_no_embed_metadata(&self) -> bool { + match self { + // rlib/libs generate .rmeta files for pipelined compilation. + // If we also include metadata inside of them, we waste disk space, since the metadata + // will be located both in the lib/rlib and the .rmeta file. + CrateType::Lib | + CrateType::Rlib | + // Dylibs do not have to contain metadata when they are used as a runtime dependency. + // If we split the metadata into a separate .rmeta file, the dylib file (that + // can be shipped as a runtime dependency) can be smaller. + CrateType::Dylib => true, + // Proc macros contain metadata that specifies what macro functions are available in + // it, but the metadata is typically very small. The metadata of proc macros is also + // self-contained (unlike rlibs/dylibs), so let's not unnecessarily split it into + // multiple files. + CrateType::ProcMacro | + // cdylib and staticlib produce artifacts that are used through the C ABI and do not + // contain Rust-specific metadata. + CrateType::Cdylib | + CrateType::Staticlib | + // Binaries also do not contain metadata + CrateType::Bin | + CrateType::Other(_) => false + } + } } impl fmt::Display for CrateType { diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 2a0fd8b40f3..9be3148fd0f 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -1131,13 +1131,31 @@ fn build_base_args( if unit.mode.is_check() { cmd.arg("--emit=dep-info,metadata"); - } else if !unit.requires_upstream_objects() { - // Always produce metadata files for rlib outputs. Metadata may be used - // in this session for a pipelined compilation, or it may be used in a - // future Cargo session as part of a pipelined compile. - cmd.arg("--emit=dep-info,metadata,link"); + } else if build_runner.bcx.gctx.cli_unstable().no_embed_metadata { + // Nightly rustc supports the -Zembed-metadata=no flag, which tells it to avoid including + // full metadata in rlib/dylib artifacts, to save space on disk. In this case, metadata + // will only be stored in .rmeta files. + // When we use this flag, we should also pass --emit=metadata to all artifacts that + // contain useful metadata (rlib/dylib/proc macros), so that a .rmeta file is actually + // generated. If we didn't do this, the full metadata would not get written anywhere. + // However, we do not want to pass --emit=metadata to artifacts that never produce useful + // metadata, such as binaries, because that would just unnecessarily create empty .rmeta + // files on disk. + if unit.benefits_from_no_embed_metadata() { + cmd.arg("--emit=dep-info,metadata,link"); + cmd.args(&["-Z", "embed-metadata=no"]); + } else { + cmd.arg("--emit=dep-info,link"); + } } else { - cmd.arg("--emit=dep-info,link"); + // If we don't use -Zembed-metadata=no, we emit .rmeta files only for rlib outputs. + // This metadata may be used in this session for a pipelined compilation, or it may + // be used in a future Cargo session as part of a pipelined compile. + if !unit.requires_upstream_objects() { + cmd.arg("--emit=dep-info,metadata,link"); + } else { + cmd.arg("--emit=dep-info,link"); + } } let prefer_dynamic = (unit.target.for_host() && !unit.target.is_custom_build()) @@ -1636,6 +1654,8 @@ pub fn extern_args( let mut result = Vec::new(); let deps = build_runner.unit_deps(unit); + let no_embed_metadata = build_runner.bcx.gctx.cli_unstable().no_embed_metadata; + // Closure to add one dependency to `result`. let mut link_to = |dep: &UnitDep, extern_crate_name: InternedString, noprelude: bool| -> CargoResult<()> { @@ -1685,6 +1705,12 @@ pub fn extern_args( if output.flavor == FileFlavor::Linkable { pass(&output.path); } + // If we use -Zembed-metadata=no, we also need to pass the path to the + // corresponding .rmeta file to the linkable artifact, because the + // normal dependency (rlib) doesn't contain the full metadata. + else if no_embed_metadata && output.flavor == FileFlavor::Rmeta { + pass(&output.path); + } } } Ok(()) diff --git a/src/cargo/core/compiler/unit.rs b/src/cargo/core/compiler/unit.rs index 628b0b784ff..631353aaeff 100644 --- a/src/cargo/core/compiler/unit.rs +++ b/src/cargo/core/compiler/unit.rs @@ -125,6 +125,13 @@ impl UnitInner { self.mode.is_any_test() || self.target.kind().requires_upstream_objects() } + /// Returns whether compilation of this unit could benefit from splitting metadata + /// into a .rmeta file. + pub fn benefits_from_no_embed_metadata(&self) -> bool { + matches!(self.mode, CompileMode::Build) + && self.target.kind().benefits_from_no_embed_metadata() + } + /// Returns whether or not this is a "local" package. /// /// A "local" package is one that the user can likely edit, or otherwise diff --git a/src/cargo/core/features.rs b/src/cargo/core/features.rs index 2c2570e09b2..71a81cfe5d5 100644 --- a/src/cargo/core/features.rs +++ b/src/cargo/core/features.rs @@ -783,6 +783,7 @@ unstable_cli_options!( msrv_policy: bool = ("Enable rust-version aware policy within cargo"), mtime_on_use: bool = ("Configure Cargo to update the mtime of used files"), next_lockfile_bump: bool, + no_embed_metadata: bool = ("Avoid embedding metadata in library artifacts"), no_index_update: bool = ("Do not update the registry index even if the cache is outdated"), package_workspace: bool = ("Handle intra-workspace dependencies when packaging"), panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"), @@ -1294,6 +1295,7 @@ impl CliUnstable { "msrv-policy" => self.msrv_policy = parse_empty(k, v)?, // can also be set in .cargo/config or with and ENV "mtime-on-use" => self.mtime_on_use = parse_empty(k, v)?, + "no-embed-metadata" => self.no_embed_metadata = parse_empty(k, v)?, "no-index-update" => self.no_index_update = parse_empty(k, v)?, "package-workspace" => self.package_workspace = parse_empty(k, v)?, "panic-abort-tests" => self.panic_abort_tests = parse_empty(k, v)?, diff --git a/src/cargo/core/manifest.rs b/src/cargo/core/manifest.rs index 2a7cac65ba5..17acd494f06 100644 --- a/src/cargo/core/manifest.rs +++ b/src/cargo/core/manifest.rs @@ -279,6 +279,17 @@ impl TargetKind { } } + /// Returns whether production of this artifact could benefit from splitting metadata + /// into a .rmeta file. + pub fn benefits_from_no_embed_metadata(&self) -> bool { + match self { + TargetKind::Lib(kinds) | TargetKind::ExampleLib(kinds) => { + kinds.iter().any(|k| k.benefits_from_no_embed_metadata()) + } + _ => false, + } + } + /// Returns the arguments suitable for `--crate-type` to pass to rustc. pub fn rustc_crate_types(&self) -> Vec { match self { diff --git a/src/cargo/ops/cargo_clean.rs b/src/cargo/ops/cargo_clean.rs index c92cde493ed..608288884c6 100644 --- a/src/cargo/ops/cargo_clean.rs +++ b/src/cargo/ops/cargo_clean.rs @@ -230,7 +230,7 @@ fn clean_specs( let (file_types, _unsupported) = target_data .info(*compile_kind) - .rustc_outputs(mode, target.kind(), triple)?; + .rustc_outputs(mode, target.kind(), triple, clean_ctx.gctx)?; let (dir, uplift_dir) = match target.kind() { TargetKind::ExampleBin | TargetKind::ExampleLib(..) => { (layout.build_examples(), Some(layout.examples())) diff --git a/src/doc/src/reference/unstable.md b/src/doc/src/reference/unstable.md index 9c499046127..459ac7ff114 100644 --- a/src/doc/src/reference/unstable.md +++ b/src/doc/src/reference/unstable.md @@ -91,6 +91,7 @@ Each new feature described below should explain how to use it. * [checksum-freshness](#checksum-freshness) --- When passed, the decision as to whether a crate needs to be rebuilt is made using file checksums instead of the file mtime. * [panic-abort-tests](#panic-abort-tests) --- Allows running tests with the "abort" panic strategy. * [host-config](#host-config) --- Allows setting `[target]`-like configuration settings for host build targets. + * [no-embed-metadata](#no-embed-metadata) --- Passes `-Zembed-metadata=no` to the compiler, which avoid embedding metadata into rlib and dylib artifacts, to save disk space. * [target-applies-to-host](#target-applies-to-host) --- Alters whether certain flags will be passed to host build targets. * [gc](#gc) --- Global cache garbage collection. * [open-namespaces](#open-namespaces) --- Allow multiple packages to participate in the same API namespace @@ -1914,6 +1915,23 @@ The `-Z rustdoc-depinfo` flag leverages rustdoc's dep-info files to determine whether documentations are required to re-generate. This can be combined with `-Z checksum-freshness` to detect checksum changes rather than file mtime. +## no-embed-metadata +* Original Pull Request: [#15378](https://github.com/rust-lang/cargo/pull/15378) +* Tracking Issue: [#15495](https://github.com/rust-lang/cargo/issues/15495) + +The default behavior of Rust is to embed crate metadata into `rlib` and `dylib` artifacts. +Since Cargo also passes `--emit=metadata` to these intermediate artifacts to enable pipelined +compilation, this means that a lot of metadata ends up being duplicated on disk, which wastes +disk space in the target directory. + +This feature tells Cargo to pass the `-Zembed-metadata=no` flag to the compiler, which instructs +it not to embed metadata within rlib and dylib artifacts. In this case, the metadata will only +be stored in `.rmeta` files. + +```console +cargo +nightly -Zno-embed-metadata build +``` + # Stabilized and removed features ## Compile progress diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index b37ae9fc63f..c2d8859996e 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -6749,3 +6749,91 @@ fn renamed_uplifted_artifact_remains_unmodified_after_rebuild() { let not_the_same = !same_file::is_same_file(bin, renamed_bin).unwrap(); assert!(not_the_same, "renamed uplifted artifact must be unmodified"); } + +#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")] +fn embed_metadata() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + + name = "foo" + version = "0.5.0" + edition = "2015" + + [dependencies.bar] + path = "bar" + "#, + ) + .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[])) + .file("bar/Cargo.toml", &basic_lib_manifest("bar")) + .file( + "bar/src/bar.rs", + r#" + pub fn gimme() -> &'static str { + "test passed" + } + "#, + ) + .build(); + + p.cargo("build -Z no-embed-metadata") + .masquerade_as_nightly_cargo(&["-Z no-embed-metadata"]) + .arg("-v") + .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`") + .with_stderr_contains( + "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar-[HASH].rmeta[..]`", + ) + .run(); +} + +// Make sure that cargo passes --extern=.rmeta even if +// is compiled as a dylib. +#[cargo_test(nightly, reason = "-Zembed-metadata is nightly only")] +fn embed_metadata_dylib_dep() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.5.0" + edition = "2015" + + [dependencies.bar] + path = "bar" + "#, + ) + .file("src/main.rs", &main_file(r#""{}", bar::gimme()"#, &[])) + .file( + "bar/Cargo.toml", + r#" + [package] + name = "bar" + version = "0.5.0" + edition = "2015" + + [lib] + crate-type = ["dylib"] + "#, + ) + .file( + "bar/src/lib.rs", + r#" + pub fn gimme() -> &'static str { + "test passed" + } + "#, + ) + .build(); + + p.cargo("build -Z no-embed-metadata") + .masquerade_as_nightly_cargo(&["-Z no-embed-metadata"]) + .arg("-v") + .with_stderr_contains("[RUNNING] `[..]-Z embed-metadata=no[..]`") + .with_stderr_contains( + "[RUNNING] `[..]--extern bar=[ROOT]/foo/target/debug/deps/libbar.rmeta[..]`", + ) + .run(); +} diff --git a/tests/testsuite/cargo/z_help/stdout.term.svg b/tests/testsuite/cargo/z_help/stdout.term.svg index c152f663f20..b8fc4407b7c 100644 --- a/tests/testsuite/cargo/z_help/stdout.term.svg +++ b/tests/testsuite/cargo/z_help/stdout.term.svg @@ -1,4 +1,4 @@ - +