Gather rustc output from compilation

Author: googleson78

rust/private/common.bzl

@@ -49,6 +49,10 @@ def _create_crate_info(**kwargs):
         kwargs.update({"wrapped_crate_type": None})
     if not "metadata" in kwargs:
         kwargs.update({"metadata": None})
+    if not "rust_metadata_rustc_output" in kwargs:
+        kwargs.update({"rust_metadata_rustc_output": None})
+    if not "rust_lib_rustc_output" in kwargs:
+        kwargs.update({"rust_lib_rustc_output": None})
     if not "rustc_env_files" in kwargs:
         kwargs.update({"rustc_env_files": []})
     return CrateInfo(**kwargs)

rust/private/providers.bzl

@@ -22,12 +22,14 @@ CrateInfo = provider(
         "deps": "depset[DepVariantInfo]: This crate's (rust or cc) dependencies' providers.",
         "edition": "str: The edition of this crate.",
         "is_test": "bool: If the crate is being compiled in a test context",
-        "metadata": "File: The rmeta file produced for this crate. It is optional.",
+        "metadata": "File: The output from rustc from producing the output file. It is optional.",
         "name": "str: The name of this crate.",
         "output": "File: The output File that will be produced, depends on crate type.",
         "owner": "Label: The label of the target that produced this CrateInfo",
         "proc_macro_deps": "depset[DepVariantInfo]: This crate's rust proc_macro dependencies' providers.",
         "root": "File: The source File entrypoint to this crate, eg. lib.rs",
+        "rust_lib_rustc_output": "File: The output from rustc from producing the output file. It is optional.",
+        "rust_metadata_rustc_output": "File: The rmeta file produced for this crate. It is optional.",
         "rustc_env": "Dict[String, String]: Additional `\"key\": \"value\"` environment variables to set for rustc.",
         "rustc_env_files": "[File]: Files containing additional environment variables to set for rustc.",
         "srcs": "depset[File]: All source Files that are part of the crate.",

rust/private/rust.bzl

@@ -64,6 +64,10 @@ def _assert_correct_dep_mapping(ctx):
                 ),
             )
 
+
+def _rustc_output_name(name):
+  return name + ".rustc-output"
+
 def _determine_lib_name(name, crate_type, toolchain, lib_hash = None):
     """See https://github.com/bazelbuild/rules_rust/issues/405
 
@@ -275,14 +279,22 @@ def _rust_library_common(ctx, crate_type):
         toolchain,
         output_hash,
     )
+
     rust_lib = ctx.actions.declare_file(rust_lib_name)
+    rust_lib_build_output = None
+    if ctx.attr._process_wrapper:
+        rust_lib_build_output = ctx.actions.declare_file(_rustc_output_name(rust_lib_name))
 
     rust_metadata = None
+    rust_metadata_build_output = None
     if can_build_metadata(toolchain, ctx, crate_type) and not ctx.attr.disable_pipelining:
+        rust_metadata_name = paths.replace_extension(rust_lib_name, ".rmeta")
+
         rust_metadata = ctx.actions.declare_file(
-            paths.replace_extension(rust_lib_name, ".rmeta"),
+            rust_metadata_name,
             sibling = rust_lib,
         )
+        rust_metadata_build_output = ctx.actions.declare_file(_rustc_output_name(rust_metadata_name))
 
     deps = transform_deps(ctx.attr.deps)
     proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx))
@@ -300,7 +312,9 @@ def _rust_library_common(ctx, crate_type):
             proc_macro_deps = depset(proc_macro_deps),
             aliases = ctx.attr.aliases,
             output = rust_lib,
+            rust_lib_rustc_output = rust_lib_build_output,
             metadata = rust_metadata,
+            rust_metadata_rustc_output = rust_metadata_build_output,
             edition = get_edition(ctx.attr, toolchain, ctx.label),
             rustc_env = ctx.attr.rustc_env,
             rustc_env_files = ctx.files.rustc_env_files,
@@ -598,7 +612,7 @@ _common_attrs = {
             The order that these files will be processed is unspecified, so
             multiple definitions of a particular variable are discouraged.
 
-            Note that the variables here are subject to 
+            Note that the variables here are subject to
             [workspace status](https://docs.bazel.build/versions/main/user-manual.html#workspace_status)
             stamping should the `stamp` attribute be enabled. Stamp variables
             should be wrapped in brackets in order to be resolved. E.g.
@@ -611,7 +625,7 @@ _common_attrs = {
             List of compiler flags passed to `rustc`.
 
             These strings are subject to Make variable expansion for predefined
-            source/output path variables like `$location`, `$execpath`, and 
+            source/output path variables like `$location`, `$execpath`, and
             `$rootpath`. This expansion is useful if you wish to pass a generated
             file of arguments to rustc: `@$(location //package:target)`.
         """),
@@ -723,7 +737,7 @@ _rust_test_attrs = dict({
         mandatory = False,
         default = True,
         doc = dedent("""\
-            Whether to use `libtest`. For targets using this flag, individual tests can be run by using the 
+            Whether to use `libtest`. For targets using this flag, individual tests can be run by using the
             [--test_arg](https://docs.bazel.build/versions/4.0.0/command-line-reference.html#flag--test_arg) flag.
             E.g. `bazel test //src:rust_test --test_arg=foo::test::test_fn`.
         """),

rust/private/rustc.bzl

@@ -855,6 +855,12 @@ def construct_arguments(
     if build_metadata:
         # Configure process_wrapper to terminate rustc when metadata are emitted
         process_wrapper_flags.add("--rustc-quit-on-rmeta", "true")
+        if crate_info.rust_metadata_rustc_output:
+            process_wrapper_flags.add("--output-file", crate_info.rust_metadata_rustc_output.path)
+    else:
+        if crate_info.rust_lib_rustc_output:
+            process_wrapper_flags.add("--output-file", crate_info.rust_lib_rustc_output.path)
+
 
     rustc_flags.add("--error-format=" + error_format)
 
@@ -1018,6 +1024,8 @@ def rustc_compile_action(
             - (DefaultInfo): The output file for this crate, and its runfiles.
     """
     build_metadata = getattr(crate_info, "metadata", None)
+    rust_lib_rustc_output = getattr(crate_info, "rust_lib_rustc_output", None)
+    rust_metadata_rustc_output = getattr(crate_info, "rust_metadata_rustc_output", None)
 
     cc_toolchain, feature_configuration = find_cc_toolchain(ctx)
 
@@ -1156,6 +1164,8 @@ def rustc_compile_action(
 
     # The action might generate extra output that we don't want to include in the `DefaultInfo` files.
     action_outputs = list(outputs)
+    if rust_lib_rustc_output:
+        action_outputs.append(rust_lib_rustc_output)
 
     # Rustc generates a pdb file (on Windows) or a dsym folder (on macos) so provide it in an output group for crate
     # types that benefit from having debug information in a separate file.
@@ -1189,7 +1199,7 @@ def rustc_compile_action(
             ctx.actions.run(
                 executable = ctx.executable._process_wrapper,
                 inputs = compile_inputs,
-                outputs = [build_metadata],
+                outputs = [build_metadata, rust_metadata_rustc_output],
                 env = env,
                 arguments = args_metadata.all,
                 mnemonic = "RustcMetadata",
@@ -1307,12 +1317,23 @@ def rustc_compile_action(
 
     if toolchain.target_arch != "wasm32":
         providers += establish_cc_info(ctx, attr, crate_info, toolchain, cc_toolchain, feature_configuration, interface_library)
+
+    output_group_info = {}
+
     if pdb_file:
-        providers.append(OutputGroupInfo(pdb_file = depset([pdb_file])))
+        output_group_info["pdb_file"] = depset([pdb_file])
     if dsym_folder:
-        providers.append(OutputGroupInfo(dsym_folder = depset([dsym_folder])))
+        output_group_info["dsym_folder"] = depset([dsym_folder])
+    if build_metadata:
+        output_group_info["build_metadata"] = depset([build_metadata])
     if build_metadata:
-        providers.append(OutputGroupInfo(build_metadata = depset([build_metadata])))
+        output_group_info["build_metadata"] = depset([build_metadata])
+        output_group_info["rust_metadata_rustc_output"] = depset([rust_metadata_rustc_output])
+    if rust_lib_rustc_output:
+        output_group_info["rust_lib_rustc_output"] = depset([rust_lib_rustc_output])
+
+    if output_group_info:
+        providers.append(OutputGroupInfo(**output_group_info))
 
     return providers
 

util/process_wrapper/main.rs

@@ -89,18 +89,26 @@ fn main() {
 
     let mut child_stderr = child.stderr.take().unwrap();
 
+    let mut output_file: Box<Option<std::fs::File>> = Box::new(opts.output_file.clone().map(|output_file_name|
+            OpenOptions::new()
+                .create(true)
+                .truncate(true)
+                .write(true)
+                .open(output_file_name)
+                .expect("process wrapper error: unable to open output_file")));
+
     let mut was_killed = false;
     let result = if let Some(format) = opts.rustc_output_format {
         let quit_on_rmeta = opts.rustc_quit_on_rmeta;
         // Process json rustc output and kill the subprocess when we get a signal
         // that we emitted a metadata file.
         let mut me = false;
         let metadata_emitted = &mut me;
-        let result = process_output(&mut child_stderr, stderr.as_mut(), move |line| {
+        let result = process_output(&mut child_stderr, stderr.as_mut(), output_file.as_mut(), move |line| {
             if quit_on_rmeta {
-                rustc::stop_on_rmeta_completion(line, format, metadata_emitted)
+                rustc::stop_on_rmeta_completion(line, format.clone(), metadata_emitted)
             } else {
-                rustc::process_json(line, format)
+                rustc::process_json(line, format.clone())
             }
         });
         if me {
@@ -112,7 +120,7 @@ fn main() {
         result
     } else {
         // Process output normally by forwarding stderr
-        process_output(&mut child_stderr, stderr.as_mut(), LineOutput::Message)
+        process_output(&mut child_stderr, stderr.as_mut(), output_file.as_mut(), LineOutput::Message)
     };
     result.expect("process wrapper error: failed to process stderr");
 

util/process_wrapper/options.rs

@@ -41,6 +41,9 @@ pub(crate) struct Options {
     pub(crate) stdout_file: Option<String>,
     // If set, redirects the child process stderr to this file.
     pub(crate) stderr_file: Option<String>,
+    // If set, also logs all unprocessed output from the  to this file.
+    // Meant to be used to get json output out of rustc for tooling usage.
+    pub(crate) output_file: Option<String>,
     // If set, it configures rustc to emit an rmeta file and then
     // quit.
     pub(crate) rustc_quit_on_rmeta: bool,
@@ -60,6 +63,7 @@ pub(crate) fn options() -> Result<Options, OptionError> {
     let mut copy_output_raw = None;
     let mut stdout_file = None;
     let mut stderr_file = None;
+    let mut output_file = None;
     let mut rustc_quit_on_rmeta_raw = None;
     let mut rustc_output_format_raw = None;
     let mut flags = Flags::new();
@@ -92,6 +96,11 @@ pub(crate) fn options() -> Result<Options, OptionError> {
         "Redirect subprocess stderr in this file.",
         &mut stderr_file,
     );
+    flags.define_flag(
+        "--output-file",
+        "Log all unprocessed subprocess stderr in this file.",
+        &mut output_file,
+    );
     flags.define_flag(
         "--rustc-quit-on-rmeta",
         "If enabled, this wrapper will terminate rustc after rmeta has been emitted.",
@@ -202,6 +211,7 @@ pub(crate) fn options() -> Result<Options, OptionError> {
         copy_output,
         stdout_file,
         stderr_file,
+        output_file,
         rustc_quit_on_rmeta,
         rustc_output_format,
     })

util/process_wrapper/output.rs

@@ -32,22 +32,27 @@ pub(crate) enum LineOutput {
 /// to write_end.
 pub(crate) fn process_output<F>(
     read_end: &mut dyn Read,
-    write_end: &mut dyn Write,
+    output_write_end: &mut dyn Write,
+    opt_file_write_end: &mut Option<std::fs::File>,
     mut process_line: F,
 ) -> io::Result<()>
 where
     F: FnMut(String) -> LineOutput,
 {
     let mut reader = io::BufReader::new(read_end);
-    let mut writer = io::LineWriter::new(write_end);
+    let mut output_writer = io::LineWriter::new(output_write_end);
+    let mut file_writer = opt_file_write_end.as_mut().map(|file_write_end| io::LineWriter::new(file_write_end));
     loop {
         let mut line = String::new();
         let read_bytes = reader.read_line(&mut line)?;
         if read_bytes == 0 {
             break;
         }
+        if let Some(ref mut file) = file_writer {
+            file.write_all(line.as_bytes())?
+        }
         match process_line(line) {
-            LineOutput::Message(to_write) => writer.write_all(to_write.as_bytes())?,
+            LineOutput::Message(to_write) => output_writer.write_all(to_write.as_bytes())?,
             LineOutput::Skip => {}
             LineOutput::Terminate => return Ok(()),
         };