From dc5997c3a0654eb013a97dc71fb7bc49cfb963de Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 2 Feb 2022 07:34:31 -0800 Subject: [PATCH 01/11] `rust_toolchain` now recreates a Rust sysroot --- rust/private/rustc.bzl | 8 +- rust/private/rustdoc_test.bzl | 5 + rust/private/toolchain_utils.bzl | 166 +++++++++++++++++++++++++++++++ rust/toolchain.bzl | 47 +++++---- 4 files changed, 203 insertions(+), 23 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index ffa3929ca0..6c63f0d47e 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -407,19 +407,16 @@ def collect_inputs( nolinkstamp_compile_inputs = depset( getattr(files, "data", []) + - [toolchain.rustc] + - toolchain.crosstool_files + ([build_info.rustc_env, build_info.flags] if build_info else []) + ([toolchain.target_json] if toolchain.target_json else []) + ([] if linker_script == None else [linker_script]), transitive = [ - toolchain.rustc_lib, - toolchain.rust_std, linker_depset, crate_info.srcs, dep_info.transitive_crate_outputs, depset(additional_transitive_inputs), crate_info.compile_data, + toolchain.all_files, ], ) @@ -681,8 +678,9 @@ def construct_arguments( data_paths, )) - # Set the SYSROOT to the directory of the rust_std files passed to the toolchain + # Ensure the sysroot is set for the target platform env["SYSROOT"] = toolchain.sysroot + rustc_flags.add("--sysroot", "${pwd}/" + toolchain.sysroot) # extra_rustc_flags apply to the target configuration, not the exec configuration. if hasattr(ctx.attr, "_extra_rustc_flags") and not is_exec_configuration(ctx): diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index a73a0c7d50..163c72e086 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -56,6 +56,11 @@ def _construct_writer_arguments(ctx, test_runner, action, crate_info, rust_toolc for var in action.env.keys(): writer_args.add("--action_env={}".format(var)) + # Ensure the rustdoc binary is always accessed via a relative path + writer_args.add("--strip_substring={}/".format( + rust_toolchain.rust_doc.root.path, + )) + # Since the test runner will be running from a runfiles directory, the # paths originally generated for the build action will not map to any # files. To ensure rustdoc can find the appropriate dependencies, the diff --git a/rust/private/toolchain_utils.bzl b/rust/private/toolchain_utils.bzl index fcd4fb7029..2aabb80e8c 100644 --- a/rust/private/toolchain_utils.bzl +++ b/rust/private/toolchain_utils.bzl @@ -1,5 +1,171 @@ """A module defining toolchain utilities""" +def _symlink_sysroot_tree(ctx, name, target): + """Generate a set of symlinks to files from another target + + Args: + ctx (ctx): The toolchain's context object + name (str): The name of the sysroot directory (typically `ctx.label.name`) + target (Target): A target owning files to symlink + + Returns: + depset[File]: A depset of the generated symlink files + """ + tree_files = [] + for file in target.files.to_list(): + # Parse the path to the file relative to the workspace root so a + # symlink matching this path can be created within the sysroot. + + # The code blow attempts to parse any workspace names out of the + # path. For local targets, this code is a noop. + if target.label.workspace_root: + file_path = file.path.split(target.label.workspace_root, 1)[-1] + else: + file_path = file.path + + symlink = ctx.actions.declare_file("{}/{}".format(name, file_path.lstrip("/"))) + + ctx.actions.symlink( + output = symlink, + target_file = file, + ) + + tree_files.append(symlink) + + return depset(tree_files) + +def _symlink_sysroot_bin(ctx, name, dir, target): + """Crete a symlink to a target file. + + Args: + ctx (ctx): The rule's context object + name (str): A common name for the output directory + dir (str): The directory under `name` to put the file in + target (File): A File object to symlink to + + Returns: + File: A newly generated symlink file + """ + symlink = ctx.actions.declare_file("{}/{}/{}".format( + name, + dir.lstrip("/"), + target.basename, + )) + + ctx.actions.symlink( + output = symlink, + target_file = target, + is_executable = True, + ) + + return symlink + +def generate_sysroot( + ctx, + rustc, + rustdoc, + rustc_lib, + cargo = None, + clippy = None, + llvm_tools = None, + rust_std = None, + rustfmt = None): + """Generate a rust sysroot from an exec and target toolchain + + Args: + ctx (ctx): A context object from a `rust_toolchain` rule. + rustc (File): The path to a `rustc` executable. + rustdoc (File): The path to a `rustdoc` executable. + rustc_lib (Target): A collection of Files containing dependencies of `rustc`. + cargo (File, optional): The path to a `cargo` executable. + clippy (File, optional): The path to a `clippy-driver` executable. + llvm_tools (Target, optional): A collection of llvm tools used by `rustc`. + rust_std (Target, optional): A collection of Files containing Rust standard library components. + rustfmt (File, optional): The path to a `rustfmt` executable. + + Returns: + struct: A struct of generated files representing the new sysroot + """ + name = ctx.label.name + + # Define runfiles + direct_files = [] + transitive_file_sets = [] + + # Rustc + sysroot_rustc = _symlink_sysroot_bin(ctx, name, "/bin", rustc) + direct_files.extend([sysroot_rustc, rustc]) + + # Rustc dependencies + sysroot_rustc_lib = _symlink_sysroot_tree(ctx, name, rustc_lib) if rustc_lib else None + if sysroot_rustc_lib: + transitive_file_sets.extend([sysroot_rustc_lib, rustc_lib.files]) + + # Rustdoc + sysroot_rustdoc = _symlink_sysroot_bin(ctx, name, "/bin", rustdoc) + direct_files.extend([sysroot_rustdoc, rustdoc]) + + # Clippy + sysroot_clippy = _symlink_sysroot_bin(ctx, name, "/bin", clippy) if clippy else None + if sysroot_clippy: + direct_files.extend([sysroot_clippy, clippy]) + + # Cargo + sysroot_cargo = _symlink_sysroot_bin(ctx, name, "/bin", cargo) if cargo else None + if sysroot_cargo: + direct_files.extend([sysroot_cargo, cargo]) + + # Rustfmt + sysroot_rustfmt = _symlink_sysroot_bin(ctx, name, "/bin", rustfmt) if rustfmt else None + if sysroot_rustfmt: + direct_files.extend([sysroot_rustfmt, rustfmt]) + + # Llvm tools + sysroot_llvm_tools = _symlink_sysroot_tree(ctx, name, llvm_tools) if llvm_tools else None + if sysroot_llvm_tools: + transitive_file_sets.extend([sysroot_llvm_tools, llvm_tools.files]) + + # Rust standard library + sysroot_rust_std = _symlink_sysroot_tree(ctx, name, rust_std) if rust_std else None + if sysroot_rust_std: + transitive_file_sets.extend([sysroot_rust_std, rust_std.files]) + + # Symlink rust-stdlib + sysroot_rust_std = _symlink_sysroot_tree(ctx, name, rust_std) if rust_std else None + if sysroot_rust_std: + transitive_file_sets.extend([sysroot_rust_std, rust_std.files]) + + # Declare a file in the root of the sysroot to make locating the sysroot easy + sysroot_anchor = ctx.actions.declare_file("{}/rust.sysroot".format(name)) + ctx.actions.write( + output = sysroot_anchor, + content = "\n".join([ + "cargo: {}".format(cargo), + "clippy: {}".format(clippy), + "llvm_tools: {}".format(llvm_tools), + "rust_std: {}".format(rust_std), + "rustc_lib: {}".format(rustc_lib), + "rustc: {}".format(rustc), + "rustdoc: {}".format(rustdoc), + "rustfmt: {}".format(rustfmt), + ]), + ) + + # Create a depset of all sysroot files (symlinks and their real paths) + all_files = depset(direct_files, transitive = transitive_file_sets) + + return struct( + all_files = all_files, + cargo = sysroot_cargo, + clippy = sysroot_clippy, + rust_std = sysroot_rust_std, + rustc = sysroot_rustc, + rustc_lib = sysroot_rustc_lib, + rustdoc = sysroot_rustdoc, + rustfmt = sysroot_rustfmt, + sysroot_anchor = sysroot_anchor, + ) + def _toolchain_files_impl(ctx): toolchain = ctx.toolchains[str(Label("//rust:toolchain"))] diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index d3cb8f43bc..c078c75d92 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -2,6 +2,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//rust/private:common.bzl", "rust_common") +load("//rust/private:toolchain_utils.bzl", "generate_sysroot") load("//rust/private:utils.bzl", "dedent", "find_cc_toolchain", "make_static_lib_symlink") load("//rust/settings:incompatible.bzl", "IncompatibleFlagInfo") @@ -246,6 +247,18 @@ def _rust_toolchain_impl(ctx): else: rust_std = ctx.attr.rust_std + sysroot = generate_sysroot( + ctx = ctx, + rustc = ctx.file.rustc, + rustdoc = ctx.file.rust_doc, + rustc_lib = ctx.attr.rustc_lib, + rust_std = rust_std, + rustfmt = ctx.file.rustfmt, + clippy = ctx.file.clippy_driver, + cargo = ctx.file.cargo, + llvm_tools = ctx.attr.llvm_tools, + ) + expanded_stdlib_linkflags = [] for flag in ctx.attr.stdlib_linkflags: expanded_stdlib_linkflags.append( @@ -268,29 +281,23 @@ def _rust_toolchain_impl(ctx): linking_context = linking_context, ) - # In cases where the toolchain uses the Rust standard library, calculate sysroot path - sysroot_path = None - if rust_std: - # Calculate the rustc sysroot path by using a file from the rust-std bundle - rust_std_files_list = rust_std.files.to_list() - if not rust_std_files_list: - fail("The `rust_std` cannot be represented by an empty list") - sysroot_path = rust_std_files_list[0].dirname + all_files = depset(ctx.files._crosstool, transitive = [sysroot.all_files]) toolchain = platform_common.ToolchainInfo( - rustc = ctx.file.rustc, - rust_doc = ctx.file.rust_doc, - rustfmt = ctx.file.rustfmt, - cargo = ctx.file.cargo, - clippy_driver = ctx.file.clippy_driver, + all_files = all_files, + rustc = sysroot.rustc, + rust_doc = sysroot.rustdoc, + rustfmt = sysroot.rustfmt, + cargo = sysroot.cargo, + clippy_driver = sysroot.clippy, target_json = ctx.file.target_json, target_flag_value = ctx.file.target_json.path if ctx.file.target_json else ctx.attr.target_triple, - rustc_lib = depset(ctx.files.rustc_lib), + rustc_lib = sysroot.rustc_lib, rustc_srcs = ctx.attr.rustc_srcs, - rust_std = rust_std.files, - rust_std_paths = depset([file.dirname for file in rust_std_files_list]), - rust_lib = rust_std.files, # `rust_lib` is deprecated and only exists for legacy support. - sysroot = sysroot_path, + rust_std = sysroot.rust_std, + rust_std_paths = depset([file.dirname for file in sysroot.rust_std.to_list()]), + rust_lib = sysroot.rust_std, # `rust_lib` is deprecated and only exists for legacy support. + sysroot = sysroot.sysroot_anchor.dirname, binary_ext = ctx.attr.binary_ext, staticlib_ext = ctx.attr.staticlib_ext, dylib_ext = ctx.attr.dylib_ext, @@ -358,6 +365,10 @@ rust_toolchain = rule( ), mandatory = True, ), + "llvm_tools": attr.label_list( + doc = "llvm-tools", + allow_files = True, + ), "opt_level": attr.string_dict( doc = "Rustc optimization levels.", default = { From a5ad2411c409d53b1c3db0583448afa8889910c0 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Thu, 3 Feb 2022 08:15:14 -0800 Subject: [PATCH 02/11] Regenerate documentation --- docs/flatten.md | 7 ++++--- docs/rust_repositories.md | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/flatten.md b/docs/flatten.md index 10df0a0f48..787d0a87a5 100644 --- a/docs/flatten.md +++ b/docs/flatten.md @@ -1110,9 +1110,9 @@ Run the test with `bazel build //hello_lib:hello_lib_test`. 
 rust_toolchain(name, allocator_library, binary_ext, cargo, clippy_driver, debug_info,
-               default_edition, dylib_ext, exec_triple, opt_level, os, rust_doc, rust_lib, rust_std,
-               rustc, rustc_lib, rustc_srcs, rustfmt, staticlib_ext, stdlib_linkflags, target_json,
-               target_triple)
+               default_edition, dylib_ext, exec_triple, llvm_tools, opt_level, os, rust_doc, rust_lib,
+               rust_std, rustc, rustc_lib, rustc_srcs, rustfmt, staticlib_ext, stdlib_linkflags,
+               target_json, target_triple)
Declares a Rust toolchain for use. @@ -1170,6 +1170,7 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r | default_edition | The edition to use for rust_* rules that don't specify an edition. | String | optional | "2018" | | dylib_ext | The extension for dynamic libraries created from rustc. | String | required | | | exec_triple | The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations | String | required | | +| llvm_tools | llvm-tools | List of labels | optional | [] | | opt_level | Rustc optimization levels. | Dictionary: String -> String | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} | | os | The operating system for the current toolchain | String | required | | | rust_doc | The location of the rustdoc binary. Can be a direct source or a filegroup containing one item. | Label | required | | diff --git a/docs/rust_repositories.md b/docs/rust_repositories.md index d912ba7759..a98729d149 100644 --- a/docs/rust_repositories.md +++ b/docs/rust_repositories.md @@ -35,9 +35,9 @@ A dedicated filegroup-like rule for Rust stdlib artifacts. 
 rust_toolchain(name, allocator_library, binary_ext, cargo, clippy_driver, debug_info,
-               default_edition, dylib_ext, exec_triple, opt_level, os, rust_doc, rust_lib, rust_std,
-               rustc, rustc_lib, rustc_srcs, rustfmt, staticlib_ext, stdlib_linkflags, target_json,
-               target_triple)
+               default_edition, dylib_ext, exec_triple, llvm_tools, opt_level, os, rust_doc, rust_lib,
+               rust_std, rustc, rustc_lib, rustc_srcs, rustfmt, staticlib_ext, stdlib_linkflags,
+               target_json, target_triple)
Declares a Rust toolchain for use. @@ -95,6 +95,7 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r | default_edition | The edition to use for rust_* rules that don't specify an edition. | String | optional | "2018" | | dylib_ext | The extension for dynamic libraries created from rustc. | String | required | | | exec_triple | The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations | String | required | | +| llvm_tools | llvm-tools | List of labels | optional | [] | | opt_level | Rustc optimization levels. | Dictionary: String -> String | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} | | os | The operating system for the current toolchain | String | required | | | rust_doc | The location of the rustdoc binary. Can be a direct source or a filegroup containing one item. | Label | required | | From a350eff41c279405718e27a9616af06abcf9b6df Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 7 Feb 2022 09:23:28 -0800 Subject: [PATCH 03/11] Updated docs --- rust/toolchain.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index 73b72a2c6f..e384bbbb29 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -362,7 +362,7 @@ rust_toolchain = rule( mandatory = True, ), "llvm_tools": attr.label_list( - doc = "llvm-tools", + doc = "LLVM tools that are shipped with the Rust toolchain.", allow_files = True, ), "opt_level": attr.string_dict( From ed1afdb43e7582636b942603b8ff4bd8afd48621 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 7 Feb 2022 09:28:19 -0800 Subject: [PATCH 04/11] Address merge conflicts --- rust/toolchain.bzl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index e384bbbb29..9fc0ed28d1 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -278,10 +278,8 @@ def _rust_toolchain_impl(ctx): linking_context = linking_context, ) - all_files = depset(ctx.files._crosstool, transitive = [sysroot.all_files]) - toolchain = platform_common.ToolchainInfo( - all_files = all_files, + all_files = sysroot.all_files, rustc = sysroot.rustc, rust_doc = sysroot.rustdoc, rustfmt = sysroot.rustfmt, From e15e87c9008399bbb15b31ce34977de24435de65 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 7 Feb 2022 09:35:37 -0800 Subject: [PATCH 05/11] Don't use leading `/` --- rust/private/toolchain_utils.bzl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rust/private/toolchain_utils.bzl b/rust/private/toolchain_utils.bzl index 2aabb80e8c..dab6ec43c8 100644 --- a/rust/private/toolchain_utils.bzl +++ b/rust/private/toolchain_utils.bzl @@ -34,13 +34,13 @@ def _symlink_sysroot_tree(ctx, name, target): return depset(tree_files) -def _symlink_sysroot_bin(ctx, name, dir, target): +def _symlink_sysroot_bin(ctx, name, directory, target): """Crete a symlink to a target file. Args: ctx (ctx): The rule's context object name (str): A common name for the output directory - dir (str): The directory under `name` to put the file in + directory (str): The directory under `name` to put the file in target (File): A File object to symlink to Returns: @@ -48,7 +48,7 @@ def _symlink_sysroot_bin(ctx, name, dir, target): """ symlink = ctx.actions.declare_file("{}/{}/{}".format( name, - dir.lstrip("/"), + directory.lstrip("/"), target.basename, )) @@ -93,7 +93,7 @@ def generate_sysroot( transitive_file_sets = [] # Rustc - sysroot_rustc = _symlink_sysroot_bin(ctx, name, "/bin", rustc) + sysroot_rustc = _symlink_sysroot_bin(ctx, name, "bin", rustc) direct_files.extend([sysroot_rustc, rustc]) # Rustc dependencies @@ -102,21 +102,21 @@ def generate_sysroot( transitive_file_sets.extend([sysroot_rustc_lib, rustc_lib.files]) # Rustdoc - sysroot_rustdoc = _symlink_sysroot_bin(ctx, name, "/bin", rustdoc) + sysroot_rustdoc = _symlink_sysroot_bin(ctx, name, "bin", rustdoc) direct_files.extend([sysroot_rustdoc, rustdoc]) # Clippy - sysroot_clippy = _symlink_sysroot_bin(ctx, name, "/bin", clippy) if clippy else None + sysroot_clippy = _symlink_sysroot_bin(ctx, name, "bin", clippy) if clippy else None if sysroot_clippy: direct_files.extend([sysroot_clippy, clippy]) # Cargo - sysroot_cargo = _symlink_sysroot_bin(ctx, name, "/bin", cargo) if cargo else None + sysroot_cargo = _symlink_sysroot_bin(ctx, name, "bin", cargo) if cargo else None if sysroot_cargo: direct_files.extend([sysroot_cargo, cargo]) # Rustfmt - sysroot_rustfmt = _symlink_sysroot_bin(ctx, name, "/bin", rustfmt) if rustfmt else None + sysroot_rustfmt = _symlink_sysroot_bin(ctx, name, "bin", rustfmt) if rustfmt else None if sysroot_rustfmt: direct_files.extend([sysroot_rustfmt, rustfmt]) From 5657319900f32ad09a5d4d23906b4e75ba4e9403 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 7 Feb 2022 09:35:59 -0800 Subject: [PATCH 06/11] Regenerate documentation --- docs/flatten.md | 2 +- docs/rust_repositories.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/flatten.md b/docs/flatten.md index b085a8f2df..0cee286bc1 100644 --- a/docs/flatten.md +++ b/docs/flatten.md @@ -1170,7 +1170,7 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r | default_edition | The edition to use for rust_* rules that don't specify an edition. | String | optional | "2018" | | dylib_ext | The extension for dynamic libraries created from rustc. | String | required | | | exec_triple | The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations | String | required | | -| llvm_tools | llvm-tools | List of labels | optional | [] | +| llvm_tools | LLVM tools that are shipped with the Rust toolchain. | List of labels | optional | [] | | opt_level | Rustc optimization levels. | Dictionary: String -> String | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} | | os | The operating system for the current toolchain | String | required | | | rust_doc | The location of the rustdoc binary. Can be a direct source or a filegroup containing one item. | Label | required | | diff --git a/docs/rust_repositories.md b/docs/rust_repositories.md index a98729d149..a5f4e9ac8e 100644 --- a/docs/rust_repositories.md +++ b/docs/rust_repositories.md @@ -95,7 +95,7 @@ See @rules_rust//rust:repositories.bzl for examples of defining the @rust_cpuX r | default_edition | The edition to use for rust_* rules that don't specify an edition. | String | optional | "2018" | | dylib_ext | The extension for dynamic libraries created from rustc. | String | required | | | exec_triple | The platform triple for the toolchains execution environment. For more details see: https://docs.bazel.build/versions/master/skylark/rules.html#configurations | String | required | | -| llvm_tools | llvm-tools | List of labels | optional | [] | +| llvm_tools | LLVM tools that are shipped with the Rust toolchain. | List of labels | optional | [] | | opt_level | Rustc optimization levels. | Dictionary: String -> String | optional | {"dbg": "0", "fastbuild": "0", "opt": "3"} | | os | The operating system for the current toolchain | String | required | | | rust_doc | The location of the rustdoc binary. Can be a direct source or a filegroup containing one item. | Label | required | | From 171ec8d66601426eacb6c6466d8ef52c03796c98 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 7 Feb 2022 09:41:44 -0800 Subject: [PATCH 07/11] no explicit `--sysroot` flag. --- rust/private/rustc.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 382a6f31bf..43559eca81 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -665,7 +665,6 @@ def construct_arguments( # Ensure the sysroot is set for the target platform env["SYSROOT"] = toolchain.sysroot - rustc_flags.add("--sysroot", "${pwd}/" + toolchain.sysroot) # extra_rustc_flags apply to the target configuration, not the exec configuration. if hasattr(ctx.attr, "_extra_rustc_flags") and not is_exec_configuration(ctx): From ccb054837d3c6d4ed30b54b1a49b83d59f4ece01 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Mon, 7 Feb 2022 10:56:52 -0800 Subject: [PATCH 08/11] Better fix for `rust_doc_test` --- rust/private/rustdoc.bzl | 6 ++++-- rust/private/rustdoc_test.bzl | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 0258183e1a..987b4461c0 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -48,7 +48,8 @@ def rustdoc_compile_action( toolchain, crate_info, output = None, - rustdoc_flags = []): + rustdoc_flags = [], + is_test = False): """Create a struct of information needed for a `rustdoc` compile action based on crate passed to the rustdoc rule. Args: @@ -57,6 +58,7 @@ def rustdoc_compile_action( crate_info (CrateInfo): The provider of the crate passed to a rustdoc rule. output (File, optional): An optional output a `rustdoc` action is intended to produce. rustdoc_flags (list, optional): A list of `rustdoc` specific flags. + is_test (bool, optional): If True, the action will be configured for `rust_doc_test` targets Returns: struct: A struct of some `ctx.actions.run` arguments. @@ -102,7 +104,7 @@ def rustdoc_compile_action( attr = ctx.attr, file = ctx.file, toolchain = toolchain, - tool_path = toolchain.rust_doc.path, + tool_path = toolchain.rust_doc.short_path if is_test else toolchain.rust_doc.path, cc_toolchain = cc_toolchain, feature_configuration = feature_configuration, crate_info = rustdoc_crate_info, diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index 163c72e086..20e960dcfb 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -56,11 +56,6 @@ def _construct_writer_arguments(ctx, test_runner, action, crate_info, rust_toolc for var in action.env.keys(): writer_args.add("--action_env={}".format(var)) - # Ensure the rustdoc binary is always accessed via a relative path - writer_args.add("--strip_substring={}/".format( - rust_toolchain.rust_doc.root.path, - )) - # Since the test runner will be running from a runfiles directory, the # paths originally generated for the build action will not map to any # files. To ensure rustdoc can find the appropriate dependencies, the @@ -121,6 +116,7 @@ def _rust_doc_test_impl(ctx): toolchain = find_toolchain(ctx), crate_info = crate_info, rustdoc_flags = rustdoc_flags, + is_test = True, ) tools = action.tools + [ctx.executable._process_wrapper] From 97a142cb29d63a828e01f4380b2a3048bace3413 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Tue, 8 Feb 2022 08:17:41 -0800 Subject: [PATCH 09/11] Set `--sysroot` in for `rust_doc_test` --- rust/private/rustdoc.bzl | 11 +++++++++++ rust/toolchain.bzl | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index 987b4461c0..799a16b6f0 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -120,6 +120,17 @@ def rustdoc_compile_action( force_link = True, ) + # Because rustdoc tests compile tests outside of the sandbox, the sysroot + # must be updated to the `short_path` equivilant as it will now be + # a part of runfiles. + if is_test: + if "SYSROOT" in env: + env.update({"SYSROOT": "${{pwd}}/{}".format(toolchain.sysroot_short_path)}) + + # `rustdoc` does not support the SYSROOT environment variable. To account + # for this, the flag must be explicitly passed to the `rustdoc` binary. + args.rustc_flags.add("--sysroot=${{pwd}}/{}".format(toolchain.sysroot_short_path)) + return struct( executable = ctx.executable._process_wrapper, inputs = depset([crate_info.output], transitive = [compile_inputs]), diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index 9fc0ed28d1..3b5d16a7e9 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -278,6 +278,10 @@ def _rust_toolchain_impl(ctx): linking_context = linking_context, ) + # Determine the path and short_path of the sysroot + sysroot_path = sysroot.sysroot_anchor.dirname + sysroot_short_path, _, _ = sysroot.sysroot_anchor.short_path.rpartition("/") + toolchain = platform_common.ToolchainInfo( all_files = sysroot.all_files, rustc = sysroot.rustc, @@ -292,7 +296,8 @@ def _rust_toolchain_impl(ctx): rust_std = sysroot.rust_std, rust_std_paths = depset([file.dirname for file in sysroot.rust_std.to_list()]), rust_lib = sysroot.rust_std, # `rust_lib` is deprecated and only exists for legacy support. - sysroot = sysroot.sysroot_anchor.dirname, + sysroot = sysroot_path, + sysroot_short_path = sysroot_short_path, binary_ext = ctx.attr.binary_ext, staticlib_ext = ctx.attr.staticlib_ext, dylib_ext = ctx.attr.dylib_ext, From 3907fde0f016c24df0b98dd7089a0414fadaedb9 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 9 Feb 2022 07:58:34 -0800 Subject: [PATCH 10/11] Update rust/private/toolchain_utils.bzl Co-authored-by: Krasimir Georgiev --- rust/private/toolchain_utils.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/private/toolchain_utils.bzl b/rust/private/toolchain_utils.bzl index dab6ec43c8..d57be6f302 100644 --- a/rust/private/toolchain_utils.bzl +++ b/rust/private/toolchain_utils.bzl @@ -48,7 +48,7 @@ def _symlink_sysroot_bin(ctx, name, directory, target): """ symlink = ctx.actions.declare_file("{}/{}/{}".format( name, - directory.lstrip("/"), + directory, target.basename, )) From 69faa0caeeac9f9bf3f7f1323db0532137f80d74 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 9 Feb 2022 12:17:48 -0800 Subject: [PATCH 11/11] Moved `generate_sysroot` --- rust/private/toolchain_utils.bzl | 166 ------------------------------ rust/toolchain.bzl | 170 ++++++++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 168 deletions(-) diff --git a/rust/private/toolchain_utils.bzl b/rust/private/toolchain_utils.bzl index d57be6f302..fcd4fb7029 100644 --- a/rust/private/toolchain_utils.bzl +++ b/rust/private/toolchain_utils.bzl @@ -1,171 +1,5 @@ """A module defining toolchain utilities""" -def _symlink_sysroot_tree(ctx, name, target): - """Generate a set of symlinks to files from another target - - Args: - ctx (ctx): The toolchain's context object - name (str): The name of the sysroot directory (typically `ctx.label.name`) - target (Target): A target owning files to symlink - - Returns: - depset[File]: A depset of the generated symlink files - """ - tree_files = [] - for file in target.files.to_list(): - # Parse the path to the file relative to the workspace root so a - # symlink matching this path can be created within the sysroot. - - # The code blow attempts to parse any workspace names out of the - # path. For local targets, this code is a noop. - if target.label.workspace_root: - file_path = file.path.split(target.label.workspace_root, 1)[-1] - else: - file_path = file.path - - symlink = ctx.actions.declare_file("{}/{}".format(name, file_path.lstrip("/"))) - - ctx.actions.symlink( - output = symlink, - target_file = file, - ) - - tree_files.append(symlink) - - return depset(tree_files) - -def _symlink_sysroot_bin(ctx, name, directory, target): - """Crete a symlink to a target file. - - Args: - ctx (ctx): The rule's context object - name (str): A common name for the output directory - directory (str): The directory under `name` to put the file in - target (File): A File object to symlink to - - Returns: - File: A newly generated symlink file - """ - symlink = ctx.actions.declare_file("{}/{}/{}".format( - name, - directory, - target.basename, - )) - - ctx.actions.symlink( - output = symlink, - target_file = target, - is_executable = True, - ) - - return symlink - -def generate_sysroot( - ctx, - rustc, - rustdoc, - rustc_lib, - cargo = None, - clippy = None, - llvm_tools = None, - rust_std = None, - rustfmt = None): - """Generate a rust sysroot from an exec and target toolchain - - Args: - ctx (ctx): A context object from a `rust_toolchain` rule. - rustc (File): The path to a `rustc` executable. - rustdoc (File): The path to a `rustdoc` executable. - rustc_lib (Target): A collection of Files containing dependencies of `rustc`. - cargo (File, optional): The path to a `cargo` executable. - clippy (File, optional): The path to a `clippy-driver` executable. - llvm_tools (Target, optional): A collection of llvm tools used by `rustc`. - rust_std (Target, optional): A collection of Files containing Rust standard library components. - rustfmt (File, optional): The path to a `rustfmt` executable. - - Returns: - struct: A struct of generated files representing the new sysroot - """ - name = ctx.label.name - - # Define runfiles - direct_files = [] - transitive_file_sets = [] - - # Rustc - sysroot_rustc = _symlink_sysroot_bin(ctx, name, "bin", rustc) - direct_files.extend([sysroot_rustc, rustc]) - - # Rustc dependencies - sysroot_rustc_lib = _symlink_sysroot_tree(ctx, name, rustc_lib) if rustc_lib else None - if sysroot_rustc_lib: - transitive_file_sets.extend([sysroot_rustc_lib, rustc_lib.files]) - - # Rustdoc - sysroot_rustdoc = _symlink_sysroot_bin(ctx, name, "bin", rustdoc) - direct_files.extend([sysroot_rustdoc, rustdoc]) - - # Clippy - sysroot_clippy = _symlink_sysroot_bin(ctx, name, "bin", clippy) if clippy else None - if sysroot_clippy: - direct_files.extend([sysroot_clippy, clippy]) - - # Cargo - sysroot_cargo = _symlink_sysroot_bin(ctx, name, "bin", cargo) if cargo else None - if sysroot_cargo: - direct_files.extend([sysroot_cargo, cargo]) - - # Rustfmt - sysroot_rustfmt = _symlink_sysroot_bin(ctx, name, "bin", rustfmt) if rustfmt else None - if sysroot_rustfmt: - direct_files.extend([sysroot_rustfmt, rustfmt]) - - # Llvm tools - sysroot_llvm_tools = _symlink_sysroot_tree(ctx, name, llvm_tools) if llvm_tools else None - if sysroot_llvm_tools: - transitive_file_sets.extend([sysroot_llvm_tools, llvm_tools.files]) - - # Rust standard library - sysroot_rust_std = _symlink_sysroot_tree(ctx, name, rust_std) if rust_std else None - if sysroot_rust_std: - transitive_file_sets.extend([sysroot_rust_std, rust_std.files]) - - # Symlink rust-stdlib - sysroot_rust_std = _symlink_sysroot_tree(ctx, name, rust_std) if rust_std else None - if sysroot_rust_std: - transitive_file_sets.extend([sysroot_rust_std, rust_std.files]) - - # Declare a file in the root of the sysroot to make locating the sysroot easy - sysroot_anchor = ctx.actions.declare_file("{}/rust.sysroot".format(name)) - ctx.actions.write( - output = sysroot_anchor, - content = "\n".join([ - "cargo: {}".format(cargo), - "clippy: {}".format(clippy), - "llvm_tools: {}".format(llvm_tools), - "rust_std: {}".format(rust_std), - "rustc_lib: {}".format(rustc_lib), - "rustc: {}".format(rustc), - "rustdoc: {}".format(rustdoc), - "rustfmt: {}".format(rustfmt), - ]), - ) - - # Create a depset of all sysroot files (symlinks and their real paths) - all_files = depset(direct_files, transitive = transitive_file_sets) - - return struct( - all_files = all_files, - cargo = sysroot_cargo, - clippy = sysroot_clippy, - rust_std = sysroot_rust_std, - rustc = sysroot_rustc, - rustc_lib = sysroot_rustc_lib, - rustdoc = sysroot_rustdoc, - rustfmt = sysroot_rustfmt, - sysroot_anchor = sysroot_anchor, - ) - def _toolchain_files_impl(ctx): toolchain = ctx.toolchains[str(Label("//rust:toolchain"))] diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index 3b5d16a7e9..8577f3dad9 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -2,7 +2,6 @@ load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//rust/private:common.bzl", "rust_common") -load("//rust/private:toolchain_utils.bzl", "generate_sysroot") load("//rust/private:utils.bzl", "dedent", "find_cc_toolchain", "make_static_lib_symlink") def _rust_stdlib_filegroup_impl(ctx): @@ -211,6 +210,173 @@ def _make_libstd_and_allocator_ccinfo(ctx, rust_std, allocator_library): ) return None +def _symlink_sysroot_tree(ctx, name, target): + """Generate a set of symlinks to files from another target + + Args: + ctx (ctx): The toolchain's context object + name (str): The name of the sysroot directory (typically `ctx.label.name`) + target (Target): A target owning files to symlink + + Returns: + depset[File]: A depset of the generated symlink files + """ + tree_files = [] + for file in target.files.to_list(): + # Parse the path to the file relative to the workspace root so a + # symlink matching this path can be created within the sysroot. + + # The code blow attempts to parse any workspace names out of the + # path. For local targets, this code is a noop. + if target.label.workspace_root: + file_path = file.path.split(target.label.workspace_root, 1)[-1] + else: + file_path = file.path + + symlink = ctx.actions.declare_file("{}/{}".format(name, file_path.lstrip("/"))) + + ctx.actions.symlink( + output = symlink, + target_file = file, + ) + + tree_files.append(symlink) + + return depset(tree_files) + +def _symlink_sysroot_bin(ctx, name, directory, target): + """Crete a symlink to a target file. + + Args: + ctx (ctx): The rule's context object + name (str): A common name for the output directory + directory (str): The directory under `name` to put the file in + target (File): A File object to symlink to + + Returns: + File: A newly generated symlink file + """ + symlink = ctx.actions.declare_file("{}/{}/{}".format( + name, + directory, + target.basename, + )) + + ctx.actions.symlink( + output = symlink, + target_file = target, + is_executable = True, + ) + + return symlink + +def _generate_sysroot( + ctx, + rustc, + rustdoc, + rustc_lib, + cargo = None, + clippy = None, + llvm_tools = None, + rust_std = None, + rustfmt = None): + """Generate a rust sysroot from collection of toolchain components + + Args: + ctx (ctx): A context object from a `rust_toolchain` rule. + rustc (File): The path to a `rustc` executable. + rustdoc (File): The path to a `rustdoc` executable. + rustc_lib (Target): A collection of Files containing dependencies of `rustc`. + cargo (File, optional): The path to a `cargo` executable. + clippy (File, optional): The path to a `clippy-driver` executable. + llvm_tools (Target, optional): A collection of llvm tools used by `rustc`. + rust_std (Target, optional): A collection of Files containing Rust standard library components. + rustfmt (File, optional): The path to a `rustfmt` executable. + + Returns: + struct: A struct of generated files representing the new sysroot + """ + name = ctx.label.name + + # Define runfiles + direct_files = [] + transitive_file_sets = [] + + # Rustc + sysroot_rustc = _symlink_sysroot_bin(ctx, name, "bin", rustc) + direct_files.extend([sysroot_rustc, rustc]) + + # Rustc dependencies + sysroot_rustc_lib = None + if rustc_lib: + sysroot_rustc_lib = _symlink_sysroot_tree(ctx, name, rustc_lib) + transitive_file_sets.extend([sysroot_rustc_lib, rustc_lib.files]) + + # Rustdoc + sysroot_rustdoc = _symlink_sysroot_bin(ctx, name, "bin", rustdoc) + direct_files.extend([sysroot_rustdoc, rustdoc]) + + # Clippy + sysroot_clippy = None + if clippy: + sysroot_clippy = _symlink_sysroot_bin(ctx, name, "bin", clippy) + direct_files.extend([sysroot_clippy, clippy]) + + # Cargo + sysroot_cargo = None + if cargo: + sysroot_cargo = _symlink_sysroot_bin(ctx, name, "bin", cargo) + direct_files.extend([sysroot_cargo, cargo]) + + # Rustfmt + sysroot_rustfmt = None + if rustfmt: + sysroot_rustfmt = _symlink_sysroot_bin(ctx, name, "bin", rustfmt) + direct_files.extend([sysroot_rustfmt, rustfmt]) + + # Llvm tools + sysroot_llvm_tools = None + if llvm_tools: + sysroot_llvm_tools = _symlink_sysroot_tree(ctx, name, llvm_tools) + transitive_file_sets.extend([sysroot_llvm_tools, llvm_tools.files]) + + # Rust standard library + sysroot_rust_std = None + if rust_std: + sysroot_rust_std = _symlink_sysroot_tree(ctx, name, rust_std) + transitive_file_sets.extend([sysroot_rust_std, rust_std.files]) + + # Declare a file in the root of the sysroot to make locating the sysroot easy + sysroot_anchor = ctx.actions.declare_file("{}/rust.sysroot".format(name)) + ctx.actions.write( + output = sysroot_anchor, + content = "\n".join([ + "cargo: {}".format(cargo), + "clippy: {}".format(clippy), + "llvm_tools: {}".format(llvm_tools), + "rust_std: {}".format(rust_std), + "rustc_lib: {}".format(rustc_lib), + "rustc: {}".format(rustc), + "rustdoc: {}".format(rustdoc), + "rustfmt: {}".format(rustfmt), + ]), + ) + + # Create a depset of all sysroot files (symlinks and their real paths) + all_files = depset(direct_files, transitive = transitive_file_sets) + + return struct( + all_files = all_files, + cargo = sysroot_cargo, + clippy = sysroot_clippy, + rust_std = sysroot_rust_std, + rustc = sysroot_rustc, + rustc_lib = sysroot_rustc_lib, + rustdoc = sysroot_rustdoc, + rustfmt = sysroot_rustfmt, + sysroot_anchor = sysroot_anchor, + ) + def _rust_toolchain_impl(ctx): """The rust_toolchain implementation @@ -244,7 +410,7 @@ def _rust_toolchain_impl(ctx): else: rust_std = ctx.attr.rust_std - sysroot = generate_sysroot( + sysroot = _generate_sysroot( ctx = ctx, rustc = ctx.file.rustc, rustdoc = ctx.file.rust_doc,