Auto merge of rust-lang#127760 - jieyouxu:rmake-support-reorganize, r=Kobzol

Reorganize the `run-make-support` library

The `run_make_support` library has a kitchen sink `lib.rs` that make discovery/learning very difficult. Let's try to improve that by breaking up `lib.rs` into smaller more organized modules. This is a precursor to improving the documentation and learnability of the `run_make_support` library.

### Changes

- Breakup `lib.rs` into smaller modules according to functionality
- Rename `recursive_diff` -> `assert_dirs_are_equal`
- Rename one of the `read_dir` with callback interface as `read_dir_entries`
- Coalesced fs-related stuff onto a `fs` module, re-exported to tests as `rfs`
- Minor doc improvements / fixes in a few places (I have a follow-up documentation PR planned)

This PR is best reviewed commit-by-commit.

r? `@Kobzol` (or Mark, or T-compiler or T-bootstrap)

try-job: x86_64-msvc
try-job: aarch64-apple
try-job: test-various
try-job: armhf-gnu
try-job: dist-x86_64-linux

Author: bors

src/tools/run-make-support/src/artifact_names.rs

@@ -0,0 +1,81 @@
+//! A collection of helpers to construct artifact names, such as names of dynamic or static
+//! librarys which are target-dependent.
+
+use crate::targets::{is_darwin, is_msvc, is_windows};
+
+/// Construct the static library name based on the target.
+#[must_use]
+pub fn static_lib_name(name: &str) -> String {
+    // See tools.mk (irrelevant lines omitted):
+    //
+    // ```makefile
+    // ifeq ($(UNAME),Darwin)
+    //     STATICLIB = $(TMPDIR)/lib$(1).a
+    // else
+    //     ifdef IS_WINDOWS
+    //         ifdef IS_MSVC
+    //             STATICLIB = $(TMPDIR)/$(1).lib
+    //         else
+    //             STATICLIB = $(TMPDIR)/lib$(1).a
+    //         endif
+    //     else
+    //         STATICLIB = $(TMPDIR)/lib$(1).a
+    //     endif
+    // endif
+    // ```
+    assert!(!name.contains(char::is_whitespace), "static library name cannot contain whitespace");
+
+    if is_msvc() { format!("{name}.lib") } else { format!("lib{name}.a") }
+}
+
+/// Construct the dynamic library name based on the target.
+#[must_use]
+pub fn dynamic_lib_name(name: &str) -> String {
+    // See tools.mk (irrelevant lines omitted):
+    //
+    // ```makefile
+    // ifeq ($(UNAME),Darwin)
+    //     DYLIB = $(TMPDIR)/lib$(1).dylib
+    // else
+    //     ifdef IS_WINDOWS
+    //         DYLIB = $(TMPDIR)/$(1).dll
+    //     else
+    //         DYLIB = $(TMPDIR)/lib$(1).so
+    //     endif
+    // endif
+    // ```
+    assert!(!name.contains(char::is_whitespace), "dynamic library name cannot contain whitespace");
+
+    let extension = dynamic_lib_extension();
+    if is_darwin() {
+        format!("lib{name}.{extension}")
+    } else if is_windows() {
+        format!("{name}.{extension}")
+    } else {
+        format!("lib{name}.{extension}")
+    }
+}
+
+/// Construct the dynamic library extension based on the target.
+#[must_use]
+pub fn dynamic_lib_extension() -> &'static str {
+    if is_darwin() {
+        "dylib"
+    } else if is_windows() {
+        "dll"
+    } else {
+        "so"
+    }
+}
+
+/// Construct the name of a rust library (rlib).
+#[must_use]
+pub fn rust_lib_name(name: &str) -> String {
+    format!("lib{name}.rlib")
+}
+
+/// Construct the binary (executable) name based on the target.
+#[must_use]
+pub fn bin_name(name: &str) -> String {
+    if is_windows() { format!("{name}.exe") } else { name.to_string() }
+}

src/tools/run-make-support/src/assertion_helpers.rs

@@ -0,0 +1,73 @@
+//! Collection of assertions and assertion-related helpers.
+
+use std::panic;
+use std::path::Path;
+
+use crate::fs;
+
+/// Assert that `actual` is equal to `expected`.
+#[track_caller]
+pub fn assert_equals<A: AsRef<str>, E: AsRef<str>>(actual: A, expected: E) {
+    let actual = actual.as_ref();
+    let expected = expected.as_ref();
+    if actual != expected {
+        eprintln!("=== ACTUAL TEXT ===");
+        eprintln!("{}", actual);
+        eprintln!("=== EXPECTED ===");
+        eprintln!("{}", expected);
+        panic!("expected text was not found in actual text");
+    }
+}
+
+/// Assert that `haystack` contains `needle`.
+#[track_caller]
+pub fn assert_contains<H: AsRef<str>, N: AsRef<str>>(haystack: H, needle: N) {
+    let haystack = haystack.as_ref();
+    let needle = needle.as_ref();
+    if !haystack.contains(needle) {
+        eprintln!("=== HAYSTACK ===");
+        eprintln!("{}", haystack);
+        eprintln!("=== NEEDLE ===");
+        eprintln!("{}", needle);
+        panic!("needle was not found in haystack");
+    }
+}
+
+/// Assert that `haystack` does not contain `needle`.
+#[track_caller]
+pub fn assert_not_contains<H: AsRef<str>, N: AsRef<str>>(haystack: H, needle: N) {
+    let haystack = haystack.as_ref();
+    let needle = needle.as_ref();
+    if haystack.contains(needle) {
+        eprintln!("=== HAYSTACK ===");
+        eprintln!("{}", haystack);
+        eprintln!("=== NEEDLE ===");
+        eprintln!("{}", needle);
+        panic!("needle was unexpectedly found in haystack");
+    }
+}
+
+/// Assert that all files in `dir1` exist and have the same content in `dir2`
+pub fn assert_dirs_are_equal(dir1: impl AsRef<Path>, dir2: impl AsRef<Path>) {
+    let dir2 = dir2.as_ref();
+    fs::read_dir_entries(dir1, |entry_path| {
+        let entry_name = entry_path.file_name().unwrap();
+        if entry_path.is_dir() {
+            assert_dirs_are_equal(&entry_path, &dir2.join(entry_name));
+        } else {
+            let path2 = dir2.join(entry_name);
+            let file1 = fs::read(&entry_path);
+            let file2 = fs::read(&path2);
+
+            // We don't use `assert_eq!` because they are `Vec<u8>`, so not great for display.
+            // Why not using String? Because there might be minified files or even potentially
+            // binary ones, so that would display useless output.
+            assert!(
+                file1 == file2,
+                "`{}` and `{}` have different content",
+                entry_path.display(),
+                path2.display(),
+            );
+        }
+    });
+}

src/tools/run-make-support/src/command.rs

@@ -5,7 +5,9 @@ use std::panic;
 use std::path::Path;
 use std::process::{Command as StdCommand, ExitStatus, Output, Stdio};
 
-use crate::{assert_contains, assert_equals, assert_not_contains, handle_failed_output};
+use crate::util::handle_failed_output;
+use crate::{assert_contains, assert_equals, assert_not_contains};
+
 use build_helper::drop_bomb::DropBomb;
 
 /// This is a custom command wrapper that simplifies working with commands and makes it easier to

src/tools/run-make-support/src/diff/mod.rs

@@ -1,10 +1,12 @@
+use std::path::{Path, PathBuf};
+
 use regex::Regex;
 use similar::TextDiff;
-use std::path::{Path, PathBuf};
 
-use crate::fs_wrapper;
 use build_helper::drop_bomb::DropBomb;
 
+use crate::fs;
+
 #[cfg(test)]
 mod tests;
 
@@ -43,7 +45,7 @@ impl Diff {
     /// Specify the expected output for the diff from a file.
     pub fn expected_file<P: AsRef<Path>>(&mut self, path: P) -> &mut Self {
         let path = path.as_ref();
-        let content = fs_wrapper::read_to_string(path);
+        let content = fs::read_to_string(path);
         let name = path.to_string_lossy().to_string();
 
         self.expected_file = Some(path.into());
@@ -62,7 +64,7 @@ impl Diff {
     /// Specify the actual output for the diff from a file.
     pub fn actual_file<P: AsRef<Path>>(&mut self, path: P) -> &mut Self {
         let path = path.as_ref();
-        let content = fs_wrapper::read_to_string(path);
+        let content = fs::read_to_string(path);
         let name = path.to_string_lossy().to_string();
 
         self.actual = Some(content);
@@ -116,7 +118,7 @@ impl Diff {
             if let Some(ref expected_file) = self.expected_file {
                 if std::env::var("RUSTC_BLESS_TEST").is_ok() {
                     println!("Blessing `{}`", expected_file.display());
-                    fs_wrapper::write(expected_file, actual);
+                    fs::write(expected_file, actual);
                     return;
                 }
             }
@@ -138,7 +140,7 @@ impl Diff {
             if let Some(ref expected_file) = self.expected_file {
                 if std::env::var("RUSTC_BLESS_TEST").is_ok() {
                     println!("Blessing `{}`", expected_file.display());
-                    fs_wrapper::write(expected_file, actual);
+                    fs::write(expected_file, actual);
                     return;
                 }
             }

src/tools/run-make-support/src/env.rs

@@ -0,0 +1,19 @@
+use std::ffi::OsString;
+
+#[track_caller]
+#[must_use]
+pub fn env_var(name: &str) -> String {
+    match std::env::var(name) {
+        Ok(v) => v,
+        Err(err) => panic!("failed to retrieve environment variable {name:?}: {err:?}"),
+    }
+}
+
+#[track_caller]
+#[must_use]
+pub fn env_var_os(name: &str) -> OsString {
+    match std::env::var_os(name) {
+        Some(v) => v,
+        None => panic!("failed to retrieve environment variable {name:?}"),
+    }
+}

src/tools/run-make-support/src/external_deps/c_build.rs

@@ -0,0 +1,27 @@
+use std::path::PathBuf;
+
+use crate::artifact_names::static_lib_name;
+use crate::external_deps::cc::cc;
+use crate::external_deps::llvm::llvm_ar;
+use crate::path_helpers::path;
+use crate::targets::is_msvc;
+
+/// Builds a static lib (`.lib` on Windows MSVC and `.a` for the rest) with the given name.
+#[track_caller]
+pub fn build_native_static_lib(lib_name: &str) -> PathBuf {
+    let obj_file = if is_msvc() { format!("{lib_name}") } else { format!("{lib_name}.o") };
+    let src = format!("{lib_name}.c");
+    let lib_path = static_lib_name(lib_name);
+    if is_msvc() {
+        cc().arg("-c").out_exe(&obj_file).input(src).run();
+    } else {
+        cc().arg("-v").arg("-c").out_exe(&obj_file).input(src).run();
+    };
+    let obj_file = if is_msvc() {
+        PathBuf::from(format!("{lib_name}.obj"))
+    } else {
+        PathBuf::from(format!("{lib_name}.o"))
+    };
+    llvm_ar().obj_to_ar().output_input(&lib_path, &obj_file).run();
+    path(lib_path)
+}

src/tools/run-make-support/src/cc.rs renamed to src/tools/run-make-support/src/external_deps/cc.rs

@@ -1,7 +1,10 @@
 use std::path::Path;
 
 use crate::command::Command;
-use crate::{cygpath_windows, env_var, is_msvc, is_windows, uname};
+use crate::{env_var, is_msvc, is_windows, uname};
+
+// FIXME(jieyouxu): can we get rid of the `cygpath` external dependency?
+use super::cygpath::get_windows_path;
 
 /// Construct a new platform-specific C compiler invocation.
 ///
@@ -20,7 +23,7 @@ pub struct Cc {
     cmd: Command,
 }
 
-crate::impl_common_helpers!(Cc);
+crate::macros::impl_common_helpers!(Cc);
 
 impl Cc {
     /// Construct a new platform-specific C compiler invocation.
@@ -72,10 +75,10 @@ impl Cc {
 
         if is_msvc() {
             path.set_extension("exe");
-            let fe_path = cygpath_windows(&path);
+            let fe_path = get_windows_path(&path);
             path.set_extension("");
             path.set_extension("obj");
-            let fo_path = cygpath_windows(path);
+            let fo_path = get_windows_path(path);
             self.cmd.arg(format!("-Fe:{fe_path}"));
             self.cmd.arg(format!("-Fo:{fo_path}"));
         } else {

src/tools/run-make-support/src/clang.rs renamed to src/tools/run-make-support/src/external_deps/clang.rs

@@ -16,7 +16,7 @@ pub struct Clang {
     cmd: Command,
 }
 
-crate::impl_common_helpers!(Clang);
+crate::macros::impl_common_helpers!(Clang);
 
 impl Clang {
     /// Construct a new `clang` invocation. `clang` is not always available for all targets.

src/tools/run-make-support/src/external_deps/cygpath.rs

@@ -0,0 +1,35 @@
+use std::panic;
+use std::path::Path;
+
+use crate::command::Command;
+use crate::util::handle_failed_output;
+
+/// Use `cygpath -w` on a path to get a Windows path string back. This assumes that `cygpath` is
+/// available on the platform!
+///
+/// # FIXME
+///
+/// FIXME(jieyouxu): we should consider not depending on `cygpath`.
+///
+/// > The cygpath program is a utility that converts Windows native filenames to Cygwin POSIX-style
+/// > pathnames and vice versa.
+/// >
+/// > [irrelevant entries omitted...]
+/// >
+/// > `-w, --windows         print Windows form of NAMEs (C:\WINNT)`
+/// >
+/// > -- *from [cygpath documentation](https://cygwin.com/cygwin-ug-net/cygpath.html)*.
+#[track_caller]
+#[must_use]
+pub fn get_windows_path<P: AsRef<Path>>(path: P) -> String {
+    let caller = panic::Location::caller();
+    let mut cygpath = Command::new("cygpath");
+    cygpath.arg("-w");
+    cygpath.arg(path.as_ref());
+    let output = cygpath.run();
+    if !output.status().success() {
+        handle_failed_output(&cygpath, output, caller.line());
+    }
+    // cygpath -w can attach a newline
+    output.stdout_utf8().trim().to_string()
+}

src/tools/run-make-support/src/external_deps/htmldocck.rs

@@ -0,0 +1,14 @@
+use crate::command::Command;
+use crate::source_root;
+
+use super::python::python_command;
+
+/// `htmldocck` is a python script which is used for rustdoc test suites, it is assumed to be
+/// available at `$SOURCE_ROOT/src/etc/htmldocck.py`.
+#[track_caller]
+#[must_use]
+pub fn htmldocck() -> Command {
+    let mut python = python_command();
+    python.arg(source_root().join("src/etc/htmldocck.py"));
+    python
+}

src/tools/run-make-support/src/llvm.rs renamed to src/tools/run-make-support/src/external_deps/llvm.rs

@@ -1,6 +1,7 @@
 use std::path::{Path, PathBuf};
 
-use crate::{env_var, Command};
+use crate::command::Command;
+use crate::env::env_var;
 
 /// Construct a new `llvm-readobj` invocation with the `GNU` output style.
 /// This assumes that `llvm-readobj` is available at `$LLVM_BIN_DIR/llvm-readobj`.
@@ -70,11 +71,11 @@ pub struct LlvmAr {
     cmd: Command,
 }
 
-crate::impl_common_helpers!(LlvmReadobj);
-crate::impl_common_helpers!(LlvmProfdata);
-crate::impl_common_helpers!(LlvmFilecheck);
-crate::impl_common_helpers!(LlvmObjdump);
-crate::impl_common_helpers!(LlvmAr);
+crate::macros::impl_common_helpers!(LlvmReadobj);
+crate::macros::impl_common_helpers!(LlvmProfdata);
+crate::macros::impl_common_helpers!(LlvmFilecheck);
+crate::macros::impl_common_helpers!(LlvmObjdump);
+crate::macros::impl_common_helpers!(LlvmAr);
 
 /// Generate the path to the bin directory of LLVM.
 #[must_use]