From 1ce6901593ef23d7d1b7a776c7850235ec6c56f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Zwoli=C5=84ski?= Date: Sun, 6 Mar 2022 15:20:17 +0100 Subject: [PATCH] test-runner: Run tests isolated by default in headless mode Adds support for isolating test cases when running in headless mode Isolated tests are ran parallely instead of sequentially Hide most of the output of headless tests behind WASM_BINDGEN_DEBUG --- crates/cli/Cargo.toml | 2 +- .../bin/wasm-bindgen-test-runner/browser.rs | 151 ++++++++++++++++ .../bin/wasm-bindgen-test-runner/headless.rs | 170 ++++++++++++++---- .../index-headless.html | 2 +- .../src/bin/wasm-bindgen-test-runner/main.rs | 70 ++------ .../bin/wasm-bindgen-test-runner/server.rs | 26 ++- 6 files changed, 317 insertions(+), 104 deletions(-) create mode 100644 crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index ff8f3fdf16a..06966c0ef78 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -15,6 +15,7 @@ edition = '2018' default-run = 'wasm-bindgen' [dependencies] +rayon = "1.5" curl = "0.4.13" docopt = "1.0" env_logger = "0.8" @@ -33,7 +34,6 @@ wasm-bindgen-shared = { path = "../shared", version = "=0.2.79" } assert_cmd = "1.0" diff = "0.1" predicates = "1.0.0" -rayon = "1.0" tempfile = "3.0" wit-printer = "0.2" wit-text = "0.8" diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs new file mode 100644 index 00000000000..df0a58f6917 --- /dev/null +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs @@ -0,0 +1,151 @@ +use anyhow::Context; +use rayon::prelude::*; +use std::env; +use std::ffi::OsString; +use std::path::Path; +use std::sync::Arc; + +use crate::headless::{tab, TestResult}; + +pub fn execute( + shell: crate::shell::Shell, + module: &str, + tmpdir: &Path, + args: &[OsString], + tests: Vec, +) -> anyhow::Result<()> { + let isolated = env::var("NO_ISOLATED").is_err(); + let headless = env::var("NO_HEADLESS").is_err(); + let debug = env::var("WASM_BINDGEN_DEBUG").is_ok(); + let timeout = env::var("WASM_BINDGEN_TEST_TIMEOUT") + .map(|timeout| { + timeout + .parse() + .expect("Could not parse 'WASM_BINDGEN_TEST_TIMEOUT'") + }) + .unwrap_or(20); + if debug { + println!("Set timeout to {} seconds...", timeout); + } + let shell = Arc::new(shell); + + let addr = if headless { + "127.0.0.1:0" + } else { + "127.0.0.1:8000" + }; + let spawn_server = |tests| { + let srv = crate::server::spawn( + &addr.parse().unwrap(), + headless, + &module, + &tmpdir, + &args, + tests, + ) + .context("failed to spawn server")?; + let addr = srv.server_addr(); + Ok::<_, anyhow::Error>((srv, addr)) + }; + if headless && isolated && tests.len() > 1 { + // Here instead of spawning a server and driver running all tests + // we create a pair of server and driver per test and spawn them + // parallely. For parallel execution we use rayon so it can automatically + // take care of limiting the resources. After all tests finish, we aggregate + // their results and present them to the user. + println!("\nrunning {} tests", tests.len()); + let tests = tests.into_iter().map(|t| vec![t]).collect::>(); + let results = tests + .par_iter() + .map(|test| { + let (srv, addr) = spawn_server(test)?; + let (handle, tx) = srv.stoppable(); + let shell = shell.clone(); + let (test_outcome, _) = rayon::join( + move || { + let res = crate::headless::run_isolated(&addr, shell, timeout, debug); + tx.send(()).map_err(Into::into).and(res) + }, + move || handle.join(), + ); + Ok::<_, anyhow::Error>((test[0].as_str(), test_outcome?)) + }) + .collect::>(); + show_test_results(results)?; + } else if headless { + let (srv, addr) = spawn_server(&tests)?; + std::thread::spawn(|| srv.run()); + crate::headless::run(&addr, shell, timeout, debug)?; + } else { + let (srv, addr) = spawn_server(&tests)?; + + // TODO: eventually we should provide the ability to exit at some point + // (gracefully) here, but for now this just runs forever. + println!( + "Interactive browsers tests are now available at http://{}", + addr + ); + println!(""); + println!("Note that interactive mode is enabled because `NO_HEADLESS`"); + println!("is specified in the environment of this process. Once you're"); + println!("done with testing you'll need to kill this server with"); + println!("Ctrl-C."); + return Ok(srv.run()); + } + Ok(()) +} + +// Create a summary of ran tests counting passed, ignored and failed. +// Combine informations about all failed test cases and print it before the final +// summary. +fn show_test_results( + results: Vec>, +) -> Result<(), anyhow::Error> { + let mut passed = 0; + let mut failed = 0; + let mut ignored = 0; + let mut all_details = String::new(); + for result in results { + let (test_name, test_result) = result?; + match test_result { + TestResult::Passed => passed += 1, + TestResult::Ignored => ignored += 1, + TestResult::Failed { details } => { + failed += 1; + all_details += &details; + all_details.push('\n'); + } + TestResult::Timeouted { + output, + errors, + logs, + } => { + failed += 1; + all_details += &format!( + "Failed to detect {} as having been run. It might have timed out.\n", + test_name + ); + if !output.is_empty() { + all_details += "output div contained:\n"; + all_details += &tab(&output); + } + if !logs.is_empty() { + all_details += "logs div contained:\n"; + all_details += &tab(&logs); + } + if !errors.is_empty() { + all_details += "errors div contained:\n"; + all_details += &tab(&errors); + } + } + } + } + println!(); + if !all_details.is_empty() { + println!("failures:\n"); + println!("{all_details}"); + } + let result = if failed == 0 { "ok" } else { "FAILED" }; + println!("test result: {result}. {passed} passed; {failed} failed; {ignored} ignored"); + Ok(()) +} diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs index 89b5252c1c1..825921f6eb2 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs @@ -11,6 +11,7 @@ use std::io::{self, Read}; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; @@ -52,10 +53,15 @@ pub struct LegacyNewSessionParameters { /// address. /// /// This function will take care of everything from spawning the WebDriver -/// binary, controlling it, running tests, scraping output, displaying output, -/// etc. It will return `Ok` if all tests finish successfully, and otherwise it -/// will return an error if some tests failed. -pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error> { +/// binary, controlling it, running tests and returning output. +/// It will return `Ok` if whole process went seamlessly, regardless of what +/// tests result was. +pub fn execute_tests( + server: &SocketAddr, + shell: Arc, + timeout: u64, + debug: bool, +) -> Result<(String, String, String), Error> { let driver = Driver::find()?; let mut drop_log: Box = Box::new(|| ()); let driver_url = match driver.location() { @@ -71,7 +77,7 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error let mut cmd = Command::new(path); cmd.args(args) .arg(format!("--port={}", driver_addr.port().to_string())); - let mut child = BackgroundChild::spawn(&path, &mut cmd, shell)?; + let mut child = BackgroundChild::spawn(&path, &mut cmd, shell.clone())?; drop_log = Box::new(move || child.print_stdio_on_drop = false); // Wait for the driver to come online and bind its port before we try to @@ -92,25 +98,33 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error Url::parse(&format!("http://{}", driver_addr)).map_err(Error::from) } }?; - println!( - "Running headless tests in {} on `{}`", - driver.browser(), - driver_url.as_str(), - ); + if debug { + println!( + "Running headless tests in {} on `{}`", + driver.browser(), + driver_url.as_str(), + ) + }; let mut client = Client { handle: Easy::new(), driver_url, session: None, }; - println!("Try find `webdriver.json` for configure browser's capabilities:"); + if debug { + println!("Try find `webdriver.json` for configure browser's capabilities:"); + } let capabilities: Capabilities = match File::open("webdriver.json") { Ok(file) => { - println!("Ok"); + if debug { + println!("Ok"); + } serde_json::from_reader(file) } Err(_) => { - println!("Not found"); + if debug { + println!("Not found"); + } Ok(Capabilities::new()) } }?; @@ -148,6 +162,10 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error let max = Duration::new(timeout, 0); while start.elapsed() < max { if client.text(&id, &output)?.contains("test result: ") { + // If the tests harness finished (either successfully or unsuccessfully) + // then in theory all the info needed to debug the failure is in its own + // output, so we shouldn't need the driver logs to get printed. + drop_log(); break; } thread::sleep(Duration::from_millis(100)); @@ -162,31 +180,109 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error let logs = client.text(&id, &logs)?; let errors = client.text(&id, &errors)?; - if output.contains("test result: ") { - println!("{}", output); + Ok((output, logs, errors)) +} - // If the tests harness finished (either successfully or unsuccessfully) - // then in theory all the info needed to debug the failure is in its own - // output, so we shouldn't need the driver logs to get printed. - drop_log(); +/// Run all tests and present their results to user +pub fn run(server: &SocketAddr, shell: Arc, timeout: u64, debug: bool) -> Result<(), Error> { + let (output, logs, errors) = execute_tests(server, shell, timeout, debug)?; + + if output.contains("test result: ok") { + // All tests succeeded. + println!("{output}"); + } else if output.contains("test result") { + // At least one test has failed but whole process managed to finish by itself. + // We can still only show output as it aggregates errors and logs + println!("{output}"); + bail!("some tests failed") } else { - println!("Failed to detect test as having been run. It might have timed out."); - if output.len() > 0 { + // Testing process hasn't finished, so we most probably timeouted. + // In that case output may not contain all informations so we print + // everything we can. + println!("Failed to detect test as having been run. It might have timed out.\n"); + if !output.is_empty() { println!("output div contained:\n{}", tab(&output)); } + if !errors.is_empty() { + println!("errors div contained:\n{}", tab(&errors)); + } + if !logs.is_empty() { + println!("logs div contained:\n{}", tab(&logs)); + } + bail!("some tests failed") } - if logs.len() > 0 { - println!("console.log div contained:\n{}", tab(&logs)); - } - if errors.len() > 0 { - println!("console.log div contained:\n{}", tab(&errors)); + Ok(()) +} + +/// Possible outcomes of single test +#[derive(Clone, Debug, PartialEq)] +pub enum TestResult { + Passed, + Failed { + details: String, + }, + Ignored, + Timeouted { + output: String, + errors: String, + logs: String, + }, +} + +/// Run single isolated test and postprocess it's output +/// In a way that it can be combined with other results of +/// isolated tests +pub fn run_isolated( + server: &SocketAddr, + shell: Arc, + timeout: u64, + debug: bool, +) -> Result { + let (output, logs, errors) = execute_tests(server, shell, timeout, debug)?; + + if !output.contains("test result:") { + // Testing process hasn't finished, so we most probably timeouted. + return Ok(TestResult::Timeouted { + output, + logs, + errors, + }); } - if !output.contains("test result: ok") { - bail!("some tests failed") + // Isolated tests are running parallelly, so we want to return all info instead + // of printing it right now. Doing the latter could result in interleaving prints + // from multiple tests and lead to confusion. + // We can tho print the short outcome of the test right now. + for line in output.lines() { + if line.starts_with("test ") && (line.ends_with("... ok") || line.ends_with("... FAIL")) { + println!("{line}"); + } } - Ok(()) + if output.contains("test result: ok") { + if output.contains("1 ignored") { + // The test has been ignored + Ok(TestResult::Ignored) + } else { + // The test has succeeded. + Ok(TestResult::Passed) + } + } else { + // The test has failed. Whole testing process went good tho as we have the final + // summary. Test suite already did a job of providing the failure context, we + // only need to extract it. + let details = output + .lines() + // Skip until we find a line indicating the start of failure summary + .skip_while(|&line| line != "failures:") + // and also this line itself with the empty one after it. + .skip(2) + // Take all lines until we reach the ending summary of all failed tests. + .take_while(|&line| line != "failures:") + .fold(String::new(), |a, b| a + b + "\n"); + + Ok(TestResult::Failed { details }) + } } enum Driver { @@ -577,7 +673,7 @@ fn read(r: &mut R) -> io::Result> { Ok(dst) } -fn tab(s: &str) -> String { +pub fn tab(s: &str) -> String { let mut result = String::new(); for line in s.lines() { result.push_str(" "); @@ -587,20 +683,16 @@ fn tab(s: &str) -> String { return result; } -struct BackgroundChild<'a> { +struct BackgroundChild { child: Child, stdout: Option>>>, stderr: Option>>>, - shell: &'a Shell, + shell: Arc, print_stdio_on_drop: bool, } -impl<'a> BackgroundChild<'a> { - fn spawn( - path: &Path, - cmd: &mut Command, - shell: &'a Shell, - ) -> Result, Error> { +impl BackgroundChild { + fn spawn(path: &Path, cmd: &mut Command, shell: Arc) -> Result { cmd.stdout(Stdio::piped()) .stderr(Stdio::piped()) .stdin(Stdio::null()); @@ -622,7 +714,7 @@ impl<'a> BackgroundChild<'a> { } } -impl<'a> Drop for BackgroundChild<'a> { +impl Drop for BackgroundChild { fn drop(&mut self) { self.child.kill().unwrap(); let status = self.child.wait().unwrap(); diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html b/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html index a65c242fd58..99381736ca1 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html @@ -36,6 +36,6 @@ window.__wbg_test_invoke = f => f(); - + diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs index 95a82601702..b4a1fdf0c98 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs @@ -15,13 +15,13 @@ use anyhow::{anyhow, bail, Context}; use std::env; use std::fs; use std::path::PathBuf; -use std::thread; use wasm_bindgen_cli_support::Bindgen; // no need for jemalloc bloat in this binary (and we don't need speed) #[global_allocator] static ALLOC: std::alloc::System = std::alloc::System; +mod browser; mod deno; mod headless; mod node; @@ -68,14 +68,13 @@ fn main() -> anyhow::Result<()> { let wasm = fs::read(&wasm_file_to_test).context("failed to read wasm file")?; let mut wasm = walrus::Module::from_buffer(&wasm).context("failed to deserialize wasm module")?; - let mut tests = Vec::new(); - for export in wasm.exports.iter() { - if !export.name.starts_with("__wbgt_") { - continue; - } - tests.push(export.name.to_string()); - } + let tests = wasm + .exports + .iter() + .filter(|exp| exp.name.starts_with("__wbgt_")) + .map(|exp| exp.name.to_string()) + .collect::>(); // Right now there's a bug where if no tests are present then the // `wasm-bindgen-test` runtime support isn't linked in, so just bail out @@ -98,9 +97,6 @@ fn main() -> anyhow::Result<()> { None => TestMode::Node, }; - let headless = env::var("NO_HEADLESS").is_err(); - let debug = env::var("WASM_BINDGEN_NO_DEBUG").is_err(); - // Gracefully handle requests to execute only node or only web tests. let node = test_mode == TestMode::Node; @@ -131,18 +127,6 @@ integration test.\ } } - let timeout = env::var("WASM_BINDGEN_TEST_TIMEOUT") - .map(|timeout| { - timeout - .parse() - .expect("Could not parse 'WASM_BINDGEN_TEST_TIMEOUT'") - }) - .unwrap_or(20); - - if debug { - println!("Set timeout to {} seconds...", timeout); - } - // Make the generated bindings available for the tests to execute against. shell.status("Executing bindgen..."); let mut b = Bindgen::new(); @@ -152,6 +136,7 @@ integration test.\ TestMode::Browser => b.web(true)?, }; + let debug = env::var("WASM_BINDGEN_DEBUG").is_ok(); b.debug(debug) .input_module(module, wasm) .keep_debug(false) @@ -163,42 +148,9 @@ integration test.\ let args: Vec<_> = args.collect(); match test_mode { - TestMode::Node => node::execute(&module, &tmpdir, &args, &tests)?, - TestMode::Deno => deno::execute(&module, &tmpdir, &args, &tests)?, - TestMode::Browser => { - let srv = server::spawn( - &if headless { - "127.0.0.1:0".parse().unwrap() - } else { - "127.0.0.1:8000".parse().unwrap() - }, - headless, - &module, - &tmpdir, - &args, - &tests, - ) - .context("failed to spawn server")?; - let addr = srv.server_addr(); - - // TODO: eventually we should provide the ability to exit at some point - // (gracefully) here, but for now this just runs forever. - if !headless { - println!( - "Interactive browsers tests are now available at http://{}", - addr - ); - println!(""); - println!("Note that interactive mode is enabled because `NO_HEADLESS`"); - println!("is specified in the environment of this process. Once you're"); - println!("done with testing you'll need to kill this server with"); - println!("Ctrl-C."); - return Ok(srv.run()); - } - - thread::spawn(|| srv.run()); - headless::run(&addr, &shell, timeout)?; - } + TestMode::Node => node::execute(module, &tmpdir, &args, &tests)?, + TestMode::Deno => deno::execute(module, &tmpdir, &args, &tests)?, + TestMode::Browser => browser::execute(shell, module, &tmpdir, &args, tests)?, } Ok(()) } diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs index 631d9ec2f8d..f40a7f12b54 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs @@ -1,5 +1,6 @@ use std::ffi::OsString; use std::fs; +use std::hash::{Hash, Hasher}; use std::net::SocketAddr; use std::path::Path; @@ -59,11 +60,18 @@ pub fn spawn( } js_to_execute.push_str("main(tests);\n"); - let js_path = tmpdir.join("run.js"); + // As headless tests may be executed parallely, we need to ensure + // that js script for each of them has unique name + let script_name = create_script_name(tests); + let js_path = tmpdir.join(&script_name); fs::write(&js_path, js_to_execute).context("failed to write JS file")?; - // For now, always run forever on this port. We may update this later! let tmpdir = tmpdir.to_path_buf(); + // And here we make sure that correct script is served by this instance + let index_headless = + include_str!("index-headless.html").replace("%%SCRIPT_NAME%%", &script_name); + let index_regular = include_str!("index.html"); + // For now, always run forever on this port. We may update this later! let srv = Server::new(addr, move |request| { // The root path gets our canned `index.html`. The two templates here // differ slightly in the default routing of `console.log`, going to an @@ -71,9 +79,9 @@ pub fn spawn( // output. if request.url() == "/" { let s = if headless { - include_str!("index-headless.html") + &index_headless } else { - include_str!("index.html") + index_regular }; return Response::from_data("text/html", s); } @@ -123,3 +131,13 @@ pub fn spawn( response } } + +// If we are running multiple tests parallely, we need to ensure +// that each script will have a unique name. Hashing the tests slice +// seems to be the easiest sollution that's kinda safe. Joining test +// names could hit max path length easily when there are a lot of them. +fn create_script_name(tests: &[String]) -> String { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + tests.hash(&mut hasher); + format!("run_{:x}.js", hasher.finish()) +}