diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index ff8f3fdf16a..06966c0ef78 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -15,6 +15,7 @@ edition = '2018' default-run = 'wasm-bindgen' [dependencies] +rayon = "1.5" curl = "0.4.13" docopt = "1.0" env_logger = "0.8" @@ -33,7 +34,6 @@ wasm-bindgen-shared = { path = "../shared", version = "=0.2.79" } assert_cmd = "1.0" diff = "0.1" predicates = "1.0.0" -rayon = "1.0" tempfile = "3.0" wit-printer = "0.2" wit-text = "0.8" diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs new file mode 100644 index 00000000000..df0a58f6917 --- /dev/null +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/browser.rs @@ -0,0 +1,151 @@ +use anyhow::Context; +use rayon::prelude::*; +use std::env; +use std::ffi::OsString; +use std::path::Path; +use std::sync::Arc; + +use crate::headless::{tab, TestResult}; + +pub fn execute( + shell: crate::shell::Shell, + module: &str, + tmpdir: &Path, + args: &[OsString], + tests: Vec, +) -> anyhow::Result<()> { + let isolated = env::var("NO_ISOLATED").is_err(); + let headless = env::var("NO_HEADLESS").is_err(); + let debug = env::var("WASM_BINDGEN_DEBUG").is_ok(); + let timeout = env::var("WASM_BINDGEN_TEST_TIMEOUT") + .map(|timeout| { + timeout + .parse() + .expect("Could not parse 'WASM_BINDGEN_TEST_TIMEOUT'") + }) + .unwrap_or(20); + if debug { + println!("Set timeout to {} seconds...", timeout); + } + let shell = Arc::new(shell); + + let addr = if headless { + "127.0.0.1:0" + } else { + "127.0.0.1:8000" + }; + let spawn_server = |tests| { + let srv = crate::server::spawn( + &addr.parse().unwrap(), + headless, + &module, + &tmpdir, + &args, + tests, + ) + .context("failed to spawn server")?; + let addr = srv.server_addr(); + Ok::<_, anyhow::Error>((srv, addr)) + }; + if headless && isolated && tests.len() > 1 { + // Here instead of spawning a server and driver running all tests + // we create a pair of server and driver per test and spawn them + // parallely. For parallel execution we use rayon so it can automatically + // take care of limiting the resources. After all tests finish, we aggregate + // their results and present them to the user. + println!("\nrunning {} tests", tests.len()); + let tests = tests.into_iter().map(|t| vec![t]).collect::>(); + let results = tests + .par_iter() + .map(|test| { + let (srv, addr) = spawn_server(test)?; + let (handle, tx) = srv.stoppable(); + let shell = shell.clone(); + let (test_outcome, _) = rayon::join( + move || { + let res = crate::headless::run_isolated(&addr, shell, timeout, debug); + tx.send(()).map_err(Into::into).and(res) + }, + move || handle.join(), + ); + Ok::<_, anyhow::Error>((test[0].as_str(), test_outcome?)) + }) + .collect::>(); + show_test_results(results)?; + } else if headless { + let (srv, addr) = spawn_server(&tests)?; + std::thread::spawn(|| srv.run()); + crate::headless::run(&addr, shell, timeout, debug)?; + } else { + let (srv, addr) = spawn_server(&tests)?; + + // TODO: eventually we should provide the ability to exit at some point + // (gracefully) here, but for now this just runs forever. + println!( + "Interactive browsers tests are now available at http://{}", + addr + ); + println!(""); + println!("Note that interactive mode is enabled because `NO_HEADLESS`"); + println!("is specified in the environment of this process. Once you're"); + println!("done with testing you'll need to kill this server with"); + println!("Ctrl-C."); + return Ok(srv.run()); + } + Ok(()) +} + +// Create a summary of ran tests counting passed, ignored and failed. +// Combine informations about all failed test cases and print it before the final +// summary. +fn show_test_results( + results: Vec>, +) -> Result<(), anyhow::Error> { + let mut passed = 0; + let mut failed = 0; + let mut ignored = 0; + let mut all_details = String::new(); + for result in results { + let (test_name, test_result) = result?; + match test_result { + TestResult::Passed => passed += 1, + TestResult::Ignored => ignored += 1, + TestResult::Failed { details } => { + failed += 1; + all_details += &details; + all_details.push('\n'); + } + TestResult::Timeouted { + output, + errors, + logs, + } => { + failed += 1; + all_details += &format!( + "Failed to detect {} as having been run. It might have timed out.\n", + test_name + ); + if !output.is_empty() { + all_details += "output div contained:\n"; + all_details += &tab(&output); + } + if !logs.is_empty() { + all_details += "logs div contained:\n"; + all_details += &tab(&logs); + } + if !errors.is_empty() { + all_details += "errors div contained:\n"; + all_details += &tab(&errors); + } + } + } + } + println!(); + if !all_details.is_empty() { + println!("failures:\n"); + println!("{all_details}"); + } + let result = if failed == 0 { "ok" } else { "FAILED" }; + println!("test result: {result}. {passed} passed; {failed} failed; {ignored} ignored"); + Ok(()) +} diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs index 89b5252c1c1..825921f6eb2 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs @@ -11,6 +11,7 @@ use std::io::{self, Read}; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; use std::thread; use std::time::{Duration, Instant}; @@ -52,10 +53,15 @@ pub struct LegacyNewSessionParameters { /// address. /// /// This function will take care of everything from spawning the WebDriver -/// binary, controlling it, running tests, scraping output, displaying output, -/// etc. It will return `Ok` if all tests finish successfully, and otherwise it -/// will return an error if some tests failed. -pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error> { +/// binary, controlling it, running tests and returning output. +/// It will return `Ok` if whole process went seamlessly, regardless of what +/// tests result was. +pub fn execute_tests( + server: &SocketAddr, + shell: Arc, + timeout: u64, + debug: bool, +) -> Result<(String, String, String), Error> { let driver = Driver::find()?; let mut drop_log: Box = Box::new(|| ()); let driver_url = match driver.location() { @@ -71,7 +77,7 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error let mut cmd = Command::new(path); cmd.args(args) .arg(format!("--port={}", driver_addr.port().to_string())); - let mut child = BackgroundChild::spawn(&path, &mut cmd, shell)?; + let mut child = BackgroundChild::spawn(&path, &mut cmd, shell.clone())?; drop_log = Box::new(move || child.print_stdio_on_drop = false); // Wait for the driver to come online and bind its port before we try to @@ -92,25 +98,33 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error Url::parse(&format!("http://{}", driver_addr)).map_err(Error::from) } }?; - println!( - "Running headless tests in {} on `{}`", - driver.browser(), - driver_url.as_str(), - ); + if debug { + println!( + "Running headless tests in {} on `{}`", + driver.browser(), + driver_url.as_str(), + ) + }; let mut client = Client { handle: Easy::new(), driver_url, session: None, }; - println!("Try find `webdriver.json` for configure browser's capabilities:"); + if debug { + println!("Try find `webdriver.json` for configure browser's capabilities:"); + } let capabilities: Capabilities = match File::open("webdriver.json") { Ok(file) => { - println!("Ok"); + if debug { + println!("Ok"); + } serde_json::from_reader(file) } Err(_) => { - println!("Not found"); + if debug { + println!("Not found"); + } Ok(Capabilities::new()) } }?; @@ -148,6 +162,10 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error let max = Duration::new(timeout, 0); while start.elapsed() < max { if client.text(&id, &output)?.contains("test result: ") { + // If the tests harness finished (either successfully or unsuccessfully) + // then in theory all the info needed to debug the failure is in its own + // output, so we shouldn't need the driver logs to get printed. + drop_log(); break; } thread::sleep(Duration::from_millis(100)); @@ -162,31 +180,109 @@ pub fn run(server: &SocketAddr, shell: &Shell, timeout: u64) -> Result<(), Error let logs = client.text(&id, &logs)?; let errors = client.text(&id, &errors)?; - if output.contains("test result: ") { - println!("{}", output); + Ok((output, logs, errors)) +} - // If the tests harness finished (either successfully or unsuccessfully) - // then in theory all the info needed to debug the failure is in its own - // output, so we shouldn't need the driver logs to get printed. - drop_log(); +/// Run all tests and present their results to user +pub fn run(server: &SocketAddr, shell: Arc, timeout: u64, debug: bool) -> Result<(), Error> { + let (output, logs, errors) = execute_tests(server, shell, timeout, debug)?; + + if output.contains("test result: ok") { + // All tests succeeded. + println!("{output}"); + } else if output.contains("test result") { + // At least one test has failed but whole process managed to finish by itself. + // We can still only show output as it aggregates errors and logs + println!("{output}"); + bail!("some tests failed") } else { - println!("Failed to detect test as having been run. It might have timed out."); - if output.len() > 0 { + // Testing process hasn't finished, so we most probably timeouted. + // In that case output may not contain all informations so we print + // everything we can. + println!("Failed to detect test as having been run. It might have timed out.\n"); + if !output.is_empty() { println!("output div contained:\n{}", tab(&output)); } + if !errors.is_empty() { + println!("errors div contained:\n{}", tab(&errors)); + } + if !logs.is_empty() { + println!("logs div contained:\n{}", tab(&logs)); + } + bail!("some tests failed") } - if logs.len() > 0 { - println!("console.log div contained:\n{}", tab(&logs)); - } - if errors.len() > 0 { - println!("console.log div contained:\n{}", tab(&errors)); + Ok(()) +} + +/// Possible outcomes of single test +#[derive(Clone, Debug, PartialEq)] +pub enum TestResult { + Passed, + Failed { + details: String, + }, + Ignored, + Timeouted { + output: String, + errors: String, + logs: String, + }, +} + +/// Run single isolated test and postprocess it's output +/// In a way that it can be combined with other results of +/// isolated tests +pub fn run_isolated( + server: &SocketAddr, + shell: Arc, + timeout: u64, + debug: bool, +) -> Result { + let (output, logs, errors) = execute_tests(server, shell, timeout, debug)?; + + if !output.contains("test result:") { + // Testing process hasn't finished, so we most probably timeouted. + return Ok(TestResult::Timeouted { + output, + logs, + errors, + }); } - if !output.contains("test result: ok") { - bail!("some tests failed") + // Isolated tests are running parallelly, so we want to return all info instead + // of printing it right now. Doing the latter could result in interleaving prints + // from multiple tests and lead to confusion. + // We can tho print the short outcome of the test right now. + for line in output.lines() { + if line.starts_with("test ") && (line.ends_with("... ok") || line.ends_with("... FAIL")) { + println!("{line}"); + } } - Ok(()) + if output.contains("test result: ok") { + if output.contains("1 ignored") { + // The test has been ignored + Ok(TestResult::Ignored) + } else { + // The test has succeeded. + Ok(TestResult::Passed) + } + } else { + // The test has failed. Whole testing process went good tho as we have the final + // summary. Test suite already did a job of providing the failure context, we + // only need to extract it. + let details = output + .lines() + // Skip until we find a line indicating the start of failure summary + .skip_while(|&line| line != "failures:") + // and also this line itself with the empty one after it. + .skip(2) + // Take all lines until we reach the ending summary of all failed tests. + .take_while(|&line| line != "failures:") + .fold(String::new(), |a, b| a + b + "\n"); + + Ok(TestResult::Failed { details }) + } } enum Driver { @@ -577,7 +673,7 @@ fn read(r: &mut R) -> io::Result> { Ok(dst) } -fn tab(s: &str) -> String { +pub fn tab(s: &str) -> String { let mut result = String::new(); for line in s.lines() { result.push_str(" "); @@ -587,20 +683,16 @@ fn tab(s: &str) -> String { return result; } -struct BackgroundChild<'a> { +struct BackgroundChild { child: Child, stdout: Option>>>, stderr: Option>>>, - shell: &'a Shell, + shell: Arc, print_stdio_on_drop: bool, } -impl<'a> BackgroundChild<'a> { - fn spawn( - path: &Path, - cmd: &mut Command, - shell: &'a Shell, - ) -> Result, Error> { +impl BackgroundChild { + fn spawn(path: &Path, cmd: &mut Command, shell: Arc) -> Result { cmd.stdout(Stdio::piped()) .stderr(Stdio::piped()) .stdin(Stdio::null()); @@ -622,7 +714,7 @@ impl<'a> BackgroundChild<'a> { } } -impl<'a> Drop for BackgroundChild<'a> { +impl Drop for BackgroundChild { fn drop(&mut self) { self.child.kill().unwrap(); let status = self.child.wait().unwrap(); diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html b/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html index a65c242fd58..99381736ca1 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html @@ -36,6 +36,6 @@ window.__wbg_test_invoke = f => f(); - + diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs index 95a82601702..b4a1fdf0c98 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/main.rs @@ -15,13 +15,13 @@ use anyhow::{anyhow, bail, Context}; use std::env; use std::fs; use std::path::PathBuf; -use std::thread; use wasm_bindgen_cli_support::Bindgen; // no need for jemalloc bloat in this binary (and we don't need speed) #[global_allocator] static ALLOC: std::alloc::System = std::alloc::System; +mod browser; mod deno; mod headless; mod node; @@ -68,14 +68,13 @@ fn main() -> anyhow::Result<()> { let wasm = fs::read(&wasm_file_to_test).context("failed to read wasm file")?; let mut wasm = walrus::Module::from_buffer(&wasm).context("failed to deserialize wasm module")?; - let mut tests = Vec::new(); - for export in wasm.exports.iter() { - if !export.name.starts_with("__wbgt_") { - continue; - } - tests.push(export.name.to_string()); - } + let tests = wasm + .exports + .iter() + .filter(|exp| exp.name.starts_with("__wbgt_")) + .map(|exp| exp.name.to_string()) + .collect::>(); // Right now there's a bug where if no tests are present then the // `wasm-bindgen-test` runtime support isn't linked in, so just bail out @@ -98,9 +97,6 @@ fn main() -> anyhow::Result<()> { None => TestMode::Node, }; - let headless = env::var("NO_HEADLESS").is_err(); - let debug = env::var("WASM_BINDGEN_NO_DEBUG").is_err(); - // Gracefully handle requests to execute only node or only web tests. let node = test_mode == TestMode::Node; @@ -131,18 +127,6 @@ integration test.\ } } - let timeout = env::var("WASM_BINDGEN_TEST_TIMEOUT") - .map(|timeout| { - timeout - .parse() - .expect("Could not parse 'WASM_BINDGEN_TEST_TIMEOUT'") - }) - .unwrap_or(20); - - if debug { - println!("Set timeout to {} seconds...", timeout); - } - // Make the generated bindings available for the tests to execute against. shell.status("Executing bindgen..."); let mut b = Bindgen::new(); @@ -152,6 +136,7 @@ integration test.\ TestMode::Browser => b.web(true)?, }; + let debug = env::var("WASM_BINDGEN_DEBUG").is_ok(); b.debug(debug) .input_module(module, wasm) .keep_debug(false) @@ -163,42 +148,9 @@ integration test.\ let args: Vec<_> = args.collect(); match test_mode { - TestMode::Node => node::execute(&module, &tmpdir, &args, &tests)?, - TestMode::Deno => deno::execute(&module, &tmpdir, &args, &tests)?, - TestMode::Browser => { - let srv = server::spawn( - &if headless { - "127.0.0.1:0".parse().unwrap() - } else { - "127.0.0.1:8000".parse().unwrap() - }, - headless, - &module, - &tmpdir, - &args, - &tests, - ) - .context("failed to spawn server")?; - let addr = srv.server_addr(); - - // TODO: eventually we should provide the ability to exit at some point - // (gracefully) here, but for now this just runs forever. - if !headless { - println!( - "Interactive browsers tests are now available at http://{}", - addr - ); - println!(""); - println!("Note that interactive mode is enabled because `NO_HEADLESS`"); - println!("is specified in the environment of this process. Once you're"); - println!("done with testing you'll need to kill this server with"); - println!("Ctrl-C."); - return Ok(srv.run()); - } - - thread::spawn(|| srv.run()); - headless::run(&addr, &shell, timeout)?; - } + TestMode::Node => node::execute(module, &tmpdir, &args, &tests)?, + TestMode::Deno => deno::execute(module, &tmpdir, &args, &tests)?, + TestMode::Browser => browser::execute(shell, module, &tmpdir, &args, tests)?, } Ok(()) } diff --git a/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs b/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs index 631d9ec2f8d..f40a7f12b54 100644 --- a/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs +++ b/crates/cli/src/bin/wasm-bindgen-test-runner/server.rs @@ -1,5 +1,6 @@ use std::ffi::OsString; use std::fs; +use std::hash::{Hash, Hasher}; use std::net::SocketAddr; use std::path::Path; @@ -59,11 +60,18 @@ pub fn spawn( } js_to_execute.push_str("main(tests);\n"); - let js_path = tmpdir.join("run.js"); + // As headless tests may be executed parallely, we need to ensure + // that js script for each of them has unique name + let script_name = create_script_name(tests); + let js_path = tmpdir.join(&script_name); fs::write(&js_path, js_to_execute).context("failed to write JS file")?; - // For now, always run forever on this port. We may update this later! let tmpdir = tmpdir.to_path_buf(); + // And here we make sure that correct script is served by this instance + let index_headless = + include_str!("index-headless.html").replace("%%SCRIPT_NAME%%", &script_name); + let index_regular = include_str!("index.html"); + // For now, always run forever on this port. We may update this later! let srv = Server::new(addr, move |request| { // The root path gets our canned `index.html`. The two templates here // differ slightly in the default routing of `console.log`, going to an @@ -71,9 +79,9 @@ pub fn spawn( // output. if request.url() == "/" { let s = if headless { - include_str!("index-headless.html") + &index_headless } else { - include_str!("index.html") + index_regular }; return Response::from_data("text/html", s); } @@ -123,3 +131,13 @@ pub fn spawn( response } } + +// If we are running multiple tests parallely, we need to ensure +// that each script will have a unique name. Hashing the tests slice +// seems to be the easiest sollution that's kinda safe. Joining test +// names could hit max path length easily when there are a lot of them. +fn create_script_name(tests: &[String]) -> String { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + tests.hash(&mut hasher); + format!("run_{:x}.js", hasher.finish()) +}