diff --git a/library/test/src/console.rs b/library/test/src/console.rs index dc0123cf43266..82cb6d4132616 100644 --- a/library/test/src/console.rs +++ b/library/test/src/console.rs @@ -106,10 +106,10 @@ impl ConsoleTestState { let TestDesc { name, ignore_message, .. } = test; format!( "{} {}", - match *result { + match result { TestResult::TrOk => "ok".to_owned(), TestResult::TrFailed => "failed".to_owned(), - TestResult::TrFailedMsg(ref msg) => format!("failed: {msg}"), + TestResult::TrFailedMsg(msg) => format!("failed: {msg}"), TestResult::TrIgnored => { if let Some(msg) = ignore_message { format!("ignored: {msg}") @@ -117,7 +117,8 @@ impl ConsoleTestState { "ignored".to_owned() } } - TestResult::TrBench(ref bs) => fmt_bench_samples(bs), + TestResult::TrIgnoredMsg(msg) => format!("ignored: {msg}"), + TestResult::TrBench(bs) => fmt_bench_samples(bs), TestResult::TrTimedFail => "failed (time limit exceeded)".to_owned(), }, name, @@ -194,7 +195,7 @@ fn handle_test_result(st: &mut ConsoleTestState, completed_test: CompletedTest) st.passed += 1; st.not_failures.push((test, stdout)); } - TestResult::TrIgnored => st.ignored += 1, + TestResult::TrIgnored | TestResult::TrIgnoredMsg(_) => st.ignored += 1, TestResult::TrBench(bs) => { st.metrics.insert_metric( test.name.as_slice(), diff --git a/library/test/src/formatters/json.rs b/library/test/src/formatters/json.rs index c07fdafb167c9..c83d3140d1e00 100644 --- a/library/test/src/formatters/json.rs +++ b/library/test/src/formatters/json.rs @@ -93,7 +93,7 @@ impl OutputFormatter for JsonFormatter { } else { None }; - match *result { + match result { TestResult::TrOk => { self.write_event("test", desc.name.as_slice(), "ok", exec_time, stdout, None) } @@ -111,7 +111,7 @@ impl OutputFormatter for JsonFormatter { Some(r#""reason": "time limit exceeded""#), ), - TestResult::TrFailedMsg(ref m) => self.write_event( + TestResult::TrFailedMsg(m) => self.write_event( "test", desc.name.as_slice(), "failed", @@ -131,7 +131,16 @@ impl OutputFormatter for JsonFormatter { .as_deref(), ), - TestResult::TrBench(ref bs) => { + TestResult::TrIgnoredMsg(msg) => self.write_event( + "test", + desc.name.as_slice(), + "ignored", + exec_time, + stdout, + Some(&*msg), + ), + + TestResult::TrBench(bs) => { let median = bs.ns_iter_summ.median as usize; let deviation = (bs.ns_iter_summ.max - bs.ns_iter_summ.min) as usize; diff --git a/library/test/src/formatters/junit.rs b/library/test/src/formatters/junit.rs index e6fb4f5707b35..e4c2dd4eda168 100644 --- a/library/test/src/formatters/junit.rs +++ b/library/test/src/formatters/junit.rs @@ -76,7 +76,7 @@ impl OutputFormatter for JunitFormatter { for (desc, result, duration) in std::mem::replace(&mut self.results, Vec::new()) { let (class_name, test_name) = parse_class_name(&desc); match result { - TestResult::TrIgnored => { /* no-op */ } + TestResult::TrIgnored | TestResult::TrIgnoredMsg(_) => { /* no-op */ } TestResult::TrFailed => { self.write_message(&*format!( " PrettyFormatter { self.write_short_result("FAILED", term::color::RED) } - pub fn write_ignored(&mut self, message: Option<&'static str>) -> io::Result<()> { + pub fn write_ignored(&mut self, message: Option<&str>) -> io::Result<()> { if let Some(message) = message { self.write_short_result(&format!("ignored, {}", message), term::color::YELLOW) } else { @@ -215,11 +215,12 @@ impl OutputFormatter for PrettyFormatter { self.write_test_name(desc)?; } - match *result { + match result { TestResult::TrOk => self.write_ok()?, TestResult::TrFailed | TestResult::TrFailedMsg(_) => self.write_failed()?, TestResult::TrIgnored => self.write_ignored(desc.ignore_message)?, - TestResult::TrBench(ref bs) => { + TestResult::TrIgnoredMsg(msg) => self.write_ignored(Some(msg))?, + TestResult::TrBench(bs) => { self.write_bench()?; self.write_plain(&format!(": {}", fmt_bench_samples(bs)))?; } diff --git a/library/test/src/formatters/terse.rs b/library/test/src/formatters/terse.rs index 5dace8baef7f8..261f1047be7a3 100644 --- a/library/test/src/formatters/terse.rs +++ b/library/test/src/formatters/terse.rs @@ -203,7 +203,7 @@ impl OutputFormatter for TerseFormatter { TestResult::TrFailed | TestResult::TrFailedMsg(_) | TestResult::TrTimedFail => { self.write_failed() } - TestResult::TrIgnored => self.write_ignored(), + TestResult::TrIgnored | TestResult::TrIgnoredMsg(_) => self.write_ignored(), TestResult::TrBench(ref bs) => { if self.is_multithreaded { self.write_test_name(desc)?; diff --git a/library/test/src/lib.rs b/library/test/src/lib.rs index 0c748da1a59cc..0c1d3a5788781 100644 --- a/library/test/src/lib.rs +++ b/library/test/src/lib.rs @@ -86,6 +86,31 @@ use options::{Concurrent, RunStrategy}; use test_result::*; use time::TestExecTime; +/// Panic payload to indicate that a test should be marked as ignored, rather +/// than failed. +/// +/// # Examples +/// +/// ``` +/// #![feature(test)] +/// extern crate test; +/// # use std::panic::panic_any; +/// # fn compute_should_ignore() -> bool { true } +/// # fn do_real_test() { unreachable!() } +/// +/// if compute_should_ignore() { +/// panic_any(test::IgnoreTest { +/// reason: Some("".into()) +/// }); +/// } else { +/// do_real_test(); +/// } +/// ``` +#[derive(Debug, Clone, Default)] +pub struct IgnoreTest { + pub reason: Option, +} + // Process exit code to be used to indicate test failures. const ERROR_EXIT_CODE: i32 = 101; @@ -683,10 +708,10 @@ fn run_test_in_spawned_subprocess(desc: TestDesc, testfn: Box process::exit(test_result::TR_OK), + TrIgnored | TrIgnoredMsg(_) => process::exit(test_result::TR_IGNORED), + _ => process::exit(test_result::TR_FAILED), } }); let record_result2 = record_result.clone(); diff --git a/library/test/src/test_result.rs b/library/test/src/test_result.rs index 7f44d6e3d0f12..2b4028c92b691 100644 --- a/library/test/src/test_result.rs +++ b/library/test/src/test_result.rs @@ -4,6 +4,7 @@ use super::bench::BenchSamples; use super::options::ShouldPanic; use super::time; use super::types::TestDesc; +use super::IgnoreTest; pub use self::TestResult::*; @@ -12,6 +13,7 @@ pub use self::TestResult::*; // it means. pub const TR_OK: i32 = 50; pub const TR_FAILED: i32 = 51; +pub const TR_IGNORED: i32 = 52; #[derive(Debug, Clone, PartialEq)] pub enum TestResult { @@ -19,6 +21,7 @@ pub enum TestResult { TrFailed, TrFailedMsg(String), TrIgnored, + TrIgnoredMsg(String), TrBench(BenchSamples), TrTimedFail, } @@ -32,8 +35,8 @@ pub fn calc_result<'a>( exec_time: &Option, ) -> TestResult { let result = match (&desc.should_panic, task_result) { - (&ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TestResult::TrOk, - (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => { + (ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TestResult::TrOk, + (ShouldPanic::YesWithMessage(msg), Err(err)) => { let maybe_panic_str = err .downcast_ref::() .map(|e| &**e) @@ -53,15 +56,24 @@ pub fn calc_result<'a>( r#"expected panic with string value, found non-string value: `{:?}` expected substring: `{:?}`"#, - (**err).type_id(), + (*err).type_id(), msg )) } } - (&ShouldPanic::Yes, Ok(())) | (&ShouldPanic::YesWithMessage(_), Ok(())) => { + (ShouldPanic::Yes, Ok(())) | (ShouldPanic::YesWithMessage(_), Ok(())) => { TestResult::TrFailedMsg("test did not panic as expected".to_string()) } - _ => TestResult::TrFailed, + (ShouldPanic::No, Err(err)) => { + if let Some(ignore) = err.downcast_ref::() { + match &ignore.reason { + Some(reason) => TestResult::TrIgnoredMsg(reason.clone()), + None => TestResult::TrIgnored, + } + } else { + TestResult::TrFailed + } + } }; // If test is already failed (or allowed to fail), do not change the result. @@ -89,6 +101,7 @@ pub fn get_result_from_exit_code( let result = match code { TR_OK => TestResult::TrOk, TR_FAILED => TestResult::TrFailed, + TR_IGNORED => TestResult::TrIgnored, _ => TestResult::TrFailedMsg(format!("got unexpected return code {code}")), }; diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 82e367427ef6f..a27f3347e1a5b 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -264,6 +264,14 @@ enum TestFailure { UnexpectedRunPass, } +/// Documentation test success modes. +enum TestSuccess { + /// The test passed normally. + Pass, + /// The test panicked with `IgnoreTest` + Ignore, +} + enum DirState { Temp(tempfile::TempDir), Perm(PathBuf), @@ -290,6 +298,10 @@ struct UnusedExterns { unused_extern_names: Vec, } +/// Exit code to specify that the test should be ignored. Happens to match the +/// one used by the test crate, but this is not strictly required. +const TR_IGNORED: i32 = 52; + fn run_test( test: &str, crate_name: &str, @@ -306,7 +318,7 @@ fn run_test( path: PathBuf, test_id: &str, report_unused_externs: impl Fn(UnusedExterns), -) -> Result<(), TestFailure> { +) -> Result { let (test, line_offset, supports_color) = make_test(test, Some(crate_name), lang_string.test_harness, opts, edition, Some(test_id)); @@ -446,7 +458,7 @@ fn run_test( } if no_run { - return Ok(()); + return Ok(TestSuccess::Pass); } // Run the code! @@ -478,12 +490,16 @@ fn run_test( if lang_string.should_panic && out.status.success() { return Err(TestFailure::UnexpectedRunPass); } else if !lang_string.should_panic && !out.status.success() { - return Err(TestFailure::ExecutionFailure(out)); + if out.status.code() == Some(TR_IGNORED) { + return Ok(TestSuccess::Ignore); + } else { + return Err(TestFailure::ExecutionFailure(out)); + } } } } - Ok(()) + Ok(TestSuccess::Pass) } /// Transforms a test into code that can be compiled into a Rust binary, and returns the number of @@ -675,6 +691,28 @@ crate fn make_test( if dont_insert_main || already_has_main || prog.contains("![no_std]") { prog.push_str(everything_else); } else { + // HACK(cad97): only try to catch IgnoreTest panic if feature(test) + // is enabled by the test and it includes the test crate. + let found_feature_test = + crate_attrs.contains("feature(test)") && prog.contains("extern crate test;"); + let (catch_ignore_pre, catch_ignore_post) = if found_feature_test { + ( + "match ::std::panic::catch_unwind(|| ", + format!( + " ) {{ \ + Ok(ok) => ok, \ + Err(err) => if let Some(_) = err.downcast_ref::<::test::IgnoreTest>() {{\ + ::std::process::exit({TR_IGNORED}) \ + }} else {{\ + ::std::panic::resume_unwind(err) \ + }} \ + }}" + ), + ) + } else { + ("", "".into()) + }; + let returns_result = everything_else.trim_end().ends_with("(())"); // Give each doctest main function a unique name. // This is for example needed for the tooling around `-C instrument-coverage`. @@ -684,6 +722,7 @@ crate fn make_test( "_inner".into() }; let inner_attr = if test_id.is_some() { "#[allow(non_snake_case)] " } else { "" }; + let (main_pre, main_post) = if returns_result { ( format!( @@ -694,7 +733,7 @@ crate fn make_test( } else if test_id.is_some() { ( format!("fn main() {{ {inner_attr}fn {inner_fn_name}() {{\n",), - format!("\n}} {inner_fn_name}() }}"), + format!("\n}} {catch_ignore_pre}{inner_fn_name}(){catch_ignore_post} }}"), ) } else { ("fn main() {\n".into(), "\n}".into()) @@ -1048,56 +1087,69 @@ impl Tester for Collector { report_unused_externs, ); - if let Err(err) = res { - match err { - TestFailure::CompileError => { - eprint!("Couldn't compile the test."); - } - TestFailure::UnexpectedCompilePass => { - eprint!("Test compiled successfully, but it's marked `compile_fail`."); - } - TestFailure::UnexpectedRunPass => { - eprint!("Test executable succeeded, but it's marked `should_panic`."); - } - TestFailure::MissingErrorCodes(codes) => { - eprint!("Some expected error codes were not found: {:?}", codes); - } - TestFailure::ExecutionError(err) => { - eprint!("Couldn't run the test: {err}"); - if err.kind() == io::ErrorKind::PermissionDenied { - eprint!(" - maybe your tempdir is mounted with noexec?"); + match res { + Ok(TestSuccess::Pass) => (), + Ok(TestSuccess::Ignore) => { + panic::resume_unwind(box test::IgnoreTest::default()) + } + Err(err) => { + match err { + TestFailure::CompileError => { + eprint!("Couldn't compile the test."); } - } - TestFailure::ExecutionFailure(out) => { - eprintln!("Test executable failed ({reason}).", reason = out.status); - - // FIXME(#12309): An unfortunate side-effect of capturing the test - // executable's output is that the relative ordering between the test's - // stdout and stderr is lost. However, this is better than the - // alternative: if the test executable inherited the parent's I/O - // handles the output wouldn't be captured at all, even on success. - // - // The ordering could be preserved if the test process' stderr was - // redirected to stdout, but that functionality does not exist in the - // standard library, so it may not be portable enough. - let stdout = str::from_utf8(&out.stdout).unwrap_or_default(); - let stderr = str::from_utf8(&out.stderr).unwrap_or_default(); - - if !stdout.is_empty() || !stderr.is_empty() { - eprintln!(); - - if !stdout.is_empty() { - eprintln!("stdout:\n{stdout}"); + TestFailure::UnexpectedCompilePass => { + eprint!( + "Test compiled successfully, but it's marked `compile_fail`." + ); + } + TestFailure::UnexpectedRunPass => { + eprint!( + "Test executable succeeded, but it's marked `should_panic`." + ); + } + TestFailure::MissingErrorCodes(codes) => { + eprint!("Some expected error codes were not found: {:?}", codes); + } + TestFailure::ExecutionError(err) => { + eprint!("Couldn't run the test: {err}"); + if err.kind() == io::ErrorKind::PermissionDenied { + eprint!(" - maybe your tempdir is mounted with noexec?"); } - - if !stderr.is_empty() { - eprintln!("stderr:\n{stderr}"); + } + TestFailure::ExecutionFailure(out) => { + eprintln!( + "Test executable failed ({reason}).", + reason = out.status + ); + + // FIXME(#12309): An unfortunate side-effect of capturing the test + // executable's output is that the relative ordering between the test's + // stdout and stderr is lost. However, this is better than the + // alternative: if the test executable inherited the parent's I/O + // handles the output wouldn't be captured at all, even on success. + // + // The ordering could be preserved if the test process' stderr was + // redirected to stdout, but that functionality does not exist in the + // standard library, so it may not be portable enough. + let stdout = str::from_utf8(&out.stdout).unwrap_or_default(); + let stderr = str::from_utf8(&out.stderr).unwrap_or_default(); + + if !stdout.is_empty() || !stderr.is_empty() { + eprintln!(); + + if !stdout.is_empty() { + eprintln!("stdout:\n{stdout}"); + } + + if !stderr.is_empty() { + eprintln!("stderr:\n{stderr}"); + } } } } - } - panic::resume_unwind(box ()); + panic::resume_unwind(box ()); + } } }), });