Add --ignore_timeout_reports to use timeout without reporting errors. #Centipede

This is to be used when timeout reports are noisy.

In this change, timeouts are almost hidden from the view of the controller (except that the controller can see an incomplete batch execution result). Having more visibility in controller is desired but not done yet.

PiperOrigin-RevId: 740033429

Author: xinhaoyuan

centipede/centipede_callbacks.cc

@@ -131,6 +131,9 @@ std::string CentipedeCallbacks::ConstructRunnerFlags(
       absl::StrCat("stack_limit_kb=", env_.stack_limit_kb),
       absl::StrCat("crossover_level=", env_.crossover_level),
   };
+  if (env_.ignore_timeout_reports) {
+    flags.emplace_back("ignore_timeout_reports");
+  }
   if (!disable_coverage) {
     flags.emplace_back(absl::StrCat("path_level=", env_.path_level));
     if (env_.use_pc_features) flags.emplace_back("use_pc_features");

centipede/environment.h

@@ -68,6 +68,7 @@ struct Environment {
   size_t timeout_per_input = 60;
   size_t timeout_per_batch = 0;
   absl::Duration force_abort_timeout = absl::Minutes(15);
+  bool ignore_timeout_reports = false;
   absl::Time stop_at = absl::InfiniteFuture();
   bool fork_server = true;
   bool full_sync = false;

centipede/environment_flags.cc

@@ -166,6 +166,9 @@ ABSL_FLAG(
                  "hangs (e.g., during stacktrace dumps). The default value is ",
                  Environment::Default().force_abort_timeout,
                  "; use 'inf' to disable."));
+ABSL_FLAG(bool, ignore_timeout_reports,
+          Environment::Default().ignore_timeout_reports,
+          "If set, will ignore reporting timeouts as errors.");
 ABSL_FLAG(absl::Time, stop_at, Environment::Default().stop_at,
           "Stop fuzzing in all shards (--total_shards) at approximately this "
           "time in ISO-8601/RFC-3339 format, e.g. 2023-04-06T23:35:02Z. "
@@ -472,6 +475,7 @@ Environment CreateEnvironmentFromFlags(const std::vector<std::string> &argv) {
       /*timeout_per_input=*/absl::GetFlag(FLAGS_timeout_per_input),
       /*timeout_per_batch=*/absl::GetFlag(FLAGS_timeout_per_batch),
       /*force_abort_timeout=*/absl::GetFlag(FLAGS_force_abort_timeout),
+      /*ignore_timeout_reports=*/absl::GetFlag(FLAGS_ignore_timeout_reports),
       /*stop_at=*/
       GetStopAtTime(absl::GetFlag(FLAGS_stop_at),
                     absl::GetFlag(FLAGS_stop_after)),

centipede/runner.cc

@@ -240,32 +240,36 @@ static void CheckWatchdogLimits() {
     const char *units;
     uint64_t value;
     uint64_t limit;
+    bool ignore_report;
     const char *failure;
   };
   const uint64_t input_start_time = state.input_start_time;
   const uint64_t batch_start_time = state.batch_start_time;
   if (input_start_time == 0 || batch_start_time == 0) return;
   const Resource resources[] = {
       {Resource{
-          /*what =*/"Per-input timeout",
-          /*units =*/"sec",
-          /*value =*/curr_time - input_start_time,
-          /*limit =*/state.run_time_flags.timeout_per_input,
-          /*failure =*/kExecutionFailurePerInputTimeout.data(),
+          /*what=*/"Per-input timeout",
+          /*units=*/"sec",
+          /*value=*/curr_time - input_start_time,
+          /*limit=*/state.run_time_flags.timeout_per_input,
+          /*ignore_report=*/state.run_time_flags.ignore_timeout_reports != 0,
+          /*failure=*/kExecutionFailurePerInputTimeout.data(),
       }},
       {Resource{
-          /*what =*/"Per-batch timeout",
-          /*units =*/"sec",
-          /*value =*/curr_time - batch_start_time,
-          /*limit =*/state.run_time_flags.timeout_per_batch,
-          /*failure =*/kExecutionFailurePerBatchTimeout.data(),
+          /*what=*/"Per-batch timeout",
+          /*units=*/"sec",
+          /*value=*/curr_time - batch_start_time,
+          /*limit=*/state.run_time_flags.timeout_per_batch,
+          /*ignore_report=*/state.run_time_flags.ignore_timeout_reports != 0,
+          /*failure=*/kExecutionFailurePerBatchTimeout.data(),
       }},
       {Resource{
-          /*what =*/"RSS limit",
-          /*units =*/"MB",
-          /*value =*/GetPeakRSSMb(),
-          /*limit =*/state.run_time_flags.rss_limit_mb,
-          /*failure =*/kExecutionFailureRssLimitExceeded.data(),
+          /*what=*/"RSS limit",
+          /*units=*/"MB",
+          /*value=*/GetPeakRSSMb(),
+          /*limit=*/state.run_time_flags.rss_limit_mb,
+          /*ignore_report=*/false,
+          /*failure=*/kExecutionFailureRssLimitExceeded.data(),
       }},
   };
   for (const auto &resource : resources) {
@@ -275,6 +279,15 @@ static void CheckWatchdogLimits() {
       // `RunOneInput()` after all the work is done.
       static std::atomic<bool> already_handling_failure = false;
       if (!already_handling_failure.exchange(true)) {
+        if (resource.ignore_report) {
+          fprintf(stderr,
+                  "========= %s exceeded: %" PRIu64 " > %" PRIu64
+                  " (%s); exiting without reporting as an error\n",
+                  resource.what, resource.value, resource.limit,
+                  resource.units);
+          std::_Exit(0);
+          // should not return here.
+        }
         fprintf(stderr,
                 "========= %s exceeded: %" PRIu64 " > %" PRIu64
                 " (%s); exiting\n",

centipede/runner.h

@@ -72,6 +72,7 @@ struct RunTimeFlags {
   std::atomic<uint64_t> rss_limit_mb;
   uint64_t crossover_level;
   uint64_t skip_seen_features : 1;
+  uint64_t ignore_timeout_reports : 1;
 };
 
 // One such object is created in runner's TLS.
@@ -177,7 +178,9 @@ struct GlobalRunnerState {
       /*stack_limit_kb=*/HasIntFlag(":stack_limit_kb=", 0),
       /*rss_limit_mb=*/HasIntFlag(":rss_limit_mb=", 0),
       /*crossover_level=*/HasIntFlag(":crossover_level=", 50),
-      /*skip_seen_features=*/HasFlag(":skip_seen_features:")};
+      /*skip_seen_features=*/HasFlag(":skip_seen_features:"),
+      /*ignore_timeout_reports=*/HasFlag(":ignore_timeout_reports:"),
+  };
 
   // Returns true iff `flag` is present.
   // Typical usage: pass ":some_flag:", i.e. the flag name surrounded with ':'.

centipede/testing/centipede_main_test.sh

@@ -176,10 +176,37 @@ test_pcpair_features() {
   centipede::assert_regex_in_file "end-fuzz.*pair: [^0]" "${LOG}"
 }
 
+test_timeouts() {
+  FUNC="${FUNCNAME[0]}"
+  WD="${TEST_TMPDIR}/${FUNC}/WD"
+  CORPUS="${TEST_TMPDIR}/${FUNC}/corpus"
+  LOG="${TEST_TMPDIR}/${FUNC}/log"
+
+  centipede::ensure_empty_dir "${WD}"
+  centipede::ensure_empty_dir "${CORPUS}"
+  echo -n "slo" >"${CORPUS}"/input
+
+  echo "============ ${FUNC}: fuzz with --timeout_per_input"
+  test_fuzz --workdir="${WD}" --corpus_dir="${CORPUS}" --num_runs=0 --timeout_per_input=2 | tee "${LOG}"
+  centipede::assert_regex_in_file "Failure.*: per-input-timeout-exceeded" "${LOG}"
+  centipede::assert_regex_in_file "end-fuzz:.*crash: 1" "${LOG}"
+
+  centipede::ensure_empty_dir "${WD}"
+  centipede::ensure_empty_dir "${CORPUS}"
+  echo -n "slo" >"${CORPUS}"/input
+
+  echo "============ ${FUNC}: fuzz with --timeout_per_input --ignore_timeout_reports"
+  test_fuzz --workdir="${WD}" --corpus_dir="${CORPUS}" --num_runs=0 --timeout_per_input=2 --ignore_timeout_reports | tee "${LOG}"
+  centipede::assert_regex_not_in_file "Failure.*: per-input-timeout-exceeded" "${LOG}"
+  centipede::assert_regex_not_in_file "end-fuzz:.*crash: 1" "${LOG}"
+}
+
+
 centipede::test_crashing_target abort_test_fuzz "foo" "AbOrT" "I AM ABOUT TO ABORT"
 test_debug_symbols
 test_dictionary
 test_for_each_blob
 test_pcpair_features
+test_timeouts
 
 echo "PASS"

centipede/testing/runner_test.sh

@@ -107,11 +107,17 @@ CENTIPEDE_RUNNER_FLAGS=":rss_limit_mb=8192:" $target "${oom}"  # must pass
 
 echo ======== Check timeout
 CENTIPEDE_RUNNER_FLAGS=":timeout_per_input=567:" "${target}" \
-  2>&1 | grep "timeout_per_input:.567"
+  2>&1 | grep "timeout_per_input: 567"
 
 CENTIPEDE_RUNNER_FLAGS=":timeout_per_input=2:" "${target}" "${slo}" \
   2>&1 | grep "Per-input timeout exceeded"
 
+{
+    CENTIPEDE_RUNNER_FLAGS=":ignore_timeout_reports:timeout_per_input=2:" "${target}" "${slo}";
+    echo "$?" > "${TEST_TMPDIR}/ignore_timeout_reports_exit_code";
+} 2>&1 | grep "Per-input timeout exceeded" | grep "exiting without reporting as an error"
+((`cat "${TEST_TMPDIR}/ignore_timeout_reports_exit_code"` == 0))
+
 echo ======== Check stack limit check with stack_limit
 CENTIPEDE_RUNNER_FLAGS=":use_pc_features:stack_limit_kb=200:" "${target}" "${stk}"  # must pass