Speed up and improve some tests (#182)

tests/cli/jobs.rs

@@ -2,19 +2,43 @@
 
 //! Test handling of `--jobs` concurrency option.
 
+use std::fs::read_to_string;
+
+use itertools::Itertools;
+use regex::Regex;
+
 use super::{copy_of_testdata, run};
 
 /// It's a bit hard to assess that multiple jobs really ran in parallel,
-/// but we can at least check that the option is accepted.
+/// but we can at least check that the option is accepted, and see that the
+/// debug log looks like it's using multiple threads.
 #[test]
-fn jobs_option_accepted() {
-    let testdata = copy_of_testdata("well_tested");
+fn jobs_option_accepted_and_causes_multiple_threads() {
+    let testdata = copy_of_testdata("small_well_tested");
     run()
         .arg("mutants")
         .arg("-d")
         .arg(testdata.path())
-        .arg("-j3")
+        .arg("-j2")
         .arg("--minimum-test-timeout=120") // to avoid flakes on slow CI
         .assert()
         .success();
+    let debug_log =
+        read_to_string(testdata.path().join("mutants.out/debug.log")).expect("read debug log");
+    println!("debug log:\n{debug_log}");
+    // This might be brittle, as the ThreadId debug form is not specified, and
+    // also _possibly_ everything will complete on one thread before the next
+    // gets going, though that seems unlikely.
+    let re = Regex::new(r#"\{thread=ThreadId\(\d+\)\}"#).expect("compile regex");
+    let matches = re
+        .find_iter(&debug_log)
+        .map(|m| m.as_str())
+        .unique()
+        .collect::<Vec<_>>();
+    println!("threadid matches: {matches:?}");
+    assert!(
+        matches.len() > 1,
+        "expected more than {} thread ids in debug log",
+        matches.len()
+    );
 }

tests/cli/main.rs

@@ -5,7 +5,7 @@
 use std::borrow::Borrow;
 use std::env;
 use std::fmt::Write;
-use std::fs::{self, read_dir};
+use std::fs::{self, read_dir, read_to_string};
 use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::thread::sleep;
@@ -550,38 +550,12 @@ fn cdylib_tree_is_well_tested() {
         }));
 }
 
-#[test]
-fn well_tested_tree_quiet() {
-    let tmp_src_dir = copy_of_testdata("well_tested");
-    run()
-        .arg("mutants")
-        .arg("--no-times")
-        .arg("--no-shuffle")
-        .current_dir(tmp_src_dir.path())
-        .assert()
-        .success()
-        .stdout(predicate::function(|stdout| {
-            insta::assert_snapshot!(stdout);
-            true
-        }));
-    let outcomes_json =
-        fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap();
-    println!("outcomes.json:\n{outcomes_json}");
-    let outcomes: serde_json::Value = outcomes_json.parse().unwrap();
-    assert_eq!(outcomes["total_mutants"], 50);
-    assert_eq!(outcomes["caught"], 50);
-    assert_eq!(outcomes["unviable"], 0);
-    assert_eq!(outcomes["missed"], 0);
-}
-
 #[test]
 fn well_tested_tree_finds_no_problems() {
     let tmp_src_dir = copy_of_testdata("well_tested");
     run()
         .arg("mutants")
-        .arg("--no-times")
-        .arg("--caught")
-        .arg("--no-shuffle")
+        .args(["--no-times", "--caught", "--unviable", "--no-shuffle"])
         .current_dir(tmp_src_dir.path())
         .assert()
         .success()
@@ -593,6 +567,17 @@ fn well_tested_tree_finds_no_problems() {
         .path()
         .join("mutants.out/outcomes.json")
         .exists());
+    let outcomes_json =
+        fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap();
+    let outcomes: serde_json::Value = outcomes_json.parse().unwrap();
+    let caught = outcomes["caught"]
+        .as_i64()
+        .expect("outcomes['caught'] is an integer");
+    assert!(caught > 40, "expected more outcomes caught than {caught}");
+    assert_eq!(outcomes["unviable"], 0);
+    assert_eq!(outcomes["missed"], 0);
+    assert_eq!(outcomes["timeout"], 0);
+    assert_eq!(outcomes["total_mutants"], outcomes["caught"]);
     check_text_list_output(tmp_src_dir.path(), "well_tested_tree_finds_no_problems");
 }
 
@@ -1129,7 +1114,18 @@ fn mutants_causing_tests_to_hang_are_stopped_by_manual_timeout() {
         .stdout(contains(
             "replace controlled_loop -> usize with 0 ... caught",
         ));
-    // TODO: Inspect outcomes.json.
+    let timeout_txt = read_to_string(tmp_src_dir.path().join("mutants.out/timeout.txt"))
+        .expect("read timeout.txt");
+    assert!(
+        timeout_txt.contains("replace should_stop"),
+        "expected text not found in:\n{timeout_txt}"
+    );
+    let outcomes_json: serde_json::Value =
+        read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json"))
+            .expect("read outcomes.json")
+            .parse()
+            .expect("parse outcomes.json");
+    assert_eq!(outcomes_json["timeout"], 1);
 }
 
 #[test]