From af77583ceca45a1a31a937f6f481c93b37c33191 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 10 Dec 2023 19:05:37 -0800 Subject: [PATCH 1/4] Combine two slower tests --- tests/cli/main.rs | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/tests/cli/main.rs b/tests/cli/main.rs index df584ee2..3a2d3156 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -550,38 +550,12 @@ fn cdylib_tree_is_well_tested() { })); } -#[test] -fn well_tested_tree_quiet() { - let tmp_src_dir = copy_of_testdata("well_tested"); - run() - .arg("mutants") - .arg("--no-times") - .arg("--no-shuffle") - .current_dir(tmp_src_dir.path()) - .assert() - .success() - .stdout(predicate::function(|stdout| { - insta::assert_snapshot!(stdout); - true - })); - let outcomes_json = - fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap(); - println!("outcomes.json:\n{outcomes_json}"); - let outcomes: serde_json::Value = outcomes_json.parse().unwrap(); - assert_eq!(outcomes["total_mutants"], 50); - assert_eq!(outcomes["caught"], 50); - assert_eq!(outcomes["unviable"], 0); - assert_eq!(outcomes["missed"], 0); -} - #[test] fn well_tested_tree_finds_no_problems() { let tmp_src_dir = copy_of_testdata("well_tested"); run() .arg("mutants") - .arg("--no-times") - .arg("--caught") - .arg("--no-shuffle") + .args(["--no-times", "--caught", "--unviable", "--no-shuffle"]) .current_dir(tmp_src_dir.path()) .assert() .success() @@ -593,6 +567,17 @@ fn well_tested_tree_finds_no_problems() { .path() .join("mutants.out/outcomes.json") .exists()); + let outcomes_json = + fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap(); + let outcomes: serde_json::Value = outcomes_json.parse().unwrap(); + let caught = outcomes["caught"] + .as_i64() + .expect("outcomes['caught'] is an integer"); + assert!(caught > 40, "expected more outcomes caught than {caught}"); + assert_eq!(outcomes["unviable"], 0); + assert_eq!(outcomes["missed"], 0); + assert_eq!(outcomes["timeout"], 0); + assert_eq!(outcomes["total_mutants"], outcomes["caught"]); check_text_list_output(tmp_src_dir.path(), "well_tested_tree_finds_no_problems"); } From f829110ab539ed878f2960097515e2f5b12a43ee Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 10 Dec 2023 19:10:18 -0800 Subject: [PATCH 2/4] Run the jobs test on a smaller tree for speed --- tests/cli/jobs.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/cli/jobs.rs b/tests/cli/jobs.rs index deca4090..401b8058 100644 --- a/tests/cli/jobs.rs +++ b/tests/cli/jobs.rs @@ -5,15 +5,16 @@ use super::{copy_of_testdata, run}; /// It's a bit hard to assess that multiple jobs really ran in parallel, -/// but we can at least check that the option is accepted. +/// but we can at least check that the option is accepted, and see that the +/// debug log looks like it's using multiple threads. #[test] fn jobs_option_accepted() { - let testdata = copy_of_testdata("well_tested"); + let testdata = copy_of_testdata("small_well_tested"); run() .arg("mutants") .arg("-d") .arg(testdata.path()) - .arg("-j3") + .arg("-j2") .arg("--minimum-test-timeout=120") // to avoid flakes on slow CI .assert() .success(); From fcbe43867824cd4c8285e4c48649095bf5c673b9 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 10 Dec 2023 19:16:38 -0800 Subject: [PATCH 3/4] Look in the debug log for ThreadId to test -j --- tests/cli/jobs.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/cli/jobs.rs b/tests/cli/jobs.rs index 401b8058..afccdcf3 100644 --- a/tests/cli/jobs.rs +++ b/tests/cli/jobs.rs @@ -2,13 +2,18 @@ //! Test handling of `--jobs` concurrency option. +use std::fs::read_to_string; + +use itertools::Itertools; +use regex::Regex; + use super::{copy_of_testdata, run}; /// It's a bit hard to assess that multiple jobs really ran in parallel, /// but we can at least check that the option is accepted, and see that the /// debug log looks like it's using multiple threads. #[test] -fn jobs_option_accepted() { +fn jobs_option_accepted_and_causes_multiple_threads() { let testdata = copy_of_testdata("small_well_tested"); run() .arg("mutants") @@ -18,4 +23,22 @@ fn jobs_option_accepted() { .arg("--minimum-test-timeout=120") // to avoid flakes on slow CI .assert() .success(); + let debug_log = + read_to_string(testdata.path().join("mutants.out/debug.log")).expect("read debug log"); + println!("debug log:\n{debug_log}"); + // This might be brittle, as the ThreadId debug form is not specified, and + // also _possibly_ everything will complete on one thread before the next + // gets going, though that seems unlikely. + let re = Regex::new(r#"\{thread=ThreadId\(\d+\)\}"#).expect("compile regex"); + let matches = re + .find_iter(&debug_log) + .map(|m| m.as_str()) + .unique() + .collect::>(); + println!("threadid matches: {matches:?}"); + assert!( + matches.len() > 1, + "expected more than {} thread ids in debug log", + matches.len() + ); } From 33845ba46bea53da8cfd6fc33305ae90c735fea8 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 10 Dec 2023 19:24:55 -0800 Subject: [PATCH 4/4] Improved timeout test --- tests/cli/main.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/cli/main.rs b/tests/cli/main.rs index 3a2d3156..c2697416 100644 --- a/tests/cli/main.rs +++ b/tests/cli/main.rs @@ -5,7 +5,7 @@ use std::borrow::Borrow; use std::env; use std::fmt::Write; -use std::fs::{self, read_dir}; +use std::fs::{self, read_dir, read_to_string}; use std::io::Read; use std::path::{Path, PathBuf}; use std::thread::sleep; @@ -1114,7 +1114,18 @@ fn mutants_causing_tests_to_hang_are_stopped_by_manual_timeout() { .stdout(contains( "replace controlled_loop -> usize with 0 ... caught", )); - // TODO: Inspect outcomes.json. + let timeout_txt = read_to_string(tmp_src_dir.path().join("mutants.out/timeout.txt")) + .expect("read timeout.txt"); + assert!( + timeout_txt.contains("replace should_stop"), + "expected text not found in:\n{timeout_txt}" + ); + let outcomes_json: serde_json::Value = + read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")) + .expect("read outcomes.json") + .parse() + .expect("parse outcomes.json"); + assert_eq!(outcomes_json["timeout"], 1); } #[test]