Add --test-tool=nextest (#223)

.github/workflows/tests.yml

@@ -42,6 +42,10 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       - name: rustfmt
         run: cargo fmt --all -- --check
+      - uses: taiki-e/install-action@v2
+        name: Install nextest using install-action
+        with:
+          tool: nextest
       - name: Build
         run: cargo build --all-targets
       - name: Test
@@ -70,6 +74,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: [release-binary]
     if: github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        test_tool: [cargo, nextest]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -82,16 +89,21 @@ jobs:
         with:
           toolchain: beta
       - uses: Swatinem/rust-cache@v2
-      - name: Download release binary
+      - uses: taiki-e/install-action@v2
+        name: Install nextest using install-action
+        with:
+          tool: nextest
+      - name: Download cargo-mutants binary
         uses: actions/download-artifact@v3
         with:
           name: cargo-mutants-linux
-      - name: Install binary artifact
+      - name: Install cargo-mutants binary
         run: |
           install cargo-mutants $HOME/.cargo/bin/
-      - name: Mutants
-        run: |
-          cargo mutants --no-shuffle -vV --in-diff git.diff
+      - name: Mutants in-diff
+        run: >
+          cargo mutants --no-shuffle -vV --in-diff git.diff --test-tool
+          ${{matrix.test_tool}}
       - name: Archive mutants.out
         uses: actions/upload-artifact@v3
         if: always()
@@ -105,22 +117,28 @@ jobs:
     strategy:
       matrix:
         shard: [0, 1, 2, 3, 4, 5, 6, 7]
+        test_tool: [cargo, nextest]
     steps:
       - uses: actions/checkout@v3
       - uses: dtolnay/rust-toolchain@master
         with:
           toolchain: beta
       - uses: Swatinem/rust-cache@v2
-      - name: Download release binary
+      - uses: taiki-e/install-action@v2
+        name: Install nextest using install-action
+        with:
+          tool: nextest
+      - name: Download cargo-mutants binary
         uses: actions/download-artifact@v3
         with:
           name: cargo-mutants-linux
-      - name: Install binary artifact
+      - name: Install cargo-mutants binary
         run: |
           install cargo-mutants $HOME/.cargo/bin/
       - name: Mutants
-        run: |
+        run: >
           cargo mutants --no-shuffle -vV --shard ${{ matrix.shard }}/8
+          --test-tool ${{ matrix.test_tool }}
       - name: Archive mutants.out
         uses: actions/upload-artifact@v3
         if: always()

CONTRIBUTING.md

@@ -33,7 +33,13 @@ Please run `cargo fmt` and `cargo clippy`. These are checked in CI.
 
 ## Testing
 
-Of course, please add tests for new features or bug fixes, and see the _Testing_ section of [the design doc](DESIGN.md).
+Of course, please add tests for new features or bug fixes. See also the _Testing_ section of [the design doc](DESIGN.md).
+
+### Running the tests
+
+cargo-mutants tests require [`cargo-nextest`](https://nexte.st/) to be installed, so that they can exercise `--test-tool=nextest`.
+
+cargo-mutants tests can be run under either `cargo test` or `cargo nextest run`.
 
 ### Test naming
 

Cargo.toml

@@ -4,7 +4,7 @@ version = "23.12.2"
 edition = "2021"
 authors = ["Martin Pool"]
 license = "MIT"
-description = "Find inadequately-tested code that can be removed without any tests failing."
+description = "Inject bugs and see if your tests catch them"
 repository = "https://github.com/sourcefrog/cargo-mutants"
 homepage = "https://mutants.rs/"
 categories = ["development-tools::testing"]
@@ -60,6 +60,7 @@ quote = "1.0"
 regex = "1.10"
 serde_json = "1"
 similar = "2.0"
+strum = { version = "0.25", features = ["derive"] }
 subprocess = "0.2.8"
 tempfile = "3.2"
 time = "0.3"
@@ -72,7 +73,6 @@ whoami = "1.2"
 [dependencies.nutmeg]
 version = "0.1.4"
 # git = "https://github.com/sourcefrog/nutmeg.git"
-# branch = "const-new"
 
 [dependencies.proc-macro2]
 features = ["span-locations"]

DESIGN.md

@@ -194,6 +194,18 @@ Cargo-mutants is primarily tested on its public interface, which is the command
 
 `cargo-mutants` runs as a subprocess of the test process so that we get the most realistic view of its behavior. In some cases it is run via the `cargo` command to test that this level of indirection works properly.
 
+### Test performance
+
+Since cargo-mutants itself runs the test suite of the program under test many times there is a risk that the test suite can get slow. Aside from slowing down developers and CI, this has a multiplicative effect on the time to run `cargo mutants` on itself.
+
+To manage test time:
+
+* Although key behaviour should be tested through integration tests that run the CLI, it's OK to handle additional cases with unit tests that run much faster.
+
+* Whenever reasonable, CLI tests can only list mutants with `--list` rather than actually testing all of them: we have just a small set of tests that check that the mutants that are listed are actually run.
+
+* Use relatively small testdata trees that are sufficient to test the right behavior.
+
 ### `testdata` trees
 
 The primary means of testing is Rust source trees under `testdata`: you can copy an existing tree and modify it to show the new behavior that you want to test.
@@ -214,7 +226,11 @@ Many features can be tested adequately by only looking at the list of mutants pr
 
 ### Unit tests
 
-Although we primarily want to test the public interface (which is the command line), unit tests can be added in a `mod test {}` within the source tree for any behavior that is inconvenient to exercise from the command line.
+Although we primarily want to test the public interface (which is the command line), unit tests can be added in a `mod test {}` within the source tree for any behavior that is inconvenient or overly slow to exercise from the command line.
+
+### Nextest tests
+
+cargo-mutants tests require `nextest` to be installed.
 
 ## UI Style
 

NEWS.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- New! `cargo mutants --test-tool nextest`, or `test_tool = "nextest"` in `.cargo/mutants.toml` runs tests under [Nextest](https://nexte.st/). Some trees have tests that only work under Nextest, and this allows them to be tested. In other cases Nextest may be significantly faster, because it will exit soon after the first test failure.
+
 - Fixed: Fixed spurious "Patch input contains repeated filenames" error when `--in-diff` is given a patch that deletes multiple files.
 
 ## 23.12.2

README.md

@@ -22,12 +22,18 @@ the tests might be insufficient.
 
 **The main documentation is the user guide at <https://mutants.rs/>.**
 
+## Prerequisites
+
+cargo-mutants can help on trees with non-flaky tests that run under `cargo test` or [`cargo nextest run`](https://nexte.st/).
+
 ## Install
 
 ```sh
 cargo install --locked cargo-mutants
 ```
 
+You can also install using [cargo-binstall](https://github.com/cargo-bins/cargo-binstall) or from binaries attached to GitHub releases.
+
 ## Quick start
 
 From within a Rust source directory, just run
@@ -42,8 +48,6 @@ To generate mutants in only one file:
 cargo mutants -f src/something.rs
 ```
 
-**For more, see the user guide is at <https://mutants.rs/>.**
-
 ## Help advance cargo-mutants
 
 If you use cargo-mutants or just like the idea you can help it get better:
@@ -53,18 +57,20 @@ If you use cargo-mutants or just like the idea you can help it get better:
 
 ## Project status
 
-As of October 2023 this is an actively-maintained spare time project. I expect to make [releases](https://github.com/sourcefrog/cargo-mutants/releases) about every one or two months.
+As of January 2024 this is an actively-maintained spare time project. I expect to make [releases](https://github.com/sourcefrog/cargo-mutants/releases) about every one or two months.
 
 It's very usable at it is and there's room for lots more future improvement,
 especially in adding new types of mutation.
 
+This software is provided as-is with no warranty of any kind.
+
 ## Further reading
 
 See also:
 
-- [cargo-mutants manual](https://mutants.rs/)
-- [How cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared).
-- [Design notes](DESIGN.md)
-- [Contributing](CONTRIBUTING.md)
-- [Release notes](NEWS.md)
-- [Discussions](https://github.com/sourcefrog/cargo-mutants/discussions)
+* [cargo-mutants manual](https://mutants.rs/)
+* [How cargo-mutants compares to other techniques and tools](https://github.com/sourcefrog/cargo-mutants/wiki/Compared).
+* [Design notes](DESIGN.md)
+* [Contributing](CONTRIBUTING.md)
+* [Release notes](NEWS.md)
+* [Discussions](https://github.com/sourcefrog/cargo-mutants/discussions)

book/src/SUMMARY.md

@@ -16,6 +16,7 @@
   - [Workspaces and packages](workspaces.md)
   - [Passing options to Cargo](cargo-args.md)
   - [Build directories](build-dirs.md)
+  - [Using nextest](nextest.md)
 - [Generating mutants](mutants.md)
   - [Error values](error-values.md)
 - [Improving performance](performance.md)

book/src/cargo-args.md

@@ -27,7 +27,9 @@ additional_cargo_args = ["--all-features"]
 ## Arguments to `cargo test`
 
 Command-line options following a `--` delimiter are passed through to
-`cargo test`. For example, this can be used to pass `--all-targets` which (unobviously)
+`cargo test` (or to [nextest](nextest.md), if you're using that).
+
+For example, this can be used to pass `--all-targets` which (unobviously)
 excludes doctests. (If the doctests are numerous and slow, and not relied upon to catch bugs, this can improve performance.)
 
 ```shell

book/src/getting-started.md

@@ -3,6 +3,17 @@
 Just run `cargo mutants` in a Rust source directory, and it will point out
 functions that may be inadequately tested.
 
+## Prerequisites
+
+For cargo-mutants to give useful results, your tree must already
+
+1. Be built with `cargo build`, and
+2. Have reliable non-flaky tests that run under either `cargo test` or `cargo nextest`.
+
+If the tests are flaky, meaning that they can pass or fail depending on factors other than the source tree, then the cargo-mutants results will be meaningless.
+
+Cross-compilation is not currently supported, so the tree must be buildable for the host platform.
+
 ## Example
 
 ```none

book/src/installation.md

@@ -6,6 +6,8 @@ Install cargo-mutants from source:
 cargo install --locked cargo-mutants
 ```
 
+You can also use `cargo binstall` from [cargo-binstall](https://github.com/cargo-bins/cargo-binstall), or install binaries from GitHub releases.
+
 ## Supported Rust versions
 
 Building cargo-mutants requires a reasonably recent stable (or nightly or beta) Rust toolchain.

book/src/nextest.md

@@ -0,0 +1,29 @@
+# cargo-mutants with nextest
+
+[nextest](https://nexte.st) is a tool for running Rust tests, as a replacement for `cargo test`.
+
+You can use nextest to run your tests with cargo-mutants, instead of `cargo test`, by either passing the `--test-tool=nextest` option, or setting `test_tool = "nextest"` in `.cargo/mutants.toml`.
+
+## How nextest works
+
+In the context of cargo-mutants the most important difference between cargo-test and nextest is that nextest runs each test in a separate process, and it can run tests from multiple test targets in parallel. (Nextest also has some nice UI improvements and other features, but they're not relevant here.)
+
+This means that nextest can stop faster if a single test fails, whereas cargo test will continue running all the tests within the test binary.
+
+This is beneficial for cargo-mutants, because it only needs to know whether at least one test caught the mutation, and so exiting as soon as there's a failure is better.
+
+However, running each test individually also means there is more per-test startup cost, and so on some trees nextest may be slower.
+
+In general, nextest will do relatively poorly on trees that have tests that are individually very fast, or on trees that establish shared or cached state across tests.
+
+## When to use nextest
+
+There are at least two reasons why you might want to use nextest:
+
+1. Some Rust source trees only support testing under nextest, and their tests fail under `cargo test`: in that case, you have to use this option! In particular, nextest's behavior of running each test in a separate process gives better isolation between tests.
+
+2. Some trees might be faster under nextest than under `cargo test`, because they have a lot of tests that fail quickly, and the startup time is a small fraction of the time for the average test. This may or may not be true for your tree, so you can try it and see. Some trees, including cargo-mutants itself, are slower under nextest.
+
+## nextest and doctests
+
+**Caution:** [nextest currently does not run doctests](https://github.com/nextest-rs/nextest/issues/16), so behaviors that are only caught by doctests will show as missed when using nextest. (cargo-mutants could separately run the doctests, but currently does not.)

book/src/shards.md

@@ -87,3 +87,7 @@ If your CI system offers a choice of VM sizes you might experiment with using sm
 You should also think about cost and capacity constraints in your CI system, and the risk of starving out other users.
 
 cargo-mutants has no internal scaling constraints to prevent you from setting `k` very large, if cost, efficiency and CI capacity are not a concern.
+
+## Sampling mutants
+
+An option like `--shard 1/100` can be used to run 1% of all the generated mutants for testing cargo-mutants, to get a sense of whether it works or to see how it performs on some tree.

src/cargo.rs

@@ -1,4 +1,4 @@
-// Copyright 2021-2023 Martin Pool
+// Copyright 2021-2024 Martin Pool
 
 //! Run Cargo as a subprocess, including timeouts and propagating signals.
 
@@ -10,6 +10,7 @@ use camino::Utf8Path;
 use itertools::Itertools;
 use tracing::{debug, debug_span};
 
+use crate::options::TestTool;
 use crate::outcome::PhaseResult;
 use crate::package::Package;
 use crate::process::Process;
@@ -63,8 +64,17 @@ fn cargo_argv(
     phase: Phase,
     options: &Options,
 ) -> Vec<String> {
-    let mut cargo_args = vec![cargo_bin(), phase.name().to_string()];
-    if phase == Phase::Check || phase == Phase::Build {
+    let mut cargo_args = vec![cargo_bin()];
+    if phase == Phase::Test {
+        match &options.test_tool {
+            TestTool::Cargo => cargo_args.push("test".to_string()),
+            TestTool::Nextest => {
+                cargo_args.push("nextest".to_string());
+                cargo_args.push("run".to_string());
+            }
+        }
+    } else {
+        cargo_args.push(phase.name().to_string());
         cargo_args.push("--tests".to_string());
     }
     if let Some([package]) = packages {