nordisk/edda

crates/nornir-bench/tests/failed_arm_exits_nonzero.rs

//! The exit status of a bench run, asserted on a **real process**.
//!
//! Until 2026-08-06 `nornir-bench` wrote `"passed": false` into its JSON envelope
//! and exited **0**. The JSON was correct; the status was a lie. Every caller that
//! judges a subprocess the way a shell does — `&&`, `set -e`, a CI step, a
//! `Command::status()` — read a bench run with red arms as a clean pass. It is
//! written down as a standing hazard in the constellation's own `PRATA.md`, and it
//! still bit `gunnar.multi_pack_serve` on the day that arm was added to catch a
//! pack-corruption bug.
//!
//! A unit test on [`nornir_bench::api::verdict`] cannot close this. `verdict`
//! returning `Err` and the process exiting non-zero are two different facts, and
//! the whole defect lived in the gap between them. So this drives the real binary,
//! `nornir-bench-redprobe`, whose arms fail on demand, and reads `ExitStatus`.
//!
//! `CARGO_BIN_EXE_nornir-bench-redprobe` is set by cargo for an integration test
//! in the package that declares the bin, so there is no nested `cargo run` here:
//! no build lock to contend for, and no chance of measuring a stale binary from
//! some other target directory.

use std::process::{Command, Output};

/// Run the probe with `NORNIR_BENCH_REDPROBE=<mode>` (empty ⇒ unset).
fn probe(mode: &str) -> Output {
    let mut cmd = Command::new(env!("CARGO_BIN_EXE_nornir-bench-redprobe"));
    // Neutralise whatever the ambient environment says, so this test measures the
    // probe and not the shell it was started from.
    cmd.env_remove("NORNIR_BENCH_ONLY")
        .env_remove("NORNIR_BENCH_SKIP")
        .env_remove("NORNIR_BENCH_STOP_ON_ERROR")
        .env_remove("NORNIR_BENCH_PREVIEW")
        .env_remove("NORNIR_BENCH_SINGLE_CORE");
    if mode.is_empty() {
        cmd.env_remove("NORNIR_BENCH_REDPROBE");
    } else {
        cmd.env("NORNIR_BENCH_REDPROBE", mode);
    }
    cmd.output().expect("run nornir-bench-redprobe")
}

/// The last non-empty stdout line, parsed as a `BenchRun` — the exact contract
/// `runner::run_bench_example` reads.
fn envelope(out: &Output) -> nornir_bench::BenchRun {
    let stdout = String::from_utf8_lossy(&out.stdout);
    let line = stdout
        .lines()
        .filter(|l| !l.trim().is_empty())
        .next_back()
        .unwrap_or_else(|| {
            panic!(
                "no stdout at all. stderr:\n{}",
                String::from_utf8_lossy(&out.stderr)
            )
        });
    serde_json::from_str(line).unwrap_or_else(|e| panic!("parse `{line}` as BenchRun: {e}"))
}

/// **The green half.** Every arm passes ⇒ exit 0. Without this the test above
/// could be satisfied by a `verdict` that failed unconditionally, which would
/// pass a red-only assertion while breaking every bench in the constellation.
#[test]
fn an_all_green_run_exits_zero() {
    let out = probe("");
    let run = envelope(&out);
    assert_eq!(
        run.tests.len(),
        2,
        "both probe arms must have reported: {:?}",
        run.tests
    );
    assert!(
        run.all_tests_passed(),
        "no arm was told to fail: {:?}",
        run.failed_tests()
    );
    assert_eq!(
        out.status.code(),
        Some(0),
        "an all-green bench must exit 0. stderr:\n{}",
        String::from_utf8_lossy(&out.stderr)
    );
}

/// **The red half — the defect.** One arm fails ⇒ non-zero exit, and the envelope
/// is STILL on stdout so nothing that parses it loses its evidence.
///
/// Seen red on 2026-08-06 by restoring the old ending of `run_main_json`
/// (`println!(…); Ok(())`): this test then reported `exit 0 while
/// redprobe.red carried "passed": false`, and `an_all_green_run_exits_zero`
/// stayed green — i.e. the probe distinguishes the two builds rather than
/// failing on both.
#[test]
fn a_failed_arm_exits_nonzero_and_still_prints_its_envelope() {
    let out = probe("red");
    let run = envelope(&out);

    // The evidence survives the failure. This is the half that makes the fix safe
    // to land: `runner::run_bench_example`, the release gate and the warehouse all
    // read this line, and they must keep reading it on a red run.
    assert_eq!(
        run.tests.len(),
        2,
        "both arms must still be reported: {:?}",
        run.tests
    );
    assert_eq!(
        run.failed_tests(),
        vec!["redprobe.red"],
        "exactly the arm that was told to fail: {:?}",
        run.tests
    );

    // …and the status is now honest.
    assert_ne!(
        out.status.code(),
        Some(0),
        "a bench run carrying `\"passed\": false` MUST NOT exit 0 — that is the whole \
         standing hazard. envelope was: {}",
        String::from_utf8_lossy(&out.stdout)
    );

    let stderr = String::from_utf8_lossy(&out.stderr);
    assert!(
        stderr.contains("redprobe.red"),
        "the failure must name the arm on stderr, or an operator watching a CI log \
         sees a non-zero exit with no subject:\n{stderr}"
    );
}

/// Both arms red ⇒ still non-zero, and both are named. A message that stopped at
/// the first failure would hide how much is broken.
#[test]
fn every_failed_arm_is_named_on_stderr() {
    let out = probe("all");
    let run = envelope(&out);
    assert_eq!(
        run.failed_tests(),
        vec!["redprobe.green", "redprobe.red"],
        "{:?}",
        run.tests
    );
    assert_ne!(out.status.code(), Some(0));
    let stderr = String::from_utf8_lossy(&out.stderr);
    assert!(stderr.contains("redprobe.green"), "{stderr}");
    assert!(stderr.contains("redprobe.red"), "{stderr}");
    assert!(
        stderr.contains("2 of 2"),
        "the count must be a ratio: {stderr}"
    );
}