nordisk/gunnar

xtask/tests/exit_code_is_the_verdict.rs

//! **`cargo xtask` must never print `FAILED` and return 0.**
//!
//! `run_installtest` spawns the detached `installtest/` crate and, when it comes
//! back non-zero, prints
//!
//! ```text
//! bare: FAILED (exit status: 1)
//! ```
//!
//! Nothing asserted what the *process* then returned. On 2026-08-04 that line
//! was observed next to `EXIT=0` and read as a bug in this wrapper; it was not —
//! the measurement was `… 2>&1 | tail`, and `$?` in a pipeline belongs to
//! `tail`. But the repository had no way to tell those two apart, and a wrapper
//! whose exit code nothing checks is one careless edit from being the bug it was
//! accused of being.
//!
//! # How this stays hermetic
//!
//! `run_installtest` resolves its build tool from `$CARGO`. Pointing that at
//! `/bin/false` makes the spawned command fail *deterministically*, in under a
//! millisecond, without compiling `installtest/` or its draupnir + tunnr + QEMU
//! tree — so this guard is a unit test in cost and an end-to-end one in reach.
//! `/bin/true` gives the matching green.
//!
//! Zero dependencies, like the rest of `xtask`.

use std::process::{Command, Output};

const EXE: &str = env!("CARGO_BIN_EXE_xtask");

fn xtask(cargo: &str, args: &[&str]) -> Output {
    Command::new(EXE)
        .args(args)
        // The tool `run_installtest` will spawn instead of the real cargo.
        .env("CARGO", cargo)
        // Never let the ambient target dir matter; nothing is built here anyway.
        .env(
            "GUNNAR_INSTALLTEST_TARGET_DIR",
            "$HOME/scratch/cargo/xtask-exitcode-guard",
        )
        .output()
        .expect("spawn xtask")
}

fn code(out: &Output) -> i32 {
    // A signal death is neither pass nor fail; -1 is not a code any path returns.
    out.status.code().unwrap_or(-1)
}

fn stderr(out: &Output) -> String {
    String::from_utf8_lossy(&out.stderr).into_owned()
}

/// The reported shape, asserted on the number rather than the word: when the
/// spawned run fails, `xtask` must say so **and** exit non-zero.
///
/// RED-when-broken: change `run_installtest`'s `Ok(s)` arm to
/// `ExitCode::SUCCESS` and this fires while the printed text is unchanged —
/// which is precisely the failure mode no existing test could see.
#[test]
fn a_failed_run_prints_failed_and_exits_non_zero() {
    for task in ["bare", "install-test"] {
        let out = xtask("/bin/false", &[task]);
        let err = stderr(&out);
        assert!(
            err.contains(&format!("{task}: FAILED")),
            "`xtask {task}` must report the failure in words:\n{err}"
        );
        assert_eq!(
            code(&out),
            1,
            "…and the EXIT CODE must agree with those words. Printing `FAILED` \
             and returning 0 is the bug this test exists for.\nstderr:\n{err}"
        );
    }
}

/// The pair. Without it, the assertion above is satisfied by an `xtask` that
/// fails at everything — the identity-value trap.
///
/// `/bin/true` stands in for a run that passed: the wrapper must then be silent
/// about failure and return 0.
#[test]
fn a_successful_run_says_nothing_about_failure_and_exits_zero() {
    for task in ["bare", "install-test"] {
        let out = xtask("/bin/true", &[task]);
        let err = stderr(&out);
        assert_eq!(
            code(&out),
            0,
            "`xtask {task}` over a run that succeeded must exit 0, or the red \
             above proves nothing.\nstderr:\n{err}"
        );
        assert!(
            !err.contains("FAILED"),
            "…and must not claim a failure that did not happen:\n{err}"
        );
    }
}

/// The spawn itself failing — `$CARGO` naming something that is not there — is a
/// third outcome, and it must not be quietly folded into either of the other
/// two.
#[test]
fn a_tool_that_cannot_be_spawned_is_its_own_failure_not_a_pass() {
    let out = xtask("/nonexistent/definitely-not-cargo", &["bare"]);
    let err = stderr(&out);
    assert_eq!(
        code(&out),
        1,
        "a cargo that cannot be spawned must exit non-zero.\nstderr:\n{err}"
    );
    assert!(
        err.contains("could not run"),
        "and must be distinguishable from a run that executed and failed:\n{err}"
    );
    assert!(
        !err.contains("FAILED ("),
        "specifically, it must NOT be reported as a completed run that failed:\n{err}"
    );
}

/// `xtask` with no task, and with an unknown one, must also exit non-zero rather
/// than printing usage and reporting success.
#[test]
fn no_task_and_an_unknown_task_both_exit_non_zero() {
    let none = xtask("/bin/true", &[]);
    assert_eq!(
        code(&none),
        1,
        "bare `cargo xtask` must exit 1:\n{}",
        stderr(&none)
    );

    let unknown = xtask("/bin/true", &["definitely-not-a-task"]);
    assert_eq!(
        code(&unknown),
        1,
        "an unknown task must exit 1:\n{}",
        stderr(&unknown)
    );
    assert!(
        stderr(&unknown).contains("unknown task"),
        "and must say which:\n{}",
        stderr(&unknown)
    );
}