crates/nornir-xtask-support/src/ci.rs
//! # `cargo xtask ci`: the one CI plan runner of the constellation
//!
//! Every repo's `cargo xtask ci` is a short list of [`Step`]s handed to [`main`]. This
//! module runs them one after the other, writes each step's output to its own log,
//! and writes ONE JSON receipt per run (like `installtest` and `gate` do). A
//! developer runs exactly the same verb before pushing that the CI runner on
//! builder-64 runs after the push; the runner holds no logic of its own.
//!
//! ## Modes
//!
//! | flag | mode | runs |
//! |---|---|---|
//! | `--quick` | [`Mode::Quick`] | the per-push set |
//! | (none) | [`Mode::Full`] | quick plus the slower steps |
//! | `--nightly` | [`Mode::Nightly`] | full plus the benches and the gates |
//!
//! A step declares the least mode it runs in ([`Step::from`]); a mode runs every step
//! whose least mode is at or below it.
//!
//! ## Where a step runs
//!
//! A [`Place::Toolchain`] step runs in the toolchain container
//! ([`CI_IMAGE`], the CI test image pinned by digest, unless `NORNIR_CI_IMAGE` says otherwise),
//! with the repo's PARENT mounted at `/src`, so the sibling checkouts resolve as they do
//! on the host. A [`Place::Host`] step runs on the host: a verb that enters the
//! container itself (holger's `cargo xtask build`, through the Build Thing) must not be
//! nested. When `cargo xtask ci` already runs inside a container, or
//! `NORNIR_CI_HOST=1` is set, toolchain steps run directly.
//!
//! The environment the runner may set, all optional:
//!
//! * `NORNIR_CI_OUT`: the directory the run's logs and receipt go to (else
//! `<target>/ci/<repo>-<rev>`);
//! * `NORNIR_CI_IMAGE`, `NORNIR_CI_ENGINE` (`podman`);
//! * `NORNIR_CI_AS_ROOT=1`: run toolchain steps as the namespace's root. By default a
//! podman step runs as the invoking user (`--userns=keep-id`, home `/tmp`), with
//! `safe.directory=*` and the host's `/etc/machine-id` mounted read-only;
//! * `NORNIR_CI_REGISTRY`: the cargo registry to mount (else `$CARGO_HOME/registry`);
//! * `NORNIR_CI_FAST`: a directory holding `sccache`, `mold` and `env.sh` (builder-64's
//! `/work/tools`), mounted at `/tools` with `--env-file env.sh`, and
//! `NORNIR_CI_SCCACHE`, the cache mounted at `/sccache`;
//! * `CARGO_BUILD_JOBS`, passed through; the container's target is
//! `$CARGO_TARGET_DIR/toolchain` (else `<repo>/target/toolchain`), or
//! `NORNIR_CI_TOOLCHAIN_TARGET`.
//!
//! std only, like the rest of this crate: the receipt is written by hand.
use std::fs::File;
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
use crate::Result;
/// The receipt's schema tag.
pub const SCHEMA: &str = "nornir-ci/1";
/// The RELEASE toolchain (the recipe): pinned by DIGEST, the one edda's
/// `.nornir/build-variants.toml` pins (and lockcheck holds every sibling to), never a tag,
/// which names another image on each box and after each re-mint. A toolchain CI step runs in
/// [`CI_IMAGE`], built on it.
pub const DEFAULT_IMAGE: &str = "localhost/nordisk-cargo-toolchain@sha256:86d575e874677da3eb9aff3ba6e38ca600781ee9b7948a9df43ef89a6213c762";
/// The image a [`Place::Toolchain`] step runs in by default: the CI TEST image, pinned by
/// DIGEST, the one dwarves' `ci/runner.sh` names (`CI_IMAGE`). It is [`DEFAULT_IMAGE`]'s recipe
/// (`recipe-53fec13a5c1c7f2c`) plus what a workspace test needs and a release row does not:
/// OpenSSL headers, clippy and rustfmt for 1.97.1, openssh-client, strace and git-lfs
/// (dwarves' `ci/ci-toolchain.Containerfile`, tag `ci4-53fec13a5c1c7f2c`). A box gets it by
/// `podman load -i ci4-53fec13a5c1c7f2c.oci.tar`, which keeps the digest; a rebuild does not.
pub const CI_IMAGE: &str = "localhost/nordisk-cargo-toolchain@sha256:875a7a325dfe7f6b838535d085a2a502e8ad250076c947f6d023b1e224f03334";
/// Pure: the image a toolchain step runs in, `NORNIR_CI_IMAGE` (`env`) when set and not
/// empty, else [`CI_IMAGE`].
pub fn toolchain_image(env: Option<String>) -> String {
env.filter(|v| !v.is_empty()).unwrap_or_else(|| CI_IMAGE.into())
}
/// How much of the CI a run does.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Mode {
/// The per-push set.
Quick,
/// Quick plus the slower steps (the default).
Full,
/// Full plus the benches and the gates.
Nightly,
}
impl Mode {
/// `--nightly` wins over `--quick`; neither is [`Mode::Full`].
pub fn from_args(args: &[String]) -> Mode {
if args.iter().any(|a| a == "--nightly") {
Mode::Nightly
} else if args.iter().any(|a| a == "--quick") {
Mode::Quick
} else {
Mode::Full
}
}
/// The mode's name in the receipt.
pub fn name(self) -> &'static str {
match self {
Mode::Quick => "quick",
Mode::Full => "full",
Mode::Nightly => "nightly",
}
}
}
/// Where a step runs.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Place {
/// On the host.
Host,
/// In the toolchain container.
Toolchain,
/// In this process (a check written in Rust, such as [`crate::lockcheck`]).
InProcess,
}
impl Place {
fn name(self) -> &'static str {
match self {
Place::Host => "host",
Place::Toolchain => "toolchain",
Place::InProcess => "in-process",
}
}
}
/// What an in-process step says when it did not fail: `None` passed, `Some(why)`
/// was skipped for that reason.
pub type Skip = Option<String>;
/// An in-process check: it writes its report to the log it is handed.
pub type Check = Box<dyn Fn(&mut dyn Write) -> Result<Skip>>;
enum Action {
Command { program: String, args: Vec<String> },
InProcess(Check),
}
/// One step of a repo's CI plan.
pub struct Step {
label: String,
place: Place,
from: Mode,
timeout_secs: Option<u64>,
dir: Option<String>,
env: Vec<(String, String)>,
action: Action,
}
impl Step {
/// `cargo <args>` in the toolchain container, from the quick set on.
pub fn cargo(label: &str, args: &[&str]) -> Step {
Step::command(label, Place::Toolchain, "cargo", args)
}
/// `<program> <args>` on the host, from the quick set on.
pub fn host(label: &str, program: &str, args: &[&str]) -> Step {
Step::command(label, Place::Host, program, args)
}
/// `<program> <args>` in the toolchain container, from the quick set on.
pub fn toolchain(label: &str, program: &str, args: &[&str]) -> Step {
Step::command(label, Place::Toolchain, program, args)
}
/// A check run in this process.
pub fn in_process(label: &str, check: impl Fn(&mut dyn Write) -> Result<Skip> + 'static) -> Step {
Step {
label: label.into(),
place: Place::InProcess,
from: Mode::Quick,
timeout_secs: None,
dir: None,
env: Vec::new(),
action: Action::InProcess(Box::new(check)),
}
}
fn command(label: &str, place: Place, program: &str, args: &[&str]) -> Step {
Step {
label: label.into(),
place,
from: Mode::Quick,
timeout_secs: None,
dir: None,
env: Vec::new(),
action: Action::Command {
program: program.into(),
args: args.iter().map(|a| a.to_string()).collect(),
},
}
}
/// The least mode this step runs in.
pub fn from(mut self, mode: Mode) -> Step {
self.from = mode;
self
}
/// Stop the step after `secs` (else `NORNIR_CI_STEP_TIMEOUT`, else one hour): a
/// hung test must cost one step, not the run.
pub fn timeout(mut self, secs: u64) -> Step {
self.timeout_secs = Some(secs);
self
}
/// Run in this directory, relative to the repo root.
pub fn dir(mut self, rel: &str) -> Step {
self.dir = Some(rel.into());
self
}
/// Set one environment variable for this step.
pub fn env(mut self, key: &str, value: &str) -> Step {
self.env.push((key.into(), value.into()));
self
}
/// The step's label.
pub fn label(&self) -> &str {
&self.label
}
/// Does this step run in `mode`?
pub fn runs_in(&self, mode: Mode) -> bool {
self.from <= mode
}
/// The step as one line: what it runs and where.
pub fn describe(&self) -> String {
let what = match &self.action {
Action::Command { program, args } => {
let mut s = program.clone();
for a in args {
s.push(' ');
s.push_str(a);
}
s
}
Action::InProcess(_) => "(in process)".into(),
};
match &self.dir {
Some(d) => format!("[{}] {what} (in {d})", self.place.name()),
None => format!("[{}] {what}", self.place.name()),
}
}
}
/// A repo's CI plan.
pub struct Plan {
/// The repo's name in the receipt (`holger`, `edda`, ...).
pub repo: String,
/// The repo's root.
pub root: PathBuf,
/// The steps, in order.
pub steps: Vec<Step>,
}
/// One step's result.
#[derive(Debug, Clone, PartialEq)]
pub struct StepResult {
pub label: String,
pub place: Place,
pub command: String,
/// `pass`, `fail` or `skip`.
pub status: &'static str,
pub exit_code: Option<i32>,
pub secs: f64,
pub log: PathBuf,
/// The tests the log names as failed.
pub failed_tests: Vec<String>,
/// The first error line of the log, or the skip reason.
pub note: String,
}
/// One run's receipt.
#[derive(Debug, Clone, PartialEq)]
pub struct Receipt {
pub repo: String,
pub head: String,
pub dirty: bool,
pub mode: Mode,
pub host: String,
pub started_unix: u64,
pub finished_unix: u64,
pub steps: Vec<StepResult>,
pub path: PathBuf,
}
impl Receipt {
/// Green when no step failed.
pub fn green(&self) -> bool {
self.steps.iter().all(|s| s.status != "fail")
}
/// The first failing step.
pub fn failing_step(&self) -> Option<&StepResult> {
self.steps.iter().find(|s| s.status == "fail")
}
/// The receipt as JSON.
pub fn to_json(&self) -> String {
let mut o = String::from("{\n");
o += &format!(" \"schema\": {},\n", json_str(SCHEMA));
o += &format!(" \"repo\": {},\n", json_str(&self.repo));
o += &format!(" \"head\": {},\n", json_str(&self.head));
o += &format!(" \"dirty\": {},\n", self.dirty);
o += &format!(" \"mode\": {},\n", json_str(self.mode.name()));
o += &format!(" \"host\": {},\n", json_str(&self.host));
o += &format!(" \"started_unix\": {},\n", self.started_unix);
o += &format!(" \"finished_unix\": {},\n", self.finished_unix);
o += &format!(" \"green\": {},\n", self.green());
match self.failing_step() {
Some(s) => {
o += &format!(" \"failing_step\": {},\n", json_str(&s.label));
let detail = if s.failed_tests.is_empty() { s.note.clone() } else { s.failed_tests.join(", ") };
o += &format!(" \"failing_detail\": {},\n", json_str(&detail));
o += &format!(" \"failing_log\": {},\n", json_str(&s.log.display().to_string()));
}
None => o += " \"failing_step\": null,\n",
}
o += " \"steps\": [";
for (i, s) in self.steps.iter().enumerate() {
o += if i == 0 { "\n" } else { ",\n" };
o += " {";
o += &format!("\"label\": {}, ", json_str(&s.label));
o += &format!("\"place\": {}, ", json_str(s.place.name()));
o += &format!("\"command\": {}, ", json_str(&s.command));
o += &format!("\"status\": {}, ", json_str(s.status));
match s.exit_code {
Some(c) => o += &format!("\"exit_code\": {c}, "),
None => o += "\"exit_code\": null, ",
}
o += &format!("\"secs\": {:.1}, ", s.secs);
o += &format!("\"log\": {}, ", json_str(&s.log.display().to_string()));
o += "\"failed_tests\": [";
o += &s.failed_tests.iter().map(|t| json_str(t)).collect::<Vec<_>>().join(", ");
o += "], ";
o += &format!("\"note\": {}", json_str(&s.note));
o += "}";
}
o += "\n ]\n}\n";
o
}
}
/// A JSON string literal for `s`.
pub fn json_str(s: &str) -> String {
let mut o = String::with_capacity(s.len() + 2);
o.push('"');
for c in s.chars() {
match c {
'"' => o.push_str("\\\""),
'\\' => o.push_str("\\\\"),
'\n' => o.push_str("\\n"),
'\r' => o.push_str("\\r"),
'\t' => o.push_str("\\t"),
c if (c as u32) < 0x20 => o.push_str(&format!("\\u{:04x}", c as u32)),
c => o.push(c),
}
}
o.push('"');
o
}
/// The tests a cargo test (libtest or nextest) log names as failed, in order, once each.
pub fn failed_tests(log: &str) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
let mut push = |t: &str| {
let t = t.trim();
if !t.is_empty() && !out.iter().any(|o| o == t) {
out.push(t.to_string());
}
};
for line in log.lines() {
let l = line.trim_end();
if let Some(rest) = l.strip_prefix("test ") {
if let Some(name) = rest.strip_suffix(" ... FAILED") {
push(name);
}
} else if let Some(rest) = l.trim_start().strip_prefix("FAIL [") {
// nextest: `FAIL [ 0.012s] crate::bin some::test`
if let Some((_, after)) = rest.split_once(']') {
push(after);
}
}
}
out
}
/// The first line of a log that reads as the reason it failed.
pub fn first_error(log: &str) -> String {
let lines: Vec<&str> = log.lines().collect();
for l in &lines {
let t = l.trim_start();
if t.starts_with("error[") || t.starts_with("error:") || t.starts_with("thread '") && t.contains("panicked") {
return t.chars().take(300).collect();
}
}
lines.iter().rev().find(|l| !l.trim().is_empty()).map(|l| l.trim().chars().take(300).collect()).unwrap_or_default()
}
/// What a toolchain step's container needs.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Container {
pub engine: String,
pub image: String,
/// The repo's parent, mounted at `/src`.
pub parent: PathBuf,
/// The repo's directory name under the parent.
pub repo_dir: String,
/// The host target dir, mounted at `/target`.
pub target: PathBuf,
pub registry: Option<PathBuf>,
pub jobs: Option<String>,
pub fast: Option<PathBuf>,
pub sccache: Option<PathBuf>,
/// Run as the invoking user (podman `--userns=keep-id`), not as the namespace's root.
pub as_user: bool,
/// The host's machine-id, mounted read-only (the image has none).
pub machine_id: Option<PathBuf>,
}
impl Container {
/// Resolved from the environment for the repo at `root`.
pub fn from_env(root: &Path) -> Container {
let root = std::path::absolute(root).unwrap_or_else(|_| root.to_path_buf());
let var = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
// The container's own subdirectory of the target: its rustc and its /src paths
// differ from the host's, so sharing one directory would only rebuild both.
let target = var("NORNIR_CI_TOOLCHAIN_TARGET").map(PathBuf::from).unwrap_or_else(|| {
var("CARGO_TARGET_DIR").map(PathBuf::from).unwrap_or_else(|| root.join("target")).join("toolchain")
});
let registry = var("NORNIR_CI_REGISTRY").map(PathBuf::from).or_else(|| {
var("CARGO_HOME")
.map(PathBuf::from)
.or_else(|| var("HOME").map(|h| PathBuf::from(h).join(".cargo")))
.map(|h| h.join("registry"))
.filter(|p| p.is_dir())
});
let engine = var("NORNIR_CI_ENGINE").unwrap_or_else(|| "podman".into());
// keep-id is podman's; `NORNIR_CI_AS_ROOT=1` asks for the namespace's root.
let as_user = Path::new(&engine).file_name().is_some_and(|n| n == "podman") && var("NORNIR_CI_AS_ROOT").as_deref() != Some("1");
let machine_id = Some(PathBuf::from("/etc/machine-id")).filter(|p| p.is_file());
Container {
engine,
image: toolchain_image(var("NORNIR_CI_IMAGE")),
parent: root.parent().map(Path::to_path_buf).unwrap_or_else(|| root.clone()),
repo_dir: root.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(),
target,
registry,
jobs: var("CARGO_BUILD_JOBS"),
fast: var("NORNIR_CI_FAST").map(PathBuf::from),
sccache: var("NORNIR_CI_SCCACHE").map(PathBuf::from),
as_user,
machine_id,
}
}
/// The engine's argv for `program args` in `dir` (relative to the repo) with `env`.
pub fn argv(&self, dir: Option<&str>, env: &[(String, String)], program: &str, args: &[String]) -> Vec<String> {
let mut workdir = format!("/src/{}", self.repo_dir);
if let Some(d) = dir.filter(|d| !d.is_empty() && *d != ".") {
workdir = format!("{workdir}/{}", d.trim_start_matches("./").trim_end_matches('/'));
}
let mut v: Vec<String> = vec![self.engine.clone(), "run".into(), "--rm".into(), "--init".into(), "--pids-limit=-1".into()];
// As the invoking user: rootless podman's root ignores every mode bit and owns no
// service account, so tests that inject a permission failure or hand a data tree
// over (Skidbladnir's install, gunnar's lfs and repo tests) were red as root.
// keep-id maps the user to itself, so what the step writes stays that user's on
// the host, as root's did. Its passwd home would be the workdir (the repo): /tmp.
if self.as_user {
v.push("--userns=keep-id".into());
v.push("--passwd-entry=$USERNAME:*:$UID:$GID:$NAME:/tmp:/bin/sh".into());
}
let mut bind = |host: &Path, at: &str, ro: bool| {
v.push("-v".into());
v.push(format!("{}:{at}{}", host.display(), if ro { ":ro" } else { "" }));
};
bind(&self.parent, "/src", false);
bind(&self.target, "/target", false);
if let Some(m) = &self.machine_id {
bind(m, "/etc/machine-id", true);
}
if let Some(r) = &self.registry {
bind(r, "/usr/local/cargo/registry", false);
}
if let Some(f) = &self.fast {
bind(f, "/tools", true);
}
if let Some(s) = &self.sccache {
bind(s, "/sccache", false);
}
if let Some(f) = &self.fast {
v.push("--env-file".into());
v.push(f.join("env.sh").display().to_string());
}
let mut setenv = |k: &str, val: &str| {
v.push("-e".into());
v.push(format!("{k}={val}"));
};
setenv("CARGO_TARGET_DIR", "/target");
setenv("NORNIR_CI_IN_CONTAINER", "1");
// git refuses a checkout another uid owns (`dubious ownership`): a build script's
// commit stamp then reads nothing. Command-scope config, which git honours here.
setenv("GIT_CONFIG_COUNT", "1");
setenv("GIT_CONFIG_KEY_0", "safe.directory");
setenv("GIT_CONFIG_VALUE_0", "*");
if let Some(j) = &self.jobs {
setenv("CARGO_BUILD_JOBS", j);
}
for (k, val) in env {
setenv(k, val);
}
v.push("-w".into());
v.push(workdir);
v.push(self.image.clone());
v.push(program.into());
v.extend(args.iter().cloned());
v
}
}
/// Is this process inside a container (or told to run toolchain steps on the host)?
pub fn toolchain_is_here() -> bool {
std::env::var("NORNIR_CI_IN_CONTAINER").is_ok_and(|v| v == "1")
|| std::env::var("NORNIR_CI_HOST").is_ok_and(|v| v == "1")
|| Path::new("/run/.containerenv").exists()
|| Path::new("/.dockerenv").exists()
}
/// The full HEAD commit of the checkout at `root`, or `unknown`.
pub fn head(root: &Path) -> String {
nornir_git::gitops::resolve_ref(root, "HEAD").ok().flatten().unwrap_or_else(|| "unknown".into())
}
fn now_unix() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)
}
fn host_name() -> String {
std::env::var("HOSTNAME")
.ok()
.filter(|h| !h.is_empty())
.or_else(|| std::fs::read_to_string("/etc/hostname").ok().map(|h| h.trim().to_string()))
.unwrap_or_else(|| "unknown".into())
}
fn slug(label: &str) -> String {
label.chars().map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' { c } else { '-' }).collect()
}
/// The step timeout when a step names none: `NORNIR_CI_STEP_TIMEOUT` seconds, else 3600.
pub fn default_timeout_secs() -> u64 {
std::env::var("NORNIR_CI_STEP_TIMEOUT").ok().and_then(|v| v.parse().ok()).unwrap_or(3600)
}
enum Waited {
Exited(std::process::ExitStatus),
TimedOut,
Lost(std::io::Error),
}
/// Wait for `child` at most `secs`. On expiry it gets SIGTERM (a `podman run` client
/// passes it to its container), then SIGKILL 30 s later. Only this child is signalled.
fn wait_bounded(mut child: std::process::Child, secs: u64) -> Waited {
let deadline = Instant::now() + std::time::Duration::from_secs(secs);
loop {
match child.try_wait() {
Ok(Some(s)) => return Waited::Exited(s),
Ok(None) if Instant::now() < deadline => std::thread::sleep(std::time::Duration::from_millis(200)),
Ok(None) => break,
Err(e) => return Waited::Lost(e),
}
}
// `kill` is often only a shell builtin (the toolchain image has no procps).
let _ = Command::new("sh").args(["-c", &format!("kill -TERM {}", child.id())]).status();
let grace = Instant::now() + std::time::Duration::from_secs(30);
while Instant::now() < grace {
if let Ok(Some(_)) = child.try_wait() {
return Waited::TimedOut;
}
std::thread::sleep(std::time::Duration::from_millis(200));
}
let _ = child.kill();
let _ = child.wait();
Waited::TimedOut
}
/// The run's output directory.
pub fn out_dir(plan: &Plan, short: &str) -> PathBuf {
match std::env::var("NORNIR_CI_OUT").ok().filter(|v| !v.is_empty()) {
Some(d) => PathBuf::from(d),
None => {
let target = std::env::var("CARGO_TARGET_DIR").ok().filter(|v| !v.is_empty()).map(PathBuf::from);
target.unwrap_or_else(|| plan.root.join("target")).join("ci").join(format!("{}-{short}", plan.repo))
}
}
}
fn tail(path: &Path, n: usize) -> Vec<String> {
let Ok(f) = File::open(path) else { return Vec::new() };
let lines: Vec<String> = BufReader::new(f).lines().map_while(std::io::Result::ok).collect();
lines[lines.len().saturating_sub(n)..].to_vec()
}
/// Run `plan` in `mode`, keeping the steps whose label starts with `only` (all when
/// `None`). Every step runs even after a red one, so one receipt names every red.
pub fn run(plan: &Plan, mode: Mode, only: Option<&str>) -> Result<Receipt> {
let head = head(&plan.root);
let short: String = head.chars().take(12).collect();
let dirty = nornir_git::gitops::is_dirty(&plan.root).unwrap_or(false);
let out = out_dir(plan, &short);
std::fs::create_dir_all(&out).map_err(|e| format!("mkdir {}: {e}", out.display()))?;
let container = Container::from_env(&plan.root);
let started = now_unix();
let here = toolchain_is_here();
let mut results = Vec::new();
let steps: Vec<&Step> = plan
.steps
.iter()
.filter(|s| s.runs_in(mode) && only.is_none_or(|p| s.label.starts_with(p)))
.collect();
println!("ci: {} {} at {short}{} ({} steps)", plan.repo, mode.name(), if dirty { " (dirty)" } else { "" }, steps.len());
for (i, step) in steps.iter().enumerate() {
let log = out.join(format!("{:02}-{}.log", i + 1, slug(&step.label)));
println!("ci: [{}/{}] {} {}", i + 1, steps.len(), step.label, step.describe());
let t0 = Instant::now();
let mut file = File::create(&log).map_err(|e| format!("create {}: {e}", log.display()))?;
let (status, code, note): (&'static str, Option<i32>, String) = match &step.action {
Action::InProcess(check) => match check(&mut file) {
Ok(None) => ("pass", None, String::new()),
Ok(Some(why)) => ("skip", None, why),
Err(e) => {
let _ = writeln!(file, "error: {e}");
("fail", None, e.to_string())
}
},
Action::Command { program, args } => {
let mut cmd = if step.place == Place::Toolchain && !here {
std::fs::create_dir_all(&container.target).ok();
let argv = container.argv(step.dir.as_deref(), &step.env, program, args);
let mut c = Command::new(&argv[0]);
c.args(&argv[1..]);
c
} else {
let mut c = Command::new(program);
c.args(args);
c.current_dir(match &step.dir {
Some(d) => plan.root.join(d),
None => plan.root.clone(),
});
for (k, v) in &step.env {
c.env(k, v);
}
c
};
let err = file.try_clone().map_err(|e| format!("log {}: {e}", log.display()))?;
let out_f = file.try_clone().map_err(|e| format!("log {}: {e}", log.display()))?;
cmd.stdin(Stdio::null()).stdout(Stdio::from(out_f)).stderr(Stdio::from(err));
let limit = step.timeout_secs.unwrap_or_else(default_timeout_secs);
match cmd.spawn() {
Ok(child) => match wait_bounded(child, limit) {
Waited::Exited(s) if s.success() => ("pass", s.code(), String::new()),
Waited::Exited(s) => ("fail", s.code(), String::new()),
Waited::TimedOut => {
let _ = writeln!(file, "error: the step ran past its {limit} s timeout and was stopped");
("fail", None, format!("timed out after {limit} s"))
}
Waited::Lost(e) => ("fail", None, format!("lost the step's process: {e}")),
},
Err(e) => {
let _ = writeln!(file, "error: could not start {program}: {e}");
("fail", None, format!("could not start {program}: {e}"))
}
}
}
};
drop(file);
let text = std::fs::read_to_string(&log).unwrap_or_default();
let failed = if status == "fail" { failed_tests(&text) } else { Vec::new() };
let note = if status == "fail" && note.is_empty() { first_error(&text) } else { note };
let secs = t0.elapsed().as_secs_f64();
let mark = match status {
"pass" => "green",
"skip" => "skipped",
_ => "RED",
};
println!("ci: {mark} in {secs:.0} s, log {}", log.display());
if status == "fail" {
for l in tail(&log, 25) {
println!("ci: | {l}");
}
} else if status == "skip" {
println!("ci: skipped: {note}");
}
results.push(StepResult {
label: step.label.clone(),
place: if step.place == Place::Toolchain && here { Place::Host } else { step.place },
command: step.describe(),
status,
exit_code: code,
secs,
log,
failed_tests: failed,
note,
});
}
let receipt = Receipt {
repo: plan.repo.clone(),
head,
dirty,
mode,
host: host_name(),
started_unix: started,
finished_unix: now_unix(),
steps: results,
path: out.join("receipt.json"),
};
std::fs::write(&receipt.path, receipt.to_json()).map_err(|e| format!("write {}: {e}", receipt.path.display()))?;
Ok(receipt)
}
/// The checkout `--repo <dir>` names (absolute), else `default`.
pub fn repo_arg(args: &[String], default: &Path) -> PathBuf {
match args.iter().position(|a| a == "--repo").and_then(|i| args.get(i + 1)) {
Some(d) => std::path::absolute(d).unwrap_or_else(|_| PathBuf::from(d)),
None => default.to_path_buf(),
}
}
/// `cargo xtask ci [--quick|--nightly] [--list] [--only <prefix>]` over `plan`: the
/// whole verb. Returns the process exit code (0 green, 1 red, 2 could not run).
pub fn main(plan: &Plan, args: &[String]) -> i32 {
// The lock verbs every repo gets with its plan: `cargo xtask ci lockcheck` (every
// tracked Cargo.lock against the sibling checkouts, red with the fix line) and
// `cargo xtask ci lockfollow [--check]` (the mechanical sibling follow).
// `--repo <dir>` points either verb at another checkout (a consumer whose own
// xtask has no lockfollow yet), else the plan's repo.
let root = repo_arg(&args[args.len().min(1)..], &plan.root);
match args.first().map(String::as_str) {
Some("lockcheck") => {
let mut out = std::io::stdout();
return match crate::lockcheck::check_repo(&root, crate::lockcheck::DEFAULT_SINGLE, &mut out) {
Ok(f) if f.iter().any(|x| x.is_red()) => 1,
Ok(_) => 0,
Err(e) => {
eprintln!("lockcheck: {e}");
2
}
};
}
Some("lockfollow") => {
let apply = !args.iter().any(|a| a == "--check");
let mut out = std::io::stdout();
return match crate::lockcheck::follow_repo(&root, apply, &mut out) {
Ok(true) => 0,
Ok(false) => 1,
Err(e) => {
eprintln!("lockfollow: {e}");
2
}
};
}
_ => {}
}
let mode = Mode::from_args(args);
let only = args.iter().position(|a| a == "--only").and_then(|i| args.get(i + 1)).map(String::as_str);
if args.iter().any(|a| a == "--list") {
for s in plan.steps.iter().filter(|s| s.runs_in(mode)) {
println!("{:<28} {}", s.label, s.describe());
}
return 0;
}
match run(plan, mode, only) {
Ok(r) => {
let green = r.green();
match r.failing_step() {
None => println!("ci: {} GREEN ({} steps), receipt {}", r.repo, r.steps.len(), r.path.display()),
Some(s) => println!(
"ci: {} RED at {} ({}), receipt {}",
r.repo,
s.label,
if s.failed_tests.is_empty() { s.note.clone() } else { s.failed_tests.join(", ") },
r.path.display()
),
}
if green { 0 } else { 1 }
}
Err(e) => {
eprintln!("ci: could not run: {e}");
2
}
}
}
// ── the status page ──────────────────────────────────────────────────────────
/// The fields of a receipt the status page shows.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Seen {
pub repo: String,
pub head: String,
pub mode: String,
pub green: bool,
pub finished_unix: u64,
pub failing_step: Option<String>,
pub failing_detail: String,
pub failing_log: String,
pub path: PathBuf,
}
fn json_bool_field(text: &str, field: &str) -> Option<bool> {
let at = text.find(&format!("\"{field}\":"))? + field.len() + 3;
let rest = text[at..].trim_start();
if rest.starts_with("true") {
Some(true)
} else if rest.starts_with("false") {
Some(false)
} else {
None
}
}
fn json_u64_field(text: &str, field: &str) -> Option<u64> {
let at = text.find(&format!("\"{field}\":"))? + field.len() + 3;
let digits: String = text[at..].trim_start().chars().take_while(|c| c.is_ascii_digit()).collect();
digits.parse().ok()
}
/// Read the top-level fields of one receipt.
pub fn read_receipt(path: &Path) -> Option<Seen> {
let t = std::fs::read_to_string(path).ok()?;
if crate::json_string_field(&t, "schema").as_deref() != Some(SCHEMA) {
return None;
}
Some(Seen {
repo: crate::json_string_field(&t, "repo")?,
head: crate::json_string_field(&t, "head").unwrap_or_default(),
mode: crate::json_string_field(&t, "mode").unwrap_or_default(),
green: json_bool_field(&t, "green")?,
finished_unix: json_u64_field(&t, "finished_unix").unwrap_or(0),
failing_step: crate::json_string_field(&t, "failing_step"),
failing_detail: crate::json_string_field(&t, "failing_detail").unwrap_or_default(),
failing_log: crate::json_string_field(&t, "failing_log").unwrap_or_default(),
path: path.to_path_buf(),
})
}
/// Every receipt (`receipt.json`) up to four levels below `dir`.
pub fn receipts_under(dir: &Path) -> Vec<Seen> {
fn walk(d: &Path, depth: usize, out: &mut Vec<Seen>) {
let Ok(rd) = std::fs::read_dir(d) else { return };
for e in rd.flatten() {
let p = e.path();
if p.is_dir() && depth < 4 {
walk(&p, depth + 1, out);
} else if p.file_name().is_some_and(|n| n == "receipt.json") {
if let Some(s) = read_receipt(&p) {
out.push(s);
}
}
}
}
let mut out = Vec::new();
walk(dir, 0, &mut out);
out
}
/// Whether the run beside `receipt` was filed into Mímir, read off the runner's
/// `file.out` (`dwarves ci-file`'s lines): "filed" when any step went in; "not filed (the
/// warehouse does not answer)" when the warehouse was off, which is the owner's order and
/// not a fault; "not filed" otherwise (no file, nothing to file, or a refused filing).
fn filed_word(receipt: &Path) -> String {
let Ok(t) = std::fs::read_to_string(receipt.with_file_name("file.out")) else { return "not filed".into() };
if t.contains(": filed as ") {
"filed".into()
} else if t.contains("the warehouse does not answer") {
"not filed (the warehouse does not answer)".into()
} else {
"not filed".into()
}
}
/// The newest receipt per repo (and per mode, so a nightly does not hide the per-push
/// verdict), sorted by repo.
pub fn latest(seen: Vec<Seen>) -> Vec<Seen> {
let mut by: std::collections::BTreeMap<(String, String), Seen> = std::collections::BTreeMap::new();
for s in seen {
let k = (s.repo.clone(), s.mode.clone());
match by.get(&k) {
Some(old) if old.finished_unix >= s.finished_unix => {}
_ => {
by.insert(k, s);
}
}
}
by.into_values().collect()
}
/// A UTC timestamp (`2026-10-23 18:04Z`) for a unix second, std only.
pub fn utc(secs: u64) -> String {
let days = (secs / 86_400) as i64;
let rem = secs % 86_400;
// Howard Hinnant's civil-from-days.
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z - era * 146_097;
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
format!("{y:04}-{m:02}-{d:02} {:02}:{:02}Z", rem / 3_600, rem % 3_600 / 60)
}
/// The status page (Markdown) for the newest receipts under `dir`.
pub fn status_md(dir: &Path, now_unix: u64) -> String {
let rows = latest(receipts_under(dir));
let mut o = format!("# CI status\n\nUpdated {}. One row per repo and mode: the newest run.\n\n", utc(now_unix));
o += "| repo | mode | head | result | failing step | detail | log | filed |\n|---|---|---|---|---|---|---|---|\n";
for r in &rows {
let head: String = r.head.chars().take(10).collect();
let (res, step, detail, log) = if r.green {
("green", String::new(), String::new(), r.path.display().to_string())
} else {
("**RED**", r.failing_step.clone().unwrap_or_default(), r.failing_detail.replace('|', "/"), r.failing_log.clone())
};
o += &format!("| {} | {} | {head} | {res} | {step} | {} | {log} | {} |\n", r.repo, r.mode, detail.chars().take(160).collect::<String>(), filed_word(&r.path));
}
if rows.is_empty() {
o += "| (no receipts yet) | | | | | | | |\n";
}
let red = rows.iter().filter(|r| !r.green).count();
o += &format!("\n{} repo run(s), {} red.\n", rows.len(), red);
o
}
#[cfg(test)]
mod tests {
use super::*;
fn s(v: &[&str]) -> Vec<String> {
v.iter().map(|x| x.to_string()).collect()
}
#[test]
fn the_default_image_is_pinned_by_digest_not_a_tag() {
// M19: a tag (`:live`) is a different image on each box and after each re-mint.
let (name, digest) = DEFAULT_IMAGE.split_once("@sha256:").expect("DEFAULT_IMAGE names a digest");
assert_eq!(name, "localhost/nordisk-cargo-toolchain");
assert!(!name.contains(':'), "no tag beside the digest: {DEFAULT_IMAGE}");
assert_eq!(digest.len(), 64, "a full sha256: {digest}");
assert!(digest.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), "lower hex: {digest}");
}
#[test]
fn the_default_image_is_the_toolchain_edda_pins() {
// The one pin: edda's own recipe (`.nornir/build-variants.toml`), which lockcheck holds
// every sibling to. A re-pin there without a re-pin here is red.
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let (image, pin) = crate::lockcheck::toolchain_pin(&root).expect("edda pins its toolchain");
assert_eq!(DEFAULT_IMAGE, format!("localhost/{image}@sha256:{pin}"));
}
#[test]
fn a_toolchain_step_runs_in_the_ci_image_pinned_by_digest() {
// OX-RED 2026-10-27: the recipe (DEFAULT_IMAGE) has no ssh client, no strace, no
// git-lfs, no OpenSSL headers, no clippy and no rustfmt, so a developer's
// `cargo xtask ci` without NORNIR_CI_IMAGE ran its tests in another image than the
// runner's. A toolchain step's default is the CI image, by digest.
let (name, digest) = CI_IMAGE.split_once("@sha256:").expect("CI_IMAGE names a digest");
assert_eq!(name, "localhost/nordisk-cargo-toolchain");
assert_eq!(digest.len(), 64, "a full sha256: {digest}");
assert!(digest.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), "lower hex: {digest}");
assert_ne!(CI_IMAGE, DEFAULT_IMAGE, "the CI image is the recipe plus the test clients");
assert_eq!(toolchain_image(None), CI_IMAGE);
assert_eq!(toolchain_image(Some("img:1".into())), "img:1", "NORNIR_CI_IMAGE still wins");
assert_eq!(toolchain_image(Some(String::new())), CI_IMAGE, "an empty NORNIR_CI_IMAGE is unset");
}
#[test]
fn the_ci_image_is_the_one_the_runner_pins() {
// One pin: dwarves' ci/runner.sh CI_IMAGE (the sibling checkout, when it is there).
let runner = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../dwarves/ci/runner.sh");
let Ok(r) = std::fs::read_to_string(&runner) else {
eprintln!("no dwarves sibling at {}: the runner's pin is not compared", runner.display());
return;
};
let line = r.lines().find(|l| l.starts_with("CI_IMAGE=")).expect("runner.sh sets CI_IMAGE");
assert_eq!(line.trim_start_matches("CI_IMAGE=").trim().trim_matches('"'), CI_IMAGE, "{}", runner.display());
}
#[test]
fn the_mode_is_read_off_the_flags() {
assert_eq!(Mode::from_args(&s(&["--quick"])), Mode::Quick);
assert_eq!(Mode::from_args(&s(&[])), Mode::Full);
assert_eq!(Mode::from_args(&s(&["--quick", "--nightly"])), Mode::Nightly);
}
#[test]
fn a_step_runs_in_its_least_mode_and_every_mode_above_it() {
let q = Step::cargo("t", &["test"]);
let n = Step::cargo("b", &["bench"]).from(Mode::Nightly);
assert!(q.runs_in(Mode::Quick) && q.runs_in(Mode::Nightly));
assert!(!n.runs_in(Mode::Quick) && !n.runs_in(Mode::Full) && n.runs_in(Mode::Nightly));
}
#[test]
fn the_failed_tests_are_read_off_libtest_and_nextest_logs() {
let log = "running 3 tests\ntest a::ok ... ok\ntest a::bad ... FAILED\n\
---- a::bad stdout ----\ntest a::bad ... FAILED\n FAIL [ 0.012s] holger::paired twin_is_certified\n";
assert_eq!(failed_tests(log), vec!["a::bad".to_string(), "holger::paired twin_is_certified".to_string()]);
assert!(failed_tests("test x ... ok\n").is_empty());
}
#[test]
fn the_first_error_is_the_compilers_or_the_panic() {
assert_eq!(first_error("Compiling x\nerror[E0063]: missing field\nmore"), "error[E0063]: missing field");
assert_eq!(first_error("a\nthread 'm' panicked at src/x.rs:1\n"), "thread 'm' panicked at src/x.rs:1");
assert_eq!(first_error("a\nlast words\n\n"), "last words");
}
#[test]
fn the_lock_verbs_take_another_checkout_with_repo() {
let d = Path::new("/here");
assert_eq!(repo_arg(&s(&["--check"]), d), PathBuf::from("/here"));
assert_eq!(repo_arg(&s(&["--check", "--repo", "/there"]), d), PathBuf::from("/there"));
}
#[test]
fn json_strings_are_escaped() {
assert_eq!(json_str("a\"b\\c\nd\u{1}"), "\"a\\\"b\\\\c\\nd\\u0001\"");
}
#[test]
fn the_container_mounts_the_parent_so_siblings_resolve_and_carries_the_fast_tools() {
let c = Container {
engine: "podman".into(),
image: "img:live".into(),
parent: "/work/ci/src".into(),
repo_dir: "holger".into(),
target: "/work/ci/target/holger".into(),
registry: Some("/work/cargo-home".into()),
jobs: Some("32".into()),
fast: Some("/work/tools".into()),
sccache: Some("/work/sccache".into()),
as_user: true,
machine_id: Some("/etc/machine-id".into()),
};
let v = c.argv(Some("estate-tests"), &[("K".into(), "V".into())], "cargo", &s(&["test", "--locked"]));
let line = v.join(" ");
// No pids cap: podman's default 2048 is exhausted on a 64-core runner by tests that
// each start a multi-threaded runtime (monetize's mock-upcloud behaviours failed with
// EAGAIN at reqwest's client build, a different set each run, 2026-10-24).
assert!(line.starts_with("podman run --rm --init --pids-limit=-1 --userns=keep-id"), "{line}");
assert!(line.contains(" -v /work/ci/src:/src -v /work/ci/target/holger:/target -v /etc/machine-id:/etc/machine-id:ro"), "{line}");
assert!(line.contains("-v /work/cargo-home:/usr/local/cargo/registry"), "{line}");
assert!(line.contains("-v /work/tools:/tools:ro -v /work/sccache:/sccache --env-file /work/tools/env.sh"), "{line}");
assert!(line.contains("-e CARGO_BUILD_JOBS=32 -e K=V -w /src/holger/estate-tests img:live cargo test --locked"), "{line}");
}
fn holger_container(as_user: bool) -> Container {
Container {
engine: "podman".into(),
image: "img:live".into(),
parent: "/work/ci/src".into(),
repo_dir: "holger".into(),
target: "/work/ci/target/holger".into(),
registry: None,
jobs: None,
fast: None,
sccache: None,
as_user,
machine_id: None,
}
}
#[test]
fn a_toolchain_step_runs_as_the_invoking_user_with_its_home_outside_the_repo() {
// As root (rootless podman's uid 0) Skidbladnir's install tests demand the
// service's account (`User=_holger`) and gunnar's injected-failure tests find
// every mode bit ignored: 26 red on the runner, 2026-10-24. keep-id maps the
// invoking user to itself, so the files the step writes stay that user's on the
// host. Its passwd home would be the workdir (the repo itself); /tmp it is.
let line = holger_container(true).argv(None, &[], "cargo", &s(&["test"])).join(" ");
assert!(line.starts_with("podman run --rm --init --pids-limit=-1 --userns=keep-id --passwd-entry=$USERNAME:*:$UID:$GID:$NAME:/tmp:/bin/sh -v "), "{line}");
}
#[test]
fn git_trusts_every_checkout_a_step_sees() {
// A mount owned by another uid of the namespace is `dubious ownership` to git,
// and a build script that stamps its commit then reads none. Command-scope config
// (GIT_CONFIG_*), the only scope besides system and global that git honours here.
for as_user in [true, false] {
let line = holger_container(as_user).argv(None, &[], "cargo", &s(&["test"])).join(" ");
assert!(line.contains("-e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e GIT_CONFIG_VALUE_0=*"), "{line}");
}
}
#[test]
fn as_root_on_request_and_the_hosts_machine_id_when_it_has_one() {
let line = holger_container(false).argv(None, &[], "cargo", &s(&["test"])).join(" ");
assert!(!line.contains("keep-id") && !line.contains("passwd-entry") && !line.contains("machine-id"), "{line}");
// The image has no /etc/machine-id, so an identity minted twice on one "machine"
// differed (Skidbladnir's fresh_dirs_mint_different_install_ids).
let c = Container { machine_id: Some("/etc/machine-id".into()), ..holger_container(true) };
let line = c.argv(None, &[], "cargo", &s(&["test"])).join(" ");
assert!(line.contains("-v /etc/machine-id:/etc/machine-id:ro"), "{line}");
}
#[test]
fn a_run_writes_a_receipt_naming_every_red_step_and_its_log() {
let dir = std::env::temp_dir().join(format!("nornir-ci-test-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let _g = crate::ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
std::env::set_var("NORNIR_CI_OUT", dir.join("out"));
let plan = Plan {
repo: "demo".into(),
root: dir.clone(),
steps: vec![
Step::in_process("ok", |_w| Ok(None)),
Step::in_process("skipped", |_w| Ok(Some("no oden".into()))),
Step::host("red", "sh", &["-c", "echo 'test a::b ... FAILED'; exit 3"]),
Step::host("late", "true", &[]).from(Mode::Nightly),
Step::host("hung", "sleep", &["600"]).timeout(1),
],
};
let r = run(&plan, Mode::Quick, None).unwrap();
std::env::remove_var("NORNIR_CI_OUT");
assert!(!r.green());
assert_eq!(r.steps.len(), 4, "the nightly step is not in the quick set");
let hung = r.steps.iter().find(|s| s.label == "hung").unwrap();
assert_eq!((hung.status, hung.note.as_str()), ("fail", "timed out after 1 s"), "a hung step costs one step");
assert!(hung.secs < 20.0, "stopped at its timeout, not at the end of the sleep: {}", hung.secs);
let red = r.failing_step().unwrap();
assert_eq!((red.label.as_str(), red.exit_code, red.failed_tests.clone()), ("red", Some(3), vec!["a::b".to_string()]));
assert_eq!(r.steps[1].status, "skip");
let json = std::fs::read_to_string(&r.path).unwrap();
assert!(json.contains("\"green\": false") && json.contains("\"failing_step\": \"red\""), "{json}");
assert!(red.log.is_file());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_status_page_names_the_newest_run_per_repo_and_its_red_step() {
let dir = std::env::temp_dir().join(format!("nornir-ci-status-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
let mk = |run: &str, repo: &str, green: bool, at: u64| {
let r = Receipt {
repo: repo.into(),
head: "0123456789abcdef".into(),
dirty: false,
mode: Mode::Quick,
host: "h".into(),
started_unix: at,
finished_unix: at,
steps: vec![StepResult {
label: "pairing".into(),
place: Place::Toolchain,
command: "cargo test".into(),
status: if green { "pass" } else { "fail" },
exit_code: Some(if green { 0 } else { 101 }),
secs: 1.0,
log: "/l/p.log".into(),
failed_tests: if green { vec![] } else { vec!["twin_is_certified".into()] },
note: String::new(),
}],
path: PathBuf::new(),
};
let d = dir.join(run).join(repo);
std::fs::create_dir_all(&d).unwrap();
std::fs::write(d.join("receipt.json"), r.to_json()).unwrap();
};
mk("r1", "holger", true, 100);
mk("r2", "holger", false, 200);
mk("r2", "edda", true, 200);
mk("r2", "tunnr", true, 200);
// The filing into Mímir, as the runner's `dwarves ci-file` wrote it beside the receipt:
// edda's went in; holger's found the warehouse off (not a fault); tunnr's is unsaid.
std::fs::write(dir.join("r2/edda/file.out"), "ci-file: lockcheck: not filed: no test outcome and no bench envelope in the log\nci-file: test: filed as tests: 12 row(s)\n").unwrap();
std::fs::write(dir.join("r2/holger/file.out"), "ci-file: pairing: not filed: the warehouse does not answer (Timeout expired)\n").unwrap();
let md = status_md(&dir, 1_761_242_640);
assert!(md.contains("Updated 2025-10-23 18:04Z"), "{md}");
assert!(md.contains("| repo | mode | head | result | failing step | detail | log | filed |"), "{md}");
assert!(md.contains("| holger | quick | 0123456789 | **RED** | pairing | twin_is_certified | /l/p.log | not filed (the warehouse does not answer) |"), "{md}");
assert!(md.contains("| edda | quick | 0123456789 | green | | | ") && md.contains("receipt.json | filed |"), "{md}");
assert!(md.contains("| tunnr | quick | 0123456789 | green | | | ") && md.contains("receipt.json | not filed |"), "{md}");
assert!(md.contains("3 repo run(s), 1 red."), "{md}");
let _ = std::fs::remove_dir_all(&dir);
}
}