xtask/src/products.rs
//! **`cargo xtask products`**: the by-hand guide "Three products by hand" (`.nornir/README.md`),
//! run as written (lane HOL-PRODUCTS, owner 2026-10-30).
//!
//! The guide is the test: its `bash` blocks between `<!-- three-products:begin -->` and
//! `<!-- three-products:end -->` run, in order, in one shell. Three things differ from a person
//! at a keyboard, and only these:
//!
//! * `cargo install <crate> [--features f]` installs the product built from THIS checkout
//! (holger's own crates, the console's from the holger-ui checkout beside it), with the
//! features the line names, instead of crates.io; `rustup` does nothing (the toolchain image
//! has the targets). Every build the guide names runs in the pinned toolchain container, one
//! `cargo build -p <crate>` each, so no feature of one product unifies into another.
//! * `pip` runs in the python image (`--network host`, the guide's directory mounted at its own
//! path): the host need not have pip.
//! * The places and the ports come from the environment (`G` a fresh directory, free ports):
//! the guide's first block takes them from there.
//!
//! Green when the script ends with `GUIDE-DONE`. A host step of the full CI set (it builds in
//! the toolchain container, starts the scan sandbox's containers and pulls from PyPI); never
//! in `--quick`. It writes `target/products/receipt.json` and the guide's log beside it.
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use anyhow::{bail, Context, Result};
/// The guide's markers in `.nornir/README.md`.
pub const BEGIN: &str = "<!-- three-products:begin -->";
pub const END: &str = "<!-- three-products:end -->";
/// The four products and the workspace each crate builds in, from holger's root
/// (`holger-downstream` is a workspace of its own; the console is the holger-ui checkout's).
pub const PRODUCTS: [(&str, &str); 4] = [
("holger", "."),
("holger-downstream", "holger-downstream"),
("holger-cli", "."),
("holger-ui", "../holger-ui"),
];
/// holger-ui's `cargo xtask console`, spelled out so it runs `--locked` (HOL-PRODUCTS-9): the
/// verb never rewrites a committed lock.
const CONSOLE_XTASK: [&str; 7] = [
"run",
"--locked",
"--quiet",
"--package",
"xtask",
"--",
"console",
];
/// The static scan worker the guide's image is built from.
const MUSL: &str = "x86_64-unknown-linux-musl";
/// The image tag the guide builds (the server's default sandbox image). The host's own image
/// under that tag is put back after the run: other work on the host scans with it.
pub const WORKER_IMAGE: &str = holger_sandbox::DEFAULT_IMAGE;
/// The image id a tag names on this host, if any.
fn image_id(tag: &str) -> Option<String> {
let o = Command::new("podman")
.args(["image", "inspect", "--format", "{{.Id}}", tag])
.output()
.ok()?;
o.status
.success()
.then(|| String::from_utf8_lossy(&o.stdout).trim().to_string())
.filter(|s| !s.is_empty())
}
/// **The guide's commands**: every ```bash block between the markers, in order.
pub fn guide_blocks(readme: &str) -> Result<Vec<String>> {
let start = readme
.find(BEGIN)
.context("the README has no three-products guide (its begin marker)")?;
let end = readme[start..]
.find(END)
.map(|e| start + e)
.context("the guide has no end marker")?;
let mut blocks = Vec::new();
let mut cur: Option<String> = None;
for line in readme[start..end].lines() {
match (&mut cur, line.trim_end()) {
(None, "```bash") => cur = Some(String::new()),
(Some(_), "```") => blocks.push(cur.take().unwrap_or_default()),
(Some(b), l) => {
b.push_str(l);
b.push('\n');
}
(None, _) => {}
}
}
if cur.is_some() {
bail!("the guide's last bash block is not closed");
}
if blocks.is_empty() {
bail!("the guide has no bash block");
}
Ok(blocks)
}
/// **The builds the guide installs**: `(crate, features)` of every `cargo install` line, the
/// features sorted and comma-joined (empty: the defaults). The key [`bin_name`] names them by.
pub fn installs(blocks: &[String]) -> Vec<(String, String)> {
let mut out: Vec<(String, String)> = Vec::new();
for line in blocks.iter().flat_map(|b| b.lines()) {
let words: Vec<&str> = line.split_whitespace().collect();
if words.len() < 3 || words[0] != "cargo" || words[1] != "install" {
continue;
}
let (mut crates, mut features) = (Vec::new(), Vec::new());
let mut it = words[2..].iter();
while let Some(w) = it.next() {
match *w {
"--features" | "-F" => features.extend(
it.next()
.map(|f| f.split(',').map(String::from).collect::<Vec<_>>())
.unwrap_or_default(),
),
"--root" | "--target" | "--path" | "--git" => {
it.next();
}
w if w.starts_with('-') => {}
w => crates.push(w.to_string()),
}
}
features.sort();
for c in crates {
let e = (c, features.join(","));
if !out.contains(&e) {
out.push(e);
}
}
}
out
}
/// The file a build is kept under in the test's bin directory: `<crate>` for the defaults,
/// `<crate>+<features>` otherwise.
pub fn bin_name(krate: &str, features: &str) -> String {
if features.is_empty() {
krate.to_string()
} else {
format!("{krate}+{features}")
}
}
/// The shell the guide runs in: strict, the three differences above, then the blocks.
pub fn script(blocks: &[String]) -> String {
let mut s = String::from(PRELUDE);
for (i, b) in blocks.iter().enumerate() {
s.push_str(&format!("echo '== guide block {}'\n", i + 1));
s.push_str(b);
s.push('\n');
}
s.push_str("echo GUIDE-DONE\njobs -p | xargs -r kill 2>/dev/null || true\n");
s
}
const PRELUDE: &str = r#"set -Eeuo pipefail
trap 'echo "GUIDE-FAILED at line $LINENO: $BASH_COMMAND" >&2; jobs -p | xargs -r kill 2>/dev/null; exit 1' ERR
# `cargo install <crate> [--features f]`: the build of this checkout's crate the test made
# ($GUIDE_BIN/<crate>[+<features>]), installed into --root/bin or the PATH's first directory.
cargo() {
if [ "$1" = install ]; then
shift; local root="" crates=() feats=""
while [ $# -gt 0 ]; do
case "$1" in --root) root=$2; shift 2;; --features|-F) feats=$(printf '%s' "$2" | tr ',' '\n' | sort | paste -sd, -); shift 2;; --target|--path|--git) shift 2;; -*) shift;; *) crates+=("$1"); shift;; esac
done
for c in "${crates[@]}"; do
local built="$GUIDE_BIN/$c${feats:++$feats}" into="${root:+$root/bin}"
[ -x "$built" ] || { echo "the test built no $built" >&2; return 1; }
mkdir -p "${into:-$GUIDE_PATH}" && cp "$built" "${into:-$GUIDE_PATH}/$c"
echo "(the test) cargo install $c${feats:+ --features $feats}: the crate built from this checkout"
done
return 0
fi
command cargo "$@"
}
rustup() { echo "(the test) rustup $*: the toolchain image has it"; }
pip() { podman run --rm --network host -v "$G:$G" -w "$PWD" "$GUIDE_PYTHON" pip --disable-pip-version-check "$@"; }
"#;
fn target_dir(root: &Path) -> PathBuf {
std::env::var_os("CARGO_TARGET_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| root.join("target"))
}
fn run(cmd: &mut Command) -> Result<()> {
let shown = format!("{cmd:?}");
let st = cmd.status().with_context(|| format!("starting {shown}"))?;
if !st.success() {
bail!("{shown} exited {st}");
}
Ok(())
}
/// One `cargo build --release --locked -p <crate>` in the toolchain container, in `dir`, into
/// `target`; the built program's path.
fn build_one(
parent: &Path,
dir: &Path,
target: &Path,
krate: &str,
features: &str,
triple: Option<&str>,
) -> Result<PathBuf> {
let registry = std::env::var_os("CARGO_REGISTRY_DIR")
.map(PathBuf::from)
.unwrap_or_else(|| {
PathBuf::from(std::env::var_os("HOME").unwrap_or_default()).join(".cargo/registry")
});
let toolchain = std::env::var("TOOLCHAIN_IMAGE")
.unwrap_or_else(|_| "localhost/nordisk-cargo-toolchain:live".into());
let mut c = Command::new("podman");
c.args(["run", "--rm", "--userns=keep-id"])
.arg(format!("--volume={0}:{0}", parent.display()))
.arg(format!("--volume={0}:{0}", target.display()))
.arg(format!(
"--volume={}:/usr/local/cargo/registry",
registry.display()
))
.arg(format!("--env=CARGO_TARGET_DIR={}", target.display()))
.arg("--env=HOME=/tmp")
.arg(format!("--workdir={}", dir.display()))
.arg(&toolchain)
.args(["cargo", "build", "--release", "--locked", "-p", krate]);
if !features.is_empty() {
c.args(["--features", features]);
}
if let Some(t) = triple {
c.args(["--target", t]);
}
run(&mut c)?;
let out = match triple {
Some(t) => target.join(t).join("release").join(krate),
None => target.join("release").join(krate),
};
anyhow::ensure!(out.is_file(), "{} was not built", out.display());
Ok(out)
}
/// A free loopback port.
fn free_port() -> Result<u16> {
Ok(std::net::TcpListener::bind("127.0.0.1:0")?
.local_addr()?
.port())
}
/// The verb.
pub fn main(root: &Path, args: &[String]) -> Result<()> {
let started = Instant::now();
let ui_root = root.join("../holger-ui").canonicalize().context("holger-ui is not checked out beside holger (../holger-ui): the console is one of the four products")?;
let parent = root.parent().unwrap_or(root).to_path_buf();
let target = target_dir(root).join("products");
let bin = target.join("bin");
std::fs::create_dir_all(&bin)?;
let readme = std::fs::read_to_string(root.join(".nornir/README.md"))?;
let blocks = guide_blocks(&readme)?;
// The console's bundle (trunk, the holger-ui repository's named exception, on the host)
// into the crate, so `cargo install holger-ui` carries it as a published crate does.
if !args.iter().any(|a| a == "--no-console") {
run(
Command::new(std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()))
.current_dir(&ui_root)
.args(CONSOLE_XTASK),
)?;
}
// Each build the guide installs on its own, so no feature of one unifies into another.
for (krate, features) in installs(&blocks) {
let at = PRODUCTS
.iter()
.find(|(p, _)| *p == krate)
.map_or(".", |(_, d)| *d);
let in_ui = at.starts_with("../");
let dir = if in_ui {
ui_root.clone()
} else {
root.join(at)
};
let t = if in_ui {
target.join("ui")
} else {
target.clone()
};
std::fs::create_dir_all(&t)?;
let triple = (krate == "holger-scan-worker").then_some(MUSL);
let built = build_one(&parent, &dir, &t, &krate, &features, triple)?;
std::fs::copy(&built, bin.join(bin_name(&krate, &features)))?;
}
let work = target.join(format!("work-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&work);
std::fs::create_dir_all(&work)?;
let g = work.join("guide");
let path_bin = work.join("path-bin");
std::fs::create_dir_all(&path_bin)?;
let script_path = work.join("guide.sh");
std::fs::write(&script_path, script(&blocks))?;
let log = target.join("guide.log");
let log_file = std::fs::File::create(&log)?;
let mut c = Command::new("bash");
c.arg(&script_path)
.env("G", &g)
.env("GUIDE_BIN", &bin)
.env("GUIDE_PATH", &path_bin)
.env(
"GUIDE_PYTHON",
std::env::var("PYTHON_IMAGE")
.unwrap_or_else(|_| "docker.io/library/python:3.13-slim-trixie".into()),
)
// Nothing of Holger's is on the PATH until the guide's `cargo install` puts it there.
.env(
"PATH",
format!(
"{}:{}",
path_bin.display(),
std::env::var("PATH").unwrap_or_default()
),
)
.stdout(log_file.try_clone()?)
.stderr(log_file);
for var in ["HQ", "HQ_GRPC", "DS", "DS_GRPC", "UI", "AIR", "AIR_GRPC"] {
c.env(var, format!("127.0.0.1:{}", free_port()?));
}
let before = image_id(WORKER_IMAGE);
let st = c.status().context("run the guide")?;
// Put the host's worker image back under its tag, and drop the one the guide built.
let built = image_id(WORKER_IMAGE);
if built != before {
if let Some(id) = &before {
let _ = Command::new("podman")
.args(["tag", id, WORKER_IMAGE])
.status();
} else {
let _ = Command::new("podman")
.args(["untag", WORKER_IMAGE])
.status();
}
if let Some(id) = built {
let _ = Command::new("podman").args(["rmi", &id]).status();
}
}
let text = std::fs::read_to_string(&log).unwrap_or_default();
let done = st.success() && text.lines().any(|l| l == "GUIDE-DONE");
let secs = started.elapsed().as_secs();
let failed_at = text
.lines()
.find(|l| l.starts_with("GUIDE-FAILED"))
.unwrap_or("")
.to_string();
let receipt = serde_json::json!({
"schema": "holger.products-receipt/v1",
"result": if done { "pass" } else { "fail" },
"seconds": secs,
"blocks": blocks.len(),
"failed_at": failed_at,
"log": log.display().to_string(),
});
std::fs::write(
target.join("receipt.json"),
serde_json::to_string_pretty(&receipt)? + "\n",
)?;
println!(
"products: the guide's {} block(s) in {secs} s: {} (log {})",
blocks.len(),
if done { "GREEN" } else { "RED" },
log.display()
);
if !done {
for l in text
.lines()
.rev()
.take(40)
.collect::<Vec<_>>()
.into_iter()
.rev()
{
println!(" | {l}");
}
bail!("the guide did not run to its end: {failed_at}");
}
if std::env::var_os("HOLGER_PRODUCTS_KEEP").is_some() {
println!("kept {}", work.display());
} else {
let _ = std::fs::remove_dir_all(&work);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
/// HOL-PRODUCTS-9: the console's bundle is built through holger-ui's own xtask with
/// `--locked`, as every build here is: the verb rewrites no committed lock.
#[test]
fn the_console_is_built_locked() {
assert_eq!(CONSOLE_XTASK.first(), Some(&"run"));
assert!(CONSOLE_XTASK.contains(&"--locked"), "{CONSOLE_XTASK:?}");
assert_eq!(CONSOLE_XTASK.last(), Some(&"console"));
}
#[test]
fn every_install_line_names_its_build() {
let b = vec!["cargo install holger holger-cli\nx\n".to_string(), "cargo install --target t --root \"$G/w\" holger-scan-worker\ncargo install holger-downstream --features fetch\ncargo install --root \"$G/air\" holger-downstream\n".to_string()];
assert_eq!(
installs(&b),
[
("holger", ""),
("holger-cli", ""),
("holger-scan-worker", ""),
("holger-downstream", "fetch")
]
.map(|(c, f)| (c.to_string(), f.to_string()))
.iter()
.cloned()
.chain([("holger-downstream".to_string(), String::new())])
.collect::<Vec<_>>()
);
assert_eq!(
bin_name("holger-downstream", "fetch"),
"holger-downstream+fetch"
);
assert_eq!(bin_name("holger", ""), "holger");
}
#[test]
fn the_blocks_between_the_markers_are_the_guide() {
let md = format!("```bash\nnot this\n```\n{BEGIN}\ntext\n```bash\necho one\n```\nmore\n```ron\n(no)\n```\n```bash\necho two\necho three\n```\n{END}\n```bash\nnor this\n```\n");
let b = guide_blocks(&md).unwrap();
assert_eq!(b, ["echo one\n", "echo two\necho three\n"]);
let s = script(&b);
assert!(s.starts_with("set -Eeuo pipefail"), "{s}");
assert!(
s.contains("echo one\n")
&& s.trim_end().ends_with("|| true")
&& s.contains("echo GUIDE-DONE"),
"{s}"
);
assert!(guide_blocks("no guide here").is_err());
assert!(
guide_blocks(&format!("{BEGIN}\n```bash\necho open\n{END}")).is_err(),
"an unclosed block is refused"
);
}
/// The README's guide installs each of the four products by its crate, and names the
/// two connections it promises: the online one and the airgap one.
#[test]
fn the_readme_guide_installs_the_four_products_and_runs_both_paths() {
let root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
for readme in [".nornir/README.md", "README.md"] {
let text = std::fs::read_to_string(root.join(readme)).unwrap();
let all = guide_blocks(&text).unwrap().concat();
for (krate, _) in PRODUCTS {
assert!(
all.lines().any(|l| l.starts_with("cargo install")
&& l.split_whitespace().any(|w| w == krate)),
"{readme}: the guide does not `cargo install {krate}`"
);
}
for must in [
"holger start --config",
"holger-ui",
"holger-downstream start --config",
"/-/sync/downstreams/enrol",
"downstream invite --name lab",
"invite: Some(",
"pip install",
"holger-cli bundle finalize",
"holger-cli verify",
"bundle import --config air.ron",
"network: Offline",
] {
assert!(
all.contains(must),
"{readme}: the guide does not run `{must}`"
);
}
}
}
}