tests/ron_orchestration.rs
//! Integration test: a real `.ron` file on disk drives infra setup → application
//! install through the public `skidbladnir::orchestration` API. Proves the
//! declarative layer end-to-end (RON file → interpreter → native modules → real
//! install lifecycle), using ONLY the crate's public surface.
use std::collections::BTreeMap;
use std::path::PathBuf;
use skidbladnir::archive::{Archive, TarZstd};
use skidbladnir::orchestration::{Flow, Registry, RunContext, modules, run_flow};
use skidbladnir::product::ComponentSpec;
/// Build a tiny real product bundle (TarZstd) for `spec`.
fn product_bundle(spec: &ComponentSpec, version: &str) -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().unwrap();
let staging = dir.path().join("staging");
let bin = staging.join(&spec.bundle_binary);
std::fs::create_dir_all(bin.parent().unwrap()).unwrap();
std::fs::write(&bin, b"#!/fake korp appliance\n").unwrap();
std::fs::write(staging.join("README"), b"korp product tree\n").unwrap();
let bundle = dir.path().join("korp.tar.zst");
TarZstd::new().pack(&staging, version, &bundle).unwrap();
(dir, bundle)
}
/// The committed `examples/korp-provision.ron` flow, loaded from disk, runs
/// end-to-end: BootOS(Kvm) → Seed → Configure → InstallApp(korp), with the
/// app-install step reaching the REAL install_product lifecycle.
#[test]
fn korp_provision_ron_file_drives_infra_then_app_install() {
let ron_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples/korp-provision.ron");
let flow = Flow::from_ron_file(&ron_path).expect("parse examples/korp-provision.ron");
assert_eq!(flow.steps.len(), 4, "boot · seed · configure · install");
// korp isn't in the built-in registry — supply its spec + a real bundle.
let korp = ComponentSpec::new("korp", "linux-x86_64/korp", "_korp")
.with_arg("serve")
.with_port(9000);
let (_guard, bundle) = product_bundle(&korp, "20260728");
let install = modules::InstallAppModule::default()
.with_spec(korp.clone())
.with_bundle("korp", &bundle);
let mut registry = Registry::native_default();
registry.register(Box::new(install));
let work = tempfile::tempdir().unwrap();
let mut ctx = RunContext {
work_dir: Some(work.path().to_path_buf()),
..Default::default()
};
let outcomes = run_flow(&flow, ®istry, &mut ctx).expect("run the RON flow");
// Dispatched, in order, to the native-first module names.
let kinds: Vec<&str> = ctx.trace.iter().map(|d| d.module_kind.as_str()).collect();
assert_eq!(kinds, vec!["KvmBoot", "Seed", "Configure", "InstallApp"]);
// Infra step is REAL (a real qemu argv was built from unfold::kvm).
assert!(outcomes[0].real);
assert_eq!(ctx.vars["booted_os"], "ubuntu-24.04");
// The final app-install step ran the REAL lifecycle and landed on disk.
let app = outcomes.last().unwrap();
assert!(app.real, "InstallApp ran install_product");
assert_eq!(app.data["mode"], "install_product");
assert_eq!(ctx.vars["installed_app"], "korp");
let installed = work
.path()
.join("software/current/korp")
.join(&korp.bundle_binary);
assert!(
installed.exists(),
"korp unfolded to {}",
installed.display()
);
assert!(
work.path().join("data/korp/identity.toml").is_file(),
"install identity minted under data/korp"
);
// Configure recorded its param into shared context.
let _: &BTreeMap<String, String> = &ctx.vars;
assert_eq!(ctx.vars["cfg.korp.KORP_DATA"], "/var/lib/korp");
}
/// **The vacuous-pass guard.** A flow with no steps must be REPORTED, not silently
/// succeed. `run_flow` used to return `Ok(vec![])` for an empty `[Step, …]` list,
/// which is the exact false-green this constellation keeps paying for: a deploy that
/// "succeeded" while applying nothing. It must fail with a NAMED error instead.
#[test]
fn empty_flow_is_reported_not_vacuously_passed() {
let flow = Flow::from_ron("[]").expect("`[]` is a valid, empty RON step list");
assert!(flow.steps.is_empty(), "parsed as zero steps");
let registry = Registry::native_default();
let mut ctx = RunContext::default();
let err = run_flow(&flow, ®istry, &mut ctx)
.expect_err("an empty flow must NOT report success — it applied nothing");
let msg = format!("{err:#}");
assert!(
msg.contains("empty flow"),
"the refusal must NAME the fault, got: {msg}"
);
assert!(ctx.trace.is_empty(), "nothing was dispatched");
}
// ───────────── two front ends, ONE implementation (typed API vs RON) ─────────────
/// Every file under `root`, as `(relative path, sha256 of root-normalised bytes)`,
/// sorted.
///
/// The applied-output fingerprint: what actually landed on disk, not what the
/// interpreter said it did.
///
/// The install root is substituted out of every file's *content* before hashing.
/// A rendered systemd unit legitimately embeds its own absolute paths
/// (`ExecStart=<root>/software/current/korp/...`), and the two runs compared here
/// deliberately install into two different roots — so without normalising, the
/// comparison would report a difference that is nothing but the tempdir's name.
/// With it, the claim is the sharper one: identical **modulo the install root**.
fn tree_fingerprint(root: &std::path::Path) -> Vec<(String, String)> {
fn walk(dir: &std::path::Path, root: &std::path::Path, out: &mut Vec<(String, String)>) {
let mut entries: Vec<_> = std::fs::read_dir(dir)
.unwrap()
.map(|e| e.unwrap().path())
.collect();
entries.sort();
for p in entries {
// `software/current` is a symlink into a dated dir; follow files only.
let meta = std::fs::symlink_metadata(&p).unwrap();
if meta.is_dir() {
walk(&p, root, out);
} else if meta.is_file() {
let rel = p.strip_prefix(root).unwrap().to_string_lossy().into_owned();
let bytes = std::fs::read(&p).unwrap();
out.push((rel, nornir_hash::sha256_hex(&normalise_root(&bytes, root))));
}
}
}
let mut out = Vec::new();
walk(root, root, &mut out);
out.sort();
out
}
/// Replace every occurrence of `root`'s path with a fixed token, byte-wise.
fn normalise_root(bytes: &[u8], root: &std::path::Path) -> Vec<u8> {
let needle = root.to_string_lossy().into_owned().into_bytes();
const TOKEN: &[u8] = b"{INSTALL_ROOT}";
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i..].starts_with(&needle) {
out.extend_from_slice(TOKEN);
i += needle.len();
} else {
out.push(bytes[i]);
i += 1;
}
}
out
}
/// **Capability parity, proven on applied output.** The same deployment expressed
/// two ways — a `.ron` file, and direct typed Rust calls — must produce the same
/// bytes on disk.
///
/// This is the property that makes the RON path a *front end* rather than a second
/// implementation. `Flow::from_ron` can only ever produce a `Flow`, and `run_flow`
/// is the only executor, so parity ought to hold by construction — which is exactly
/// the kind of claim that deserves to be measured rather than asserted. If these two
/// paths can ever disagree, that IS the bug.
///
/// Note what is compared: not a status enum, not the trace — the actual file tree,
/// path by path, hashed. And the files that legitimately differ (the per-install
/// identity and key material) are asserted to differ, so this cannot silently pass
/// by comparing two copies of nothing.
///
/// # What this test does NOT discriminate — measured, not assumed
///
/// The file comparison only has teeth over steps that actually write to disk. That
/// is `InstallApp` (the real install lifecycle) and the unit it renders. Verified by
/// deliberate sabotage:
///
/// * giving the second run's `ComponentSpec` an extra `ExecStart` argument **is**
/// caught (`korp.service` differs) — so the comparison is not hollow;
/// * changing the `Configure` step's params between the two paths is **NOT** caught,
/// because `ConfigureModule` is an honest mock: it records into `RunContext::vars`
/// and writes nothing. Same for `Seed` and `Federate`.
///
/// So this proves parity of *applied output* for the real steps, and parity of the
/// parsed `Flow` value for all steps (the `assert_eq!` below). Once Seed/Configure
/// become real, they come under the byte comparison for free — and until they do,
/// claiming this test covers them would be exactly the false green it exists to
/// prevent.
#[test]
fn a_ron_flow_and_direct_typed_calls_apply_byte_identical_output() {
let korp = ComponentSpec::new("korp", "linux-x86_64/korp", "_korp")
.with_arg("serve")
.with_port(9000);
// ── the SAME deployment, said twice ──
//
// 1. as data (this is the text a `.ron` file holds)
const RON: &str = r##"[
BootOS(backend: Kvm, os: "ubuntu-24.04"),
Seed(config: "#cloud-config\npackages: [curl]\n"),
Configure(target: "korp", params: {"KORP_DATA": "/var/lib/korp"}),
InstallApp(spec: "korp"),
]"##;
let from_ron = Flow::from_ron(RON).expect("the declarative front end parses");
// 2. as ordinary typed Rust — no strings, no parsing, no RON synthesised in
// order to call our own deployer. This is the API a consumer calls directly.
let by_hand = Flow {
steps: vec![
skidbladnir::orchestration::Step::BootOS {
backend: skidbladnir::orchestration::BootBackend::Kvm,
os: "ubuntu-24.04".to_string(),
},
skidbladnir::orchestration::Step::Seed {
config: "#cloud-config\npackages: [curl]\n".to_string(),
},
skidbladnir::orchestration::Step::Configure {
target: "korp".to_string(),
params: BTreeMap::from([("KORP_DATA".to_string(), "/var/lib/korp".to_string())]),
},
skidbladnir::orchestration::Step::InstallApp {
spec: "korp".to_string(),
},
],
};
// The front end adds NOTHING: it lands on the very same typed value.
assert_eq!(
from_ron, by_hand,
"RON must deserialise into exactly the typed API's types"
);
// ── run each one against its own filesystem root ──
// The files a correct install makes UNIQUE every time: the time-ordered install
// identity and the per-machine key material. Comparing these for equality would
// be comparing two random numbers; they get the opposite assertion below.
const PER_INSTALL_UNIQUE: &[&str] = &["data/korp/identity.toml", "data/korp/node_key"];
let mut applied = Vec::new();
let mut uniques: Vec<Vec<(String, String)>> = Vec::new();
for flow in [&from_ron, &by_hand] {
let (_bundle_guard, bundle) = product_bundle(&korp, "20260801");
let install = modules::InstallAppModule::default()
.with_spec(korp.clone())
.with_bundle("korp", &bundle);
let mut registry = Registry::native_default();
registry.register(Box::new(install));
let work = tempfile::tempdir().unwrap();
let mut ctx = RunContext {
work_dir: Some(work.path().to_path_buf()),
..Default::default()
};
let outcomes = run_flow(flow, ®istry, &mut ctx).expect("run the flow");
// Both paths really applied something.
assert_eq!(ctx.trace.len(), 4, "all four steps dispatched");
assert!(
outcomes.last().unwrap().real,
"the install step ran the real lifecycle"
);
let installed = work
.path()
.join("software/current/korp")
.join(&korp.bundle_binary);
assert!(
installed.is_file(),
"korp landed at {}",
installed.display()
);
uniques.push(
PER_INSTALL_UNIQUE
.iter()
.map(|rel| {
let p = work.path().join(rel);
let bytes = std::fs::read(&p)
.unwrap_or_else(|e| panic!("{rel} must exist after an install: {e}"));
(rel.to_string(), nornir_hash::sha256_hex(&bytes))
})
.collect(),
);
applied.push(tree_fingerprint(work.path()));
// `work` drops here, so the comparison below is on captured facts.
}
let (ron_tree, typed_tree) = (&applied[0], &applied[1]);
// Guard against a vacuous pass: an empty tree would compare equal to an empty
// tree and prove nothing at all.
assert!(
ron_tree.len() >= 3,
"the deployment must have written a real tree, got {} files: {ron_tree:?}",
ron_tree.len()
);
// Same set of paths, both ways.
let paths = |t: &Vec<(String, String)>| t.iter().map(|(p, _)| p.clone()).collect::<Vec<_>>();
assert_eq!(
paths(ron_tree),
paths(typed_tree),
"the two front ends wrote different paths"
);
// Same BYTES, both ways — except the per-install unique files.
let mut compared = 0usize;
for ((p1, h1), (p2, h2)) in ron_tree.iter().zip(typed_tree.iter()) {
assert_eq!(p1, p2);
if PER_INSTALL_UNIQUE.contains(&p1.as_str()) {
continue;
}
assert_eq!(h1, h2, "`{p1}` differs between the RON and the typed path");
compared += 1;
}
assert!(
compared > 0,
"every file got excluded — the byte comparison proved nothing"
);
// …and prove the exclusions are not hiding a hole. Each one must be PRESENT in
// the applied output and DIFFERENT between the two runs: identical here would
// mean the runs shared state, which would make the whole comparison hollow.
for rel in PER_INSTALL_UNIQUE {
assert!(
ron_tree.iter().any(|(p, _)| p == rel),
"{rel} must be part of the applied output"
);
}
assert_ne!(
uniques[0], uniques[1],
"each install must mint its OWN identity and key material"
);
for ((r1, h1), (_, h2)) in uniques[0].iter().zip(uniques[1].iter()) {
assert_ne!(
h1, h2,
"{r1} was reused across two installs instead of being freshly minted"
);
}
}
// ───────────── the link-drop path: resume without redoing applied work ─────────────
/// A steering module that APPENDS a line to a file every time it runs, and can be
/// told to fail. The append is the applied output: counting lines answers "did this
/// step run again?" without trusting any status the interpreter reports.
struct Appender {
kind: &'static str,
log: PathBuf,
fail: bool,
}
impl skidbladnir::orchestration::SteeringModule for Appender {
fn kind(&self) -> &'static str {
self.kind
}
fn run(
&self,
step: &skidbladnir::orchestration::Step,
_ctx: &mut RunContext,
) -> anyhow::Result<skidbladnir::orchestration::StepOutcome> {
use std::io::Write;
let label = match step {
skidbladnir::orchestration::Step::Configure { target, .. } => target.clone(),
other => format!("{other:?}"),
};
if self.fail {
anyhow::bail!("the link dropped during `{label}`");
}
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&self.log)
.unwrap();
writeln!(f, "applied {label}").unwrap();
Ok(skidbladnir::orchestration::StepOutcome {
module_kind: self.kind.to_string(),
detail: format!("appended {label}"),
data: BTreeMap::new(),
real: true,
})
}
}
/// **A resumed deployment must not redo work that already landed.**
///
/// Proven on applied output: each step appends one line to a file, so the file IS
/// the record of what was applied. A flow of four steps is interrupted at step 3;
/// the resume must bring the total to four lines, not seven.
///
/// Seen red first, in the same test: re-running the plain `run_flow` after the
/// failure re-applies the first two steps (4 lines from 2 steps), which is exactly
/// the behaviour a link drop over an unreliable connection must not have.
#[test]
fn a_resumed_flow_skips_what_already_landed_and_reruns_nothing() {
let dir = tempfile::tempdir().unwrap();
let log = dir.path().join("applied.log");
let journal_path = dir.path().join("state/resume.ron");
let flow = Flow::from_ron(
r#"[
Configure(target: "one", params: {}),
Configure(target: "two", params: {}),
Configure(target: "three", params: {}),
Configure(target: "four", params: {}),
]"#,
)
.unwrap();
let lines = || match std::fs::read_to_string(&log) {
Ok(s) => s.lines().map(str::to_string).collect::<Vec<_>>(),
Err(_) => Vec::new(),
};
// A registry whose `Configure` module appends — and one that fails, to stand in
// for the link dropping mid-deployment.
let good = || {
let mut r = Registry::native_default();
r.register(Box::new(Appender {
kind: "Configure",
log: log.clone(),
fail: false,
}));
r
};
let dropped = || {
let mut r = Registry::native_default();
r.register(Box::new(Appender {
kind: "Configure",
log: log.clone(),
fail: true,
}));
r
};
// ── attempt 1: two steps land, then the link drops ──
// Run the first two through a journal, then hit the failure on step 3.
let two_steps = Flow {
steps: flow.steps[..2].to_vec(),
};
let mut journal = skidbladnir::orchestration::Journal::open(&journal_path).unwrap();
let mut ctx = RunContext::default();
skidbladnir::orchestration::resume_flow(&two_steps, &good(), &mut ctx, &mut journal).unwrap();
assert_eq!(
lines(),
vec!["applied one", "applied two"],
"two steps applied"
);
assert_eq!(journal.entries().len(), 2, "and both were journalled");
let mut ctx = RunContext::default();
let err = skidbladnir::orchestration::resume_flow(&flow, &dropped(), &mut ctx, &mut journal)
.expect_err("step 3 must fail");
assert!(format!("{err:#}").contains("link dropped"), "{err:#}");
assert_eq!(lines().len(), 2, "a failed step applied nothing new");
// ── RED: the plain path redoes everything ──
// `run_flow` has no memory, so re-running after the drop re-applies steps 1-2.
let mut ctx = RunContext::default();
let _ = run_flow(&two_steps, &good(), &mut ctx).unwrap();
assert_eq!(
lines(),
vec!["applied one", "applied two", "applied one", "applied two"],
"this is the behaviour the journal exists to fix: work re-applied on retry"
);
// Reset the applied-output log to the state the journal describes, so the
// resume below is measured from the same place the drop left it.
std::fs::write(&log, "applied one\napplied two\n").unwrap();
// ── GREEN: resume from the journal ──
let mut journal =
skidbladnir::orchestration::Journal::open(&journal_path).expect("journal survived on disk");
assert_eq!(
journal.entries().len(),
2,
"read back from disk, not from memory"
);
let mut ctx = RunContext::default();
let report =
skidbladnir::orchestration::resume_flow(&flow, &good(), &mut ctx, &mut journal).unwrap();
assert_eq!(
report.skipped,
vec![0, 1],
"steps 0-1 were skipped, not re-run"
);
assert_eq!(report.ran.len(), 2, "only steps 2-3 executed");
assert_eq!(
lines(),
vec![
"applied one",
"applied two",
"applied three",
"applied four"
],
"the deployment completed exactly once end to end"
);
// Running again now is a no-op: everything is done.
let mut ctx = RunContext::default();
let again =
skidbladnir::orchestration::resume_flow(&flow, &good(), &mut ctx, &mut journal).unwrap();
assert!(again.fully_resumed(), "a completed flow re-runs nothing");
assert_eq!(lines().len(), 4, "and applies nothing further");
}
/// **A flow that changed under its journal is refused, not half-applied.**
///
/// Resuming an edited flow would run the new step 3 on top of the old steps 1-2 and
/// report success for a deployment that never existed as a whole.
#[test]
fn resuming_a_changed_flow_is_refused_by_name() {
let dir = tempfile::tempdir().unwrap();
let log = dir.path().join("applied.log");
let journal_path = dir.path().join("resume.ron");
let reg = || {
let mut r = Registry::native_default();
r.register(Box::new(Appender {
kind: "Configure",
log: log.clone(),
fail: false,
}));
r
};
let first = Flow::from_ron(r#"[Configure(target: "one", params: {})]"#).unwrap();
let mut journal = skidbladnir::orchestration::Journal::open(&journal_path).unwrap();
let mut ctx = RunContext::default();
skidbladnir::orchestration::resume_flow(&first, ®(), &mut ctx, &mut journal).unwrap();
// Someone edits step 0 and re-runs.
let edited = Flow::from_ron(
r#"[Configure(target: "one-EDITED", params: {}), Configure(target: "two", params: {})]"#,
)
.unwrap();
let mut journal = skidbladnir::orchestration::Journal::open(&journal_path).unwrap();
let mut ctx = RunContext::default();
let err = skidbladnir::orchestration::resume_flow(&edited, ®(), &mut ctx, &mut journal)
.expect_err("a changed flow must not resume");
let msg = format!("{err:#}");
assert!(
msg.contains("changed since the journal recorded it"),
"must name the fault: {msg}"
);
assert!(msg.contains("step 0"), "must name the step: {msg}");
// And nothing further was applied.
assert_eq!(std::fs::read_to_string(&log).unwrap().lines().count(), 1);
}