server/scan-worker/tests/jobs_through_the_sidecar.rs
//! **A job runs in the real worker** (lane HOL-SANDBOX-TRAIT): the real
//! `holger-scan-worker serve` sidecar, Holger's interim [`ContainerRunner`] over a
//! `Remote` sandbox that asks it, and the server's [`Jobs`] that believes only the
//! signed result. The worker reads `/in/job.json` as a `holger.job/v1` document, runs
//! the named program over its input and answers; the runner hashes what it wrote.
//!
//! * a good bundle is verified in the worker, and its members come back as files;
//! * a malformed bundle is the job's own refusal, by name, and the sidecar stays up;
//! * a program the worker does not have is refused by name.
mod common;
use common::signed_bundle;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::sync::Arc;
use std::time::{Duration, Instant};
use holger_sandbox::{EngineKind, Sandbox, SandboxConfig};
use server_lib::job_runner::heimdall_core::{Job, JobId, Nonce, Policy, Verdict};
use server_lib::job_runner::holger_job_runner::{worker, ContainerRunner};
use server_lib::job_runner::verify_bundle as vb;
use server_lib::job_runner::{JobDirs, Jobs};
struct Sidecar {
child: Child,
tmp: tempfile::TempDir,
share: PathBuf,
socket: PathBuf,
}
impl Drop for Sidecar {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
}
fn start() -> Sidecar {
let tmp = tempfile::tempdir().unwrap();
let share = tmp.path().join("share");
let scratch = tmp.path().join("scratch");
std::fs::create_dir_all(&share).unwrap();
std::fs::create_dir_all(&scratch).unwrap();
let socket = share.join("worker.sock");
let child = Command::new(env!("CARGO_BIN_EXE_holger-scan-worker"))
.args(["serve", "--socket"])
.arg(&socket)
.arg("--share")
.arg(&share)
.arg("--scratch")
.arg(&scratch)
.stdin(Stdio::null())
.spawn()
.expect("the sidecar starts");
let t = Instant::now();
while !socket.exists() {
assert!(
t.elapsed() < Duration::from_secs(20),
"the sidecar never bound {}",
socket.display()
);
std::thread::sleep(Duration::from_millis(20));
}
Sidecar {
child,
tmp,
share,
socket,
}
}
fn jobs(side: &Sidecar) -> Jobs {
let cfg = SandboxConfig {
engine: EngineKind::Remote,
socket: Some(side.socket.display().to_string()),
work_dir: Some(side.share.display().to_string()),
..Default::default()
};
Jobs::new(Arc::new(ContainerRunner::new(
Sandbox::from_config(cfg).unwrap(),
)))
}
/// A verify-bundle job over `bundle` with `policy`, its directories under `base`.
fn verify_job(
base: &Path,
bundle: &[u8],
policy: &update_bundle::TrustPolicy,
argv: &[&str],
) -> (Job, JobDirs) {
let id = JobId::random();
let dirs = JobDirs::new(base, &id).unwrap();
std::fs::write(dirs.input.join(vb::BUNDLE_FILE), bundle).unwrap();
std::fs::write(
dirs.input.join(vb::POLICY_FILE),
serde_json::to_vec(policy).unwrap(),
)
.unwrap();
let params = serde_json::to_value(vb::Params {
time: vb::TimeParam::At {
unix_ms: chrono::Utc::now().timestamp_millis(),
trusted: false,
},
product: None,
model: None,
revision: None,
guest: None,
})
.unwrap();
let job = Job {
id,
nonce: Nonce::random(),
kind: worker(argv),
input_dir: dirs.input.clone(),
output_dir: dirs.output.clone(),
params,
policy: Policy::default(),
};
(job, dirs)
}
#[test]
fn a_good_bundle_is_verified_in_the_worker_and_its_members_come_back() {
let side = start();
let j = jobs(&side);
let (bundle, policy) = signed_bundle(side.tmp.path());
let (job, dirs) = verify_job(
&side.tmp.path().join("jobs"),
&std::fs::read(&bundle).unwrap(),
&policy,
&[vb::PROGRAM],
);
let r = j.run(job).expect("a believed result");
assert_eq!(r.verdict, Verdict::Accepted, "{r:?}");
assert_eq!(r.runner.runner, "holger-sidecar");
let (members, manifest, raw) = vb::read_outputs(&dirs.output).unwrap();
assert_eq!(
members.get("payload/firmware.bin").map(Vec::as_slice),
Some(&b"hello"[..])
);
assert!(!manifest.export);
assert_eq!(
raw.unwrap(),
std::fs::read(&bundle).unwrap(),
"the bytes that verified come back"
);
}
#[test]
fn a_malformed_bundle_is_the_jobs_refusal_and_the_sidecar_stays_up() {
let side = start();
let j = jobs(&side);
let (_, policy) = signed_bundle(side.tmp.path());
let (job, dirs) = verify_job(
&side.tmp.path().join("jobs"),
b"ZNIPPY not an archive",
&policy,
&[vb::PROGRAM],
);
let r = j.run(job).expect("a believed result");
match &r.verdict {
Verdict::Refused { code, reason } => {
assert_eq!(code, "holger-import-5");
assert!(reason.contains("bundle verification failed"), "{reason}");
}
v => panic!("{v:?}"),
}
assert!(r.outputs.is_empty() && std::fs::read_dir(&dirs.output).unwrap().next().is_none());
// The sidecar still answers the next job.
let (bundle, policy) = signed_bundle(side.tmp.path());
let (job, _d) = verify_job(
&side.tmp.path().join("jobs"),
&std::fs::read(&bundle).unwrap(),
&policy,
&[vb::PROGRAM],
);
assert_eq!(j.run(job).unwrap().verdict, Verdict::Accepted);
}
#[test]
fn a_program_the_worker_does_not_have_is_refused_by_name() {
let side = start();
let j = jobs(&side);
let (_, policy) = signed_bundle(side.tmp.path());
let (job, _d) = verify_job(
&side.tmp.path().join("jobs"),
b"x",
&policy,
&["format-the-disk"],
);
match j.run(job).unwrap().verdict {
Verdict::Refused { code, reason } => {
assert_eq!(code, "holger-sandbox-10");
assert!(reason.contains("format-the-disk"), "{reason}");
}
v => panic!("{v:?}"),
}
}
// ── the released split guest (lane HOL-HEIMDALL-02) ─────────────────────────────────
/// Holger's own `verify-bundle` program over `bundle` (in this process; the worker runs
/// the same code): its verdict.
fn holgers_verdict(base: &Path, bundle: &[u8], policy: &update_bundle::TrustPolicy) -> Verdict {
use server_lib::job_runner::holger_job_runner::wire::{Dirs, WireJob, JOB_SCHEMA};
let (job, dirs) = verify_job(base, bundle, policy, &[vb::PROGRAM]);
let w = WireJob {
schema: JOB_SCHEMA.into(),
job_id: job.id.to_hex(),
argv: vec![vb::PROGRAM.into()],
params: job.params.clone(),
reads: 0,
answer: None,
};
let d = Dirs {
input: dirs.input.clone(),
read: vec![],
out: dirs.output.clone(),
};
server_lib::job_runner::programs::run(&w, &d).verdict
}
/// The released split guest (vendored, pinned, signed under the korp root) as a job of
/// Heimdall's wasm runner over `bundle`: its believed result and its output directory.
fn split_guests_result(
base: &Path,
bundle: &[u8],
policy: &update_bundle::TrustPolicy,
) -> (
server_lib::job_runner::heimdall_core::JobResult,
tempfile::TempDir,
) {
use base64::Engine as _;
use holger_job_runner::guests;
let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let pem = std::fs::read_to_string(repo.join("holger-recipe/trust/korp-root-ca.pem")).unwrap();
let der = base64::engine::general_purpose::STANDARD
.decode(
pem.lines()
.filter(|l| !l.starts_with("-----"))
.collect::<String>(),
)
.unwrap();
std::fs::create_dir_all(base).unwrap();
let root = base.join("korp-root-ca.der");
std::fs::write(&root, der).unwrap();
let input = tempfile::tempdir_in(base).unwrap();
std::fs::write(input.path().join("bundle.holger.znippy"), bundle).unwrap();
let out = tempfile::tempdir_in(base).unwrap();
let runner = heimdall_run::WasmJobRunner::new(vec![root], base.to_path_buf());
let job = Job::new(
guests::split_kind(&repo.join(guests::DIR), &guests::RELEASE),
input.path().to_path_buf(),
out.path().to_path_buf(),
serde_json::json!({
"now": chrono::Utc::now().to_rfc3339(),
"policy": policy.to_ron().unwrap(),
"product": "ExampleDrive",
}),
Policy::default(),
);
let ticket = job.ticket();
use server_lib::job_runner::heimdall_core::JobRunner as _;
let signed = runner.run(job).expect("the guest ran");
let r = server_lib::job_runner::heimdall_core::verify(&signed, &runner.verifying_key())
.unwrap()
.clone();
assert_eq!((r.job_id, r.nonce), (ticket.job_id, ticket.nonce));
server_lib::job_runner::heimdall_core::check_outputs(&r, out.path()).unwrap();
(r, out)
}
/// The released split guest decides a Holger bundle as Holger's own verifier does: a good
/// one is accepted with its payload, one with a changed byte is refused by both.
#[test]
fn the_released_split_guest_decides_a_bundle_as_holgers_own_verifier() {
let tmp = tempfile::tempdir().unwrap();
let (bundle, policy) = signed_bundle(tmp.path());
let good = std::fs::read(&bundle).unwrap();
assert_eq!(
holgers_verdict(&tmp.path().join("jobs"), &good, &policy),
Verdict::Accepted
);
let (r, out) = split_guests_result(&tmp.path().join("wasm"), &good, &policy);
assert_eq!(r.verdict, Verdict::Accepted, "{r:?}");
assert_eq!(r.runner.runner, "heimdall-wasm");
assert!(r.runner.wasm);
let payload = r
.outputs
.iter()
.find(|o| o.name.contains("firmware.bin"))
.unwrap_or_else(|| panic!("the payload among {:?}", r.outputs));
assert_eq!(
std::fs::read(out.path().join(&payload.name)).unwrap(),
b"hello"
);
assert!(r.outputs.iter().any(|o| o.name == "verify-report.json"));
let mut bad = good.clone();
let n = bad.len() / 2;
bad[n] ^= 0x40;
assert!(matches!(
holgers_verdict(&tmp.path().join("jobs2"), &bad, &policy),
Verdict::Refused { .. }
));
let (r, _out) = split_guests_result(&tmp.path().join("wasm2"), &bad, &policy);
assert!(matches!(r.verdict, Verdict::Refused { .. }), "{r:?}");
assert!(r.outputs.is_empty());
}
// ── the verify-bundle program through the released split guest (lane HOL-HEIMDALL-04) ──
/// Holger's own `verify-bundle` program over `bundle`, through the released split guest when
/// `guests` names the release's directory: the answer and the job's directories.
fn verify_program(
base: &Path,
bundle: &[u8],
policy: &update_bundle::TrustPolicy,
guests: Option<&Path>,
) -> (
server_lib::job_runner::holger_job_runner::wire::Answer,
JobDirs,
) {
use server_lib::job_runner::holger_job_runner::wire::{Dirs, WireJob, JOB_SCHEMA};
let (job, dirs) = verify_job(base, bundle, policy, &[vb::PROGRAM]);
let mut params: vb::Params = serde_json::from_value(job.params.clone()).unwrap();
let mut read = Vec::new();
if let Some(g) = guests {
let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let pem = std::fs::read(repo.join("holger-recipe/trust/korp-root-ca.pem")).unwrap();
std::fs::write(
dirs.input.join(vb::GUEST_ROOT_FILE),
holger_job_runner::guests::der_of(&pem).unwrap(),
)
.unwrap();
params.guest = Some(vb::GuestParam { read: 0 });
read.push(g.to_path_buf());
}
let w = WireJob {
schema: JOB_SCHEMA.into(),
job_id: job.id.to_hex(),
argv: vec![vb::PROGRAM.into()],
params: serde_json::to_value(¶ms).unwrap(),
reads: read.len(),
answer: None,
};
let d = Dirs {
input: dirs.input.clone(),
read,
out: dirs.output.clone(),
};
(server_lib::job_runner::programs::run(&w, &d), dirs)
}
/// The program verifies through the released split guest and hands the door exactly the
/// members, manifest and bytes its native verification hands it; a changed byte is refused
/// both ways; and the guest path is the one that ran (a decoder that is not the pinned one
/// refuses a bundle the native path accepts).
#[test]
fn the_verify_bundle_program_runs_the_released_split_guest_and_gives_the_natives_members() {
let tmp = tempfile::tempdir().unwrap();
let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
let guests = repo.join(holger_job_runner::guests::DIR);
let (bundle, policy) = signed_bundle(tmp.path());
let good = std::fs::read(&bundle).unwrap();
let (native, nd) = verify_program(&tmp.path().join("n"), &good, &policy, None);
let (split, sd) = verify_program(&tmp.path().join("g"), &good, &policy, Some(&guests));
assert_eq!(native.verdict, Verdict::Accepted, "{native:?}");
assert_eq!(split.verdict, Verdict::Accepted, "{split:?}");
let n = vb::read_outputs(&nd.output).unwrap();
let g = vb::read_outputs(&sd.output).unwrap();
assert_eq!(g, n, "the same members, manifest and bundle bytes");
assert_eq!(
g.0.get("payload/firmware.bin").map(Vec::as_slice),
Some(&b"hello"[..])
);
let mut bad = good.clone();
let half = bad.len() / 2;
bad[half] ^= 0x40;
let (n, _) = verify_program(&tmp.path().join("n2"), &bad, &policy, None);
let (g, _) = verify_program(&tmp.path().join("g2"), &bad, &policy, Some(&guests));
assert!(matches!(n.verdict, Verdict::Refused { .. }), "{n:?}");
let Verdict::Refused { code, reason } = &g.verdict else {
panic!("{g:?}")
};
assert!(code.starts_with("holger-"), "{code}: {reason}");
assert!(g.outputs.is_empty());
// A decoder that is not the pinned one: the guest path refuses what the native accepts.
let other = tmp.path().join("other-guests");
std::fs::create_dir_all(&other).unwrap();
for e in std::fs::read_dir(&guests).unwrap().flatten() {
std::fs::copy(e.path(), other.join(e.file_name())).unwrap();
}
let dec = other.join("heimdall-verify-bundle-decoder.wasm");
let mut b = std::fs::read(&dec).unwrap();
let mid = b.len() / 2;
b[mid] ^= 1;
std::fs::write(&dec, b).unwrap();
let (g, _) = verify_program(&tmp.path().join("g3"), &good, &policy, Some(&other));
let Verdict::Refused { code, reason } = &g.verdict else {
panic!("{g:?}")
};
assert_eq!(code, "holger-import-5", "{reason}");
assert!(
reason.contains("decoder") && reason.contains("sha256"),
"{reason}"
);
}