server/lib/src/job_runner/mod.rs
//! # Holger's untrusted work runs as jobs (lane HOL-SANDBOX-TRAIT, owner 2026-10-30)
//!
//! Every piece of work over bytes Holger does not trust (verifying a bundle that came
//! through the inbox, a door or the CLI; the scans) is a [`heimdall_core::Job`] run by
//! the ONE [`Jobs`] of this server: a runner chosen once at boot, its per-boot key
//! pinned at that moment. The server accepts only a result that
//! [`holger_job_runner::accept`] believes (the pinned key, the job id it issued,
//! exactly the listed outputs with their hashes) and uses only those outputs. It
//! never opens the untrusted input itself: it links it into the job's input
//! directory (a hard link, no open) and reads back the job's verified output files.
//!
//! | `promotion.intake.rules.sandbox` | The runner |
//! |---|---|
//! | `required` (the default) | [`holger_job_runner::ContainerRunner`] over the `promotion.intake.sandbox` block: the INTERIM runner (podman, its API socket, crun, the sidecar), not Heimdall |
//! | `off` | the in-process test runner: a test build only (`sandbox-off`); a release build refuses it at boot (`holger-sandbox-6`) |
//! | `required`, `sandbox: (engine: heimdall, heimdall: (…))` | `holger_job_runner::heimdall::HeimdallJobs` over Heimdall's protected runner (feature `heimdall`, lane HOL-HEIMDALL-02); youki-plain, then crun over `rootfs`, where Heimdall cannot start, each said by name ([`Jobs::fallback`]) |
//!
//! Heimdall (`git.gunnar.rs/nordisk/heimdall`) drops in by changing which runner
//! [`Jobs::from_config`] builds; no caller changes. What the runner in use has, and
//! has not, is stated by [`Jobs::features`]: the Security page and the CLI show it,
//! and in strict CDS mode (`boundary: OneWay`) the start warns that the runner is the
//! interim one.
//!
//! The programs a job runs ([`programs`]) are compiled into `holger-scan-worker`,
//! the one program the sandbox starts, and into the test runner.
pub mod programs;
pub mod verify_bundle;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use heimdall_core::{Features, Job, JobId, JobResult, JobRunner, ReplayGuard, VerifyingKey};
use holger_errcode::ErrCode;
pub use holger_job_runner;
pub use holger_job_runner::heimdall_core;
/// The sandbox config (the `engine: heimdall` block among it), for callers and tests.
pub use holger_sandbox;
use holger_sandbox::{Refusal, Sandbox, SandboxConfig};
use serde::{Deserialize, Serialize};
/// Where each job's input and output directories are made, under the data root.
pub const JOBS_DIR: &str = ".holger-jobs";
/// Can this build run a job in-process (`sandbox: off`)? Only a test build.
pub const IN_PROCESS_ALLOWED: bool = cfg!(any(test, feature = "sandbox-off"));
/// `required` (the default) or `off` (a test build only).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SandboxRule {
#[default]
Required,
Off,
}
/// The intake's `rules:` (G0+G2's vocabulary; lane SANDBOX-SCAN added `sandbox`).
///
/// ```ron
/// rules: (sandbox: required),
/// ```
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct IntakeRules {
pub sandbox: SandboxRule,
}
impl SandboxRule {
/// Refuse `off` in a release build, by name.
pub fn check(self) -> Result<(), Refusal> {
check_in(self, IN_PROCESS_ALLOWED)
}
}
/// [`SandboxRule::check`] for a build that does (`true`) or does not allow
/// running in-process.
pub fn check_in(rule: SandboxRule, in_process_allowed: bool) -> Result<(), Refusal> {
match (rule, in_process_allowed) {
(SandboxRule::Off, false) => Err(Refusal::OffInRelease),
_ => Ok(()),
}
}
/// A job that did not produce a result the server believes, by name.
#[derive(Debug, Clone)]
pub struct JobFailure {
pub code: &'static ErrCode,
pub message: String,
}
impl std::fmt::Display for JobFailure {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.code.tag(&self.message))
}
}
/// **The jobs of one server**: its runner and the runner's key, pinned when this
/// was made.
pub struct Jobs {
runner: Arc<dyn JobRunner>,
pinned: VerifyingKey,
/// Every result this process accepted, by job: the same result twice is a replay.
seen: std::sync::Mutex<ReplayGuard>,
runs: AtomicU64,
/// What Heimdall's runner found in the kernel at its start (lane HOL-HEIMDALL-02).
kernel: Option<heimdall_core::KernelFeatures>,
/// Why the config's runner is not the one in use (`engine: heimdall` that fell back).
fallback: Option<String>,
/// Heimdall's released verify-bundle guests (lane HOL-HEIMDALL-04): checked against their
/// pins when this was made; an `Err` refuses every bundle import by name (fail closed).
guests: Option<Result<VerifyGuests, String>>,
}
/// **The verify-bundle guests every bundle job runs** (lane HOL-HEIMDALL-04): the laid release
/// (every file its pin) and the one trust root they are verified against, as DER.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VerifyGuests {
pub dir: PathBuf,
pub root_der: Vec<u8>,
}
impl VerifyGuests {
/// The guests in `dir`, each file against Holger's own pins, verified later (inside the
/// job) against `root` (a PEM or DER certificate file).
#[cfg(feature = "heimdall")]
pub fn check(dir: &Path, root: &Path) -> Result<Self, String> {
use holger_job_runner::guests;
guests::check_release(dir, &guests::RELEASE)
.map_err(|e| format!("the verify-bundle guests in {}: {e}", dir.display()))?;
let bytes = std::fs::read(root)
.map_err(|e| format!("the guests' trust root {}: {e}", root.display()))?;
Ok(Self {
dir: dir.to_path_buf(),
root_der: guests::der_of(&bytes)?,
})
}
}
impl std::fmt::Debug for Jobs {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Jobs")
.field("runner", &self.runner.features().runner)
.field("key", &self.key_id())
.finish()
}
}
impl Jobs {
/// The jobs over `runner`, its key pinned now.
pub fn new(runner: Arc<dyn JobRunner>) -> Self {
let pinned = runner.verifying_key();
Self {
runner,
pinned,
seen: std::sync::Mutex::new(ReplayGuard::new()),
runs: AtomicU64::new(0),
kernel: None,
fallback: None,
guests: None,
}
}
/// These jobs with the verify-bundle guests (tests; the config's `heimdall.guests`).
pub fn with_guests(mut self, g: Result<VerifyGuests, String>) -> Self {
self.guests = Some(g);
self
}
/// The verify-bundle guests every bundle job runs, if the config names them.
pub fn guests(&self) -> Option<&Result<VerifyGuests, String>> {
self.guests.as_ref()
}
/// The kernel features the runner found at its start (Heimdall's runner only).
pub fn kernel(&self) -> Option<&heimdall_core::KernelFeatures> {
self.kernel.as_ref()
}
/// Why the runner in use is not the one the config asked for (`engine: heimdall`
/// falling back to youki-plain or crun), by name.
pub fn fallback(&self) -> Option<&str> {
self.fallback.as_deref()
}
/// **`engine: heimdall`** (lane HOL-HEIMDALL-02): Heimdall's protected runner; where it
/// cannot start, youki-plain (its shim there), then the interim crun runner over
/// `rootfs`; neither refuses the boot (`holger-sandbox-1`). Each fallback is said by name.
#[cfg(feature = "heimdall")]
fn heimdall(cfg: SandboxConfig) -> Result<Self, Refusal> {
cfg.validate()?;
let h = cfg.heimdall.clone().unwrap_or_default();
let work = h
.work_dir
.clone()
.or_else(|| cfg.state_dir.clone())
.or_else(|| cfg.work_dir.clone())
.map(PathBuf::from)
.unwrap_or_else(std::env::temp_dir);
let guests = match (&h.guests, &h.guests_trust_root) {
(Some(d), Some(r)) => Some(VerifyGuests::check(Path::new(d), Path::new(r))),
_ => None,
};
let with = |mut j: Self| {
j.guests = guests.clone();
j
};
match holger_job_runner::heimdall::start(&h, &work, cfg.require_rootless, cfg.userns_base) {
Ok(st) => {
let kernel = st.jobs.kernel().cloned();
let mut j = Self::new(Arc::new(st.jobs));
j.kernel = kernel;
j.fallback = st.fallback;
Ok(with(j))
}
Err(why) => match cfg.crun_fallback() {
Some(c) => {
let mut j = Self::new(Arc::new(holger_job_runner::ContainerRunner::new(
Sandbox::from_config(c)?,
)));
j.fallback = Some(format!(
"{why}: the jobs run in the interim crun runner, not Heimdall"
));
Ok(with(j))
}
None => Err(Refusal::NoEngine(format!(
"engine `heimdall`: {why}; and no `rootfs` for the crun fallback"
))),
},
}
}
/// A build without Heimdall (lane HOL-HEIMDALL-03): the crun fallback over `rootfs` when the
/// config names one, said by name, so a medium whose binary lacks the feature still scans;
/// without `rootfs` the engine is refused (`holger-sandbox-10`).
#[cfg(not(feature = "heimdall"))]
fn heimdall(cfg: SandboxConfig) -> Result<Self, Refusal> {
cfg.validate()?;
let why = "engine `heimdall`: this build has no Heimdall runner (feature `heimdall`)";
match cfg.crun_fallback() {
Some(c) => {
let mut j = Self::new(Arc::new(holger_job_runner::ContainerRunner::new(
Sandbox::from_config(c)?,
)));
j.fallback = Some(format!(
"{why}: the jobs run in the interim crun runner, not Heimdall"
));
Ok(j)
}
None => Err(Refusal::Config(why.into())),
}
}
/// The runner the config declares (`promotion.intake.rules.sandbox` and the
/// `promotion.intake.sandbox` block). `off` in a release build is refused
/// (`holger-sandbox-6`).
pub fn from_config(rule: SandboxRule, cfg: Option<SandboxConfig>) -> Result<Self, Refusal> {
rule.check()?;
match rule {
SandboxRule::Required => {
let cfg = cfg.unwrap_or_default();
if cfg.engine == holger_sandbox::EngineKind::Heimdall {
return Self::heimdall(cfg);
}
let sb = Sandbox::from_config(cfg)?;
Ok(Self::new(Arc::new(
holger_job_runner::ContainerRunner::new(sb),
)))
}
SandboxRule::Off => Self::in_process(),
}
}
#[cfg(feature = "sandbox-off")]
fn in_process() -> Result<Self, Refusal> {
Ok(Self::new(Arc::new(
holger_job_runner::InProcessRunner::new(Arc::new(programs::run)),
)))
}
#[cfg(not(feature = "sandbox-off"))]
fn in_process() -> Result<Self, Refusal> {
Err(Refusal::OffInRelease)
}
/// What the runner in use has, stated truthfully.
pub fn features(&self) -> Features {
self.runner.features()
}
/// Does the runner run jobs in this very process (the test runner)?
pub fn is_in_process(&self) -> bool {
self.features().runner == "holger-in-process-test"
}
/// Is this the interim runner (anything but Heimdall)?
pub fn is_interim(&self) -> bool {
self.features().runner != "heimdall"
}
/// The pinned key's id.
pub fn key_id(&self) -> String {
holger_job_runner::key_words(&self.pinned)
}
/// How many jobs ran.
pub fn runs(&self) -> u64 {
self.runs.load(Ordering::Relaxed)
}
/// The one-line words for the boot log and the CLI.
pub fn words(&self) -> String {
let f = self.features();
let missing: Vec<&str> = holger_job_runner::feature_rows(&f)
.into_iter()
.filter(|(k, has, _)| !has && *k != "runner")
.map(|(k, _, _)| k)
.collect();
format!(
"jobs run in {}{}; it has no {}{}",
f.runner,
if self.is_interim() {
" (the interim runner, not Heimdall)"
} else {
""
},
if missing.is_empty() {
"gap".to_string()
} else {
missing.join(", ")
},
match &self.fallback {
Some(why) => format!(" ({why})"),
None => String::new(),
}
)
}
/// ★ **Run `job` and believe its result** only through
/// [`holger_job_runner::accept`] (Heimdall's `verify` with the pinned key, `job`'s own
/// id and nonce, `check_outputs` on `job.output_dir`), and only once: the same
/// result offered again is a replay ([`ReplayGuard`]).
pub fn run(&self, job: Job) -> Result<JobResult, JobFailure> {
use crate::lock::LockOrRecover as _;
let (ticket, out) = (job.ticket(), job.output_dir.clone());
let id = ticket.job_id;
self.runs.fetch_add(1, Ordering::Relaxed);
let signed = self.runner.run(job).map_err(|e| JobFailure {
code: &holger_errcode::SANDBOX_JOB_RUN_FAILED,
message: format!("job {id} did not run: {e}"),
})?;
let r = holger_job_runner::accept(&signed, &self.pinned, &ticket, &out)
.cloned()
.map_err(|r| JobFailure {
code: r.code,
message: format!("job {id}: {}", r.why),
})?;
self.seen
.lock_or_recover()
.check_and_record(&signed)
.map_err(|e| JobFailure {
code: &holger_errcode::SANDBOX_JOB_RESULT_REFUSED,
message: format!("job {id}: {e}"),
})?;
Ok(r)
}
}
/// One job's directories under `<base>/<job id>/`: `in` and a fresh, empty `out`.
/// Removed when dropped.
pub struct JobDirs {
base: PathBuf,
root: PathBuf,
pub input: PathBuf,
pub output: PathBuf,
}
impl JobDirs {
pub fn new(base: &Path, id: &JobId) -> std::io::Result<Self> {
let root = base.join(id.to_hex());
let (input, output) = (root.join("in"), root.join("out"));
std::fs::create_dir_all(&input)?;
std::fs::create_dir_all(&output)?;
Ok(Self {
base: base.to_path_buf(),
root,
input,
output,
})
}
}
impl Drop for JobDirs {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
// The shared base too, when this was its last job (`remove_dir` takes an empty
// directory only, so a job still running keeps it).
let _ = std::fs::remove_dir(&self.base);
}
}
/// Put `file` into a job's input as `name` without opening it: a hard link. Only
/// when the two are on different file systems is it copied (and that is logged).
pub fn link_input(file: &Path, input: &Path, name: &str) -> std::io::Result<()> {
let to = input.join(name);
match std::fs::hard_link(file, &to) {
Ok(()) => Ok(()),
Err(e) => {
log::warn!(
"job input: {} could not be linked into the job ({e}); it is copied instead",
file.display()
);
std::fs::copy(file, &to).map(|_| ())
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A job's directories leave nothing behind under the work directory: not its own
/// `<id>/`, and not the shared base once the last job is gone (the scan sandbox's live
/// test counts the work directory's entries before and after a scan).
#[test]
fn a_jobs_directories_leave_nothing_behind() {
let work = tempfile::tempdir().unwrap();
let base = work.path().join(JOBS_DIR);
let (a, b) = (JobId::random(), JobId::random());
let da = JobDirs::new(&base, &a).unwrap();
let db = JobDirs::new(&base, &b).unwrap();
assert!(da.input.is_dir() && da.output.is_dir());
drop(da);
assert!(
base.join(b.to_hex()).is_dir(),
"another job's directories stay while it runs"
);
drop(db);
assert_eq!(
std::fs::read_dir(work.path()).unwrap().count(),
0,
"nothing is left in the work directory"
);
}
}