jobs/runner/src/heimdall.rs
//! **Heimdall as Holger's protected runner** (lane HOL-HEIMDALL-02, feature `heimdall`).
//!
//! [`HeimdallJobs`] runs Holger's worker (`holger-scan-worker`) as a job of Heimdall's
//! protected native runner, `heimdall_container::HeimdallRunner` (nordisk-heimdall-container
//! 0.2, crates.io): a youki container per job with Landlock, Heimdall's hand-reviewed seccomp
//! profile, rlimits, cgroup v2 limits and the worker image verified at every start. Where
//! the kernel cannot give Heimdall what it needs, the same adapter runs over
//! `heimdall_plain::PlainYoukiRunner` (`youki-plain`, an ordinary youki container, NOT
//! Heimdall), and its features say so.
//!
//! It follows Heimdall's "Scan-worker job contract (HeimdallRunner)" (heimdall
//! docs/API.md) exactly:
//!
//! | The contract | What Holger does |
//! |---|---|
//! | `input_dir` is `/in`, read-only, no single input file; `params` are ignored | a fresh staging input per job: `/in/job.json` (the [`WireJob`], the parameters inside), `/in/i/` (the job's input) and `/in/r<n>` (the policy's read paths), hard links of the caller's files (a copy only across file systems) |
//! | `/out` write-only, stdout is `/dev/null`, the answer is a file | the worker writes [`ANSWER_FILE`] in one pass beside its outputs ([`crate::wire::deliver`]) |
//! | the image: an OCI layout pinned by digest, `name` ignored, only `config.Env` used, argv exactly as given | [`HeimdallSettings::image`] (or one minted from the worker's verified rootfs tar, [`mint_layout`]); argv `[<worker>]`; the env `PATH` and `TMPDIR=/tmp` |
//! | limits from `policy.limits` (wasm limits ignored) | the job's own limits, unchanged |
//! | refusal codes (`heimdall-limit-*`, `heimdall-exit-*`, `heimdall-seccomp-1`, `heimdall-out-*`) | Holger's frozen codes ([`holger_code_of`]), the Heimdall code first in the reason |
//! | verification: the pinned key, the job id and nonce, `check_outputs`, replays | `Verifier::accept` over the pinned boot key (key id, boot id, job, nonce, replay), then `check_outputs` on the staging output, before one byte of it is used |
//!
//! Only then are the answer read and the named outputs moved into the caller's output
//! directory; the result the caller sees is re-signed with this adapter's own per-boot
//! key and carries Heimdall's per-job `runner` features and its kernel report unchanged.
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use heimdall_core::{
Features, ImageRef, Job, JobKind, JobResult, JobRunner, KernelFeatures, PinnedKey, Policy,
RunError, SeccompProfileRef, SignedResult, Verdict, Verifier, VerifyingKey,
};
use crate::collect::{collect, empty, refused};
use crate::container::{check_dirs, native_argv};
use crate::key::ResultKey;
use crate::wire::{read_mount, Answer, WireJob, ANSWER_FILE, ANSWER_MAX, INPUT_MOUNT, JOB_SCHEMA};
/// The worker's path inside its image (`media::worker_rootfs_tar` lays it there).
pub const WORKER_BIN: &str = holger_sandbox::HEIMDALL_WORKER_BIN;
/// The env of a minted worker image: Heimdall passes exactly the image's `Env`, and its
/// only writable scratch is the 64 MiB `/tmp` tmpfs.
pub const WORKER_ENV: &[&str] = &["PATH=/", "TMPDIR=/tmp"];
/// The `heimdall:` block of the sandbox config: holger-sandbox's, read here.
pub use holger_sandbox::HeimdallSettings;
/// **Holger's worker as a job of Heimdall's runner** (or of `youki-plain`). See the module.
pub struct HeimdallJobs {
inner: Box<dyn JobRunner>,
verifier: Mutex<Verifier>,
kernel: Option<KernelFeatures>,
image: ImageRef,
worker: String,
key: ResultKey,
/// Stage the inputs as copies, never hard links (a host id range is set).
copy_inputs: bool,
/// The host ids a job's root maps to, when set.
host_ids: Option<(u32, u32)>,
/// The worker's GENERATED seccomp profile ([`crate::seccomp`]), installed by Heimdall on
/// top of its fixed one for every job but the probe. `None`: the fixed profile only.
seccomp: Option<SeccompProfileRef>,
/// The fetching medium's adapter: its `fetch` jobs get their `policy.net`.
network: bool,
}
impl std::fmt::Debug for HeimdallJobs {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HeimdallJobs")
.field("runner", &self.inner.features().runner)
.field("image", &self.image.digest)
.field("key", &self.key)
.finish()
}
}
impl HeimdallJobs {
/// The adapter over `inner`, whose boot key is `pinned` (pinned now), running the
/// worker at `worker` in `image`. `kernel`: what `inner` found at its start.
pub fn over(
inner: Box<dyn JobRunner>,
pinned: PinnedKey,
kernel: Option<KernelFeatures>,
image: ImageRef,
worker: impl Into<String>,
key: ResultKey,
) -> Result<Self, RunError> {
if image.sha256().is_none() {
return Err(RunError::InvalidJob {
reason: format!(
"the worker image {:?} is not pinned by sha256",
image.digest
),
});
}
let verifier = Verifier::new(pinned).map_err(|e| RunError::Setup {
reason: format!("the runner's boot key does not pin: {e}"),
})?;
Ok(Self {
inner,
verifier: Mutex::new(verifier),
kernel,
image,
worker: worker.into(),
key,
copy_inputs: false,
host_ids: None,
seccomp: None,
network: false,
})
}
/// These jobs with a job's root mapped to `ids` (Heimdall 0.2.1 `with_host_ids`): the
/// inputs are then staged as copies (the job's root is given its input tree).
pub fn with_host_ids(mut self, ids: Option<(u32, u32)>) -> Self {
self.copy_inputs = ids.is_some();
self.host_ids = ids;
self
}
/// These jobs under the worker's generated seccomp profile (lane HEIMDALL-ALLIN prof):
/// every job but the probe names it in its policy (a job whose caller named one keeps
/// its own). The probe looks at the box's walls (it tries a connect on purpose), so it
/// keeps Heimdall's fixed profile.
pub fn with_seccomp(mut self, profile: Option<SeccompProfileRef>) -> Self {
self.seccomp = profile;
self
}
/// The generated seccomp profile the jobs run under, if any.
pub fn seccomp(&self) -> Option<&SeccompProfileRef> {
self.seccomp.as_ref()
}
/// **The fetching medium's adapter** (lane cds4, HOL-CDS-4): its `fetch` jobs get their
/// `policy.net` (its declared set of peers, the addresses Heimdall's broker may connect and
/// inject); every other
/// program of this adapter, and every job of an adapter without it, gets no network.
#[must_use]
pub fn with_network(mut self) -> Self {
self.network = true;
self
}
/// The host ids a job's root maps to, if a range is set.
pub fn host_ids(&self) -> Option<(u32, u32)> {
self.host_ids
}
/// The kernel features the inner runner found at its start (Heimdall's; `None` for
/// `youki-plain`, which probes nothing).
pub fn kernel(&self) -> Option<&KernelFeatures> {
self.kernel.as_ref()
}
/// The worker image, pinned.
pub fn image(&self) -> &ImageRef {
&self.image
}
}
/// The staging directories of one job, beside the caller's: removed when dropped.
struct Stage {
root: PathBuf,
input: PathBuf,
out: PathBuf,
}
impl Stage {
fn new(job: &Job) -> std::io::Result<Self> {
// Beside the caller's input, so the hard links stay on one file system.
let parent = job
.input_dir
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(std::env::temp_dir);
let root = parent.join(format!(".heimdall-{}", job.id.to_hex()));
let (input, out) = (root.join("in"), root.join("out"));
std::fs::create_dir_all(&input)?;
std::fs::create_dir_all(&out)?;
Ok(Self { root, input, out })
}
}
impl Drop for Stage {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
/// `from` (a file or a directory tree) at `to`: hard links, a copy only across file
/// systems; anything but regular files and directories is left out.
/// With `copy` (a host id range: Heimdall 0.2.1 gives the job's input tree to the job's root,
/// and a hard link would hand the caller's own files over too) every file is copied.
fn link_tree(from: &Path, to: &Path, copy: bool) -> std::io::Result<()> {
let m = std::fs::symlink_metadata(from)?;
if m.is_file() {
if copy || std::fs::hard_link(from, to).is_err() {
std::fs::copy(from, to)?;
}
} else if m.is_dir() {
std::fs::create_dir(to)?;
for e in std::fs::read_dir(from)? {
let e = e?;
link_tree(&e.path(), &to.join(e.file_name()), copy)?;
}
}
Ok(())
}
/// **Heimdall's refusal code as Holger's frozen one** (the Heimdall code stays first in
/// the reason): the wall clock is a timeout, the memory limit, a refused fork, a signal
/// or a seccomp kill is a killed worker, a non-zero exit a failed one, an output Heimdall
/// refused an undeclared one. `None`: a code this adapter does not know.
pub fn holger_code_of(heimdall_code: &str) -> Option<&'static holger_errcode::ErrCode> {
use holger_errcode as ec;
Some(match heimdall_code {
"heimdall-limit-wall" => &ec::SANDBOX_TIMEOUT,
"heimdall-limit-memory"
| "heimdall-limit-pids"
| "heimdall-exit-2"
| "heimdall-seccomp-1" => &ec::SANDBOX_KILLED,
"heimdall-exit-1" => &ec::SANDBOX_WORKER_FAILED,
"heimdall-out-1" | "heimdall-out-2" => &ec::SANDBOX_JOB_UNDECLARED,
_ => return None,
})
}
fn setup(reason: impl Into<String>) -> RunError {
RunError::Setup {
reason: reason.into(),
}
}
impl JobRunner for HeimdallJobs {
fn run(&self, job: Job) -> Result<SignedResult, RunError> {
let argv = native_argv(&job)?;
let fetch =
self.network && argv.first().map(String::as_str) == Some(crate::fetch_wire::PROGRAM);
if job.policy.net.is_some() && !fetch {
return Err(RunError::Unsupported {
what: "a network allowance: only the fetching medium's fetch jobs get one".into(),
});
}
if !job.policy.write.is_empty() {
return Err(RunError::Unsupported {
what: "a writable path besides the output directory".into(),
});
}
check_dirs(&job)?;
let io = |e: std::io::Error| RunError::Io {
reason: format!("staging the job for Heimdall: {e}"),
};
let stage = Stage::new(&job).map_err(io)?;
let wire = WireJob {
schema: JOB_SCHEMA.into(),
job_id: job.id.to_hex(),
argv,
params: job.params.clone(),
reads: job.policy.read.len(),
answer: Some(ANSWER_FILE.into()),
};
let doc = serde_json::to_vec(&wire).map_err(|e| RunError::Io {
reason: format!("the job document: {e}"),
})?;
std::fs::write(stage.input.join("job.json"), doc).map_err(io)?;
link_tree(
&job.input_dir,
&stage.input.join(INPUT_MOUNT),
self.copy_inputs,
)
.map_err(io)?;
for (n, p) in job.policy.read.iter().enumerate() {
if !p.is_absolute() {
return Err(RunError::InvalidJob {
reason: format!("the read path {} is not absolute", p.display()),
});
}
link_tree(p, &stage.input.join(read_mount(n)), self.copy_inputs).map_err(io)?;
}
let inner_job = Job::new(
JobKind::Native {
image: self.image.clone(),
argv: vec![self.worker.clone()],
},
stage.input.clone(),
stage.out.clone(),
serde_json::Value::Null,
Policy {
net: if fetch { job.policy.net.clone() } else { None },
limits: job.policy.limits,
// The generated profile is the scan worker's own: a fetch job runs the fetch
// worker (its own build), under Heimdall's fixed profile and the broker.
seccomp: job.policy.seccomp.clone().or_else(|| {
(wire.argv.first().map(String::as_str) != Some(crate::PROBE_PROGRAM) && !fetch)
.then(|| self.seccomp.clone())
.flatten()
}),
..Default::default()
},
);
let ticket = inner_job.ticket();
let signed = self.inner.run(inner_job)?;
// ★ Believe Heimdall's result before one byte of its output is used: the pinned
// boot key (key id, boot id, the signature over the wire v2), this job's id and
// nonce, no replay, and exactly the listed outputs.
let r = self
.verifier
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.accept(&signed.to_bytes(), &ticket)
.map_err(|e| setup(format!("the Heimdall result is not believed: {e}")))?;
heimdall_core::check_outputs(&r, &stage.out)
.map_err(|e| setup(format!("the Heimdall outputs are not the listed ones: {e}")))?;
let (verdict, outputs) = match &r.verdict {
Verdict::Refused { code, reason } => {
empty(&job.output_dir);
let v = match holger_code_of(code) {
Some(c) => refused(c, format!("{code}: {reason}")),
None => Verdict::Refused {
code: code.clone(),
reason: reason.clone(),
},
};
(v, Vec::new())
}
Verdict::Accepted => match answer_of(&r, &stage.out) {
Err(why) => {
empty(&job.output_dir);
(
refused(&holger_errcode::SANDBOX_JOB_ANSWER_INVALID, why),
Vec::new(),
)
}
Ok(a) => {
for o in r.outputs.iter().filter(|o| o.name != ANSWER_FILE) {
std::fs::rename(stage.out.join(&o.name), job.output_dir.join(&o.name))
.map_err(io)?;
}
collect(&a, &job.output_dir)
}
},
};
let result = JobResult {
job_id: job.id,
nonce: job.nonce,
verdict,
outputs,
exit: r.exit.clone(),
started_unix: r.started_unix,
finished_unix: r.finished_unix,
runner: r.runner.clone(),
events: r.events.clone(),
kernel: r.kernel.clone(),
};
drop(stage);
self.key.sign(result)
}
fn verifying_key(&self) -> VerifyingKey {
self.key.verifying_key()
}
fn features(&self) -> Features {
self.inner.features()
}
}
/// The worker's answer: the listed [`ANSWER_FILE`], under its cap and schema.
fn answer_of(r: &JobResult, out: &Path) -> Result<Answer, String> {
let o = r
.outputs
.iter()
.find(|o| o.name == ANSWER_FILE)
.ok_or_else(|| format!("the job left no {ANSWER_FILE} in /out"))?;
if o.bytes > ANSWER_MAX as u64 {
return Err(format!(
"the answer is {} bytes, more than {ANSWER_MAX}",
o.bytes
));
}
let bytes = std::fs::read(out.join(ANSWER_FILE)).map_err(|e| format!("{ANSWER_FILE}: {e}"))?;
Answer::parse(&bytes)
}
// ── the image ───────────────────────────────────────────────────────────────
const MT_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json";
const MT_CONFIG: &str = "application/vnd.oci.image.config.v1+json";
const MT_LAYER: &str = "application/vnd.oci.image.layer.v1.tar";
fn sha256_hex(b: &[u8]) -> String {
use sha2::Digest as _;
hex::encode(sha2::Sha256::digest(b))
}
/// **Mint the worker image** into the OCI layout at `layout`: ONE uncompressed layer
/// (`rootfs_tar`, the medium's verified worker rootfs), a config whose `Env` is `env`, and
/// a manifest. Deterministic: the same tar gives the same digest. Returns the image,
/// pinned by its manifest's digest.
pub fn mint_layout(layout: &Path, rootfs_tar: &[u8], env: &[&str]) -> std::io::Result<ImageRef> {
let blobs = layout.join("blobs").join("sha256");
std::fs::create_dir_all(&blobs)?;
let put = |bytes: &[u8]| -> std::io::Result<String> {
let h = sha256_hex(bytes);
let p = blobs.join(&h);
if std::fs::read(&p).ok().as_deref() != Some(bytes) {
let tmp = blobs.join(format!(".{h}.tmp"));
std::fs::write(&tmp, bytes)?;
std::fs::rename(&tmp, &p)?;
}
Ok(format!("sha256:{h}"))
};
let layer = put(rootfs_tar)?;
let config = serde_json::to_vec(&serde_json::json!({
"architecture": "amd64",
"os": "linux",
"config": { "Env": env },
"rootfs": { "type": "layers", "diff_ids": [layer] },
}))
.map_err(std::io::Error::other)?;
let config_digest = put(&config)?;
let manifest = serde_json::to_vec(&serde_json::json!({
"schemaVersion": 2,
"mediaType": MT_MANIFEST,
"config": { "mediaType": MT_CONFIG, "digest": config_digest, "size": config.len() },
"layers": [ { "mediaType": MT_LAYER, "digest": layer, "size": rootfs_tar.len() } ],
}))
.map_err(std::io::Error::other)?;
let digest = put(&manifest)?;
std::fs::write(
layout.join("oci-layout"),
br#"{"imageLayoutVersion":"1.0.0"}"#,
)?;
let index = serde_json::json!({
"schemaVersion": 2,
"manifests": [ { "mediaType": MT_MANIFEST, "digest": digest, "size": manifest.len() } ],
});
std::fs::write(layout.join("index.json"), index.to_string())?;
Ok(ImageRef {
name: crate::WORKER.into(),
digest,
})
}
// ── building the runner ─────────────────────────────────────────────────────
/// Which runner [`start`] got, and why not a better one.
#[derive(Debug)]
pub struct Started {
pub jobs: HeimdallJobs,
/// Why Heimdall itself is not the runner (`None`: it is).
pub fallback: Option<String>,
}
/// The worker image the settings name, minted when they name a rootfs tar instead.
pub fn image_of(s: &HeimdallSettings) -> Result<ImageRef, String> {
match (&s.image, &s.rootfs_tar) {
(Some(d), _) => Ok(ImageRef {
name: crate::WORKER.into(),
digest: d.clone(),
}),
(None, Some(t)) => {
let tar = std::fs::read(t).map_err(|e| format!("heimdall: rootfs_tar {t}: {e}"))?;
mint_layout(Path::new(&s.image_layout), &tar, WORKER_ENV).map_err(|e| {
format!(
"heimdall: minting the worker image into {}: {e}",
s.image_layout
)
})
}
(None, None) => Err("heimdall: no worker image".into()),
}
}
fn shim_of(
p: &Option<String>,
located: heimdall_plain::ShimCommand,
) -> heimdall_plain::ShimCommand {
match p {
Some(p) => heimdall_plain::ShimCommand {
program: p.into(),
args: vec![],
},
None => located,
}
}
/// **Start Heimdall's runner** for `s` (work under `work_dir`), or, when the kernel or
/// the host refuses it and `fallback_plain` allows, `youki-plain` over the same image
/// (its shim must be there). `Err`: neither; the caller's next fallback is crun.
pub fn start(
s: &HeimdallSettings,
work_dir: &Path,
rootless: bool,
userns_base: u32,
) -> Result<Started, String> {
use std::os::unix::fs::MetadataExt as _;
let uid = std::fs::metadata("/proc/self")
.map(|m| m.uid())
.map_err(|e| format!("heimdall: /proc/self: {e}"))?;
start_as(
s,
work_dir,
host_ids_for(rootless, uid, userns_base, s.host_id_count),
)
}
/// The host ids a job's root maps to (Heimdall 0.2.1 `with_host_ids`, lane HOL-HEIMDALL-03):
/// a server that runs as root (the appliance's PID 1) under `require_rootless` maps every
/// job's root to `base..base + count` (crun's own user namespace base, 100000 by default),
/// never to host root; any other server maps it to its own uid, as before.
pub const HOST_IDS_COUNT: u32 = 65536;
/// `Some((base, count))` when a job's root must not be this process's root.
pub fn host_ids_for(rootless: bool, uid: u32, base: u32, count: u32) -> Option<(u32, u32)> {
(rootless && uid == 0).then_some((base, count))
}
/// [`start`] with the host ids decided ([`host_ids_for`]; tests).
pub fn start_as(
s: &HeimdallSettings,
work_dir: &Path,
host_ids: Option<(u32, u32)>,
) -> Result<Started, String> {
use heimdall_container::{HeimdallConfig, HeimdallRunner};
s.validate()?;
let image = image_of(s)?;
let mut cfg = HeimdallConfig::new(&s.image_layout, work_dir.join("heimdall"));
cfg.shim = shim_of(&s.shim, cfg.shim.clone());
cfg.strict = s.strict;
cfg.cgroup_root = s.cgroup_root.clone().into();
cfg.cgroup_parent = s.cgroup_parent.clone();
let refused = if !cfg.shim.program.is_file() {
format!(
"heimdall: its shim {} is not there",
cfg.shim.program.display()
)
} else {
match HeimdallRunner::new(cfg).and_then(|r| match host_ids {
Some((base, count)) => r.with_host_ids(base, count),
None => Ok(r),
}) {
Ok(r) => {
let (pinned, kernel) = (r.pinned(), r.kernel().features());
let jobs = HeimdallJobs::over(
Box::new(r),
pinned,
Some(kernel),
image,
s.worker.clone(),
ResultKey::per_boot(),
)
.map_err(|e| e.to_string())?
.with_host_ids(host_ids)
.with_seccomp(if s.generated_seccomp {
crate::seccomp::scan_worker_profile(&work_dir.join("heimdall-seccomp"))?
} else {
None
});
return Ok(Started {
jobs,
fallback: None,
});
}
Err(e) => e.to_string(),
}
};
if !s.fallback_plain {
return Err(refused);
}
let mut pc = heimdall_plain::PlainConfig::new(&s.image_layout, work_dir.join("heimdall-plain"));
pc.shim = shim_of(&s.plain_shim, heimdall_plain::ShimCommand::locate());
pc.cgroup_root = s.cgroup_root.clone().into();
pc.cgroup_parent = s.cgroup_parent.clone();
if !pc.shim.program.is_file() {
return Err(format!(
"{refused}; and the youki-plain shim {} is not there",
pc.shim.program.display()
));
}
let r = heimdall_plain::PlainYoukiRunner::new(pc);
let r = match host_ids {
Some((base, count)) => r
.with_host_ids(base, count)
.map_err(|e| format!("{refused}; and youki-plain: {e}"))?,
None => r,
};
let pinned = r.pinned();
let jobs = HeimdallJobs::over(
Box::new(r),
pinned,
None,
image,
s.worker.clone(),
ResultKey::per_boot(),
)
.map_err(|e| e.to_string())?
.with_host_ids(host_ids);
Ok(Started {
jobs,
fallback: Some(format!(
"{refused}: the jobs run in youki-plain, not Heimdall"
)),
})
}