crates/heimdall-container/src/lib.rs
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Ignalina ApS and Vetra Automation AB
//! heimdall-container: [`HeimdallRunner`], the protected native runner
//! (spec §5, milestone 2).
//!
//! **Heimdall is a defensive sandbox.** It runs untrusted parsing,
//! verification and scanning jobs so that a compromised job can reach only
//! its own input and output. This crate is layer 1 of the spec: one fresh
//! youki (`libcontainer`) container per job, with
//!
//! - user, mount, pid, ipc, uts and cgroup namespaces and an EMPTY network
//! namespace;
//! - a read-only root from the image pinned by digest, verified (sha256 of
//! the manifest, config and every layer) and unpacked fresh at EVERY start
//! (heimdall-plain's [`heimdall_plain::image`]); a tmpfs `/tmp`;
//! `input_dir` at `/in` read-only and `output_dir` at `/out`, which the job
//! can write but not read back;
//! - Landlock ([`confine`]), applied inside the job right before `execve`:
//! read and execute the image, read `/in` and `/proc`, write `/out`, read
//! and write `/tmp` (no execute), no TCP bind/connect (ABI 4), no signals
//! or abstract unix sockets outside the job (ABI 6);
//! - no-new-privs, every capability dropped, rlimits (nofile, nproc, fsize,
//! core = 0), cgroup v2 memory (swap 0), cpu, pids and io limits;
//! - a fixed, hand-reviewed seccomp profile ([`profile`]), stricter than
//! moby's default, reported as [`Protection::Default`]; and, when the job's
//! policy or the runner's config names one, the GENERATED profile of the
//! job's workload on top of it ([`genprof`]), pinned by digest and reported
//! as [`Protection::Generated`] only when the job's executor confirmed it
//! was installed;
//! - the syscall broker ([`brokered`], heimdall-broker, spec §4): the job's
//! `openat`/`openat2`/`open`/`creat`, `socket`, `connect`, `bind` and
//! `execve` pause on seccomp USER_NOTIF; the shim checks them against the
//! job's broker policy, opens the file itself and injects the fd with
//! `SECCOMP_IOCTL_NOTIF_ADDFD`, and every decision is a signed
//! [`heimdall_core::BrokerEvent`]. `Features.broker` is true only in full
//! mode (spec §10: check-only without ADDFD, deny-only without USER_NOTIF).
//!
//! The per-boot signing key lives only in the runner process: each job runs
//! through `heimdall-container-shim`, a separate process that never receives
//! the key.
//!
//! On every start the runner probes the kernel ([`kernel::KernelReport`],
//! spec §10) and refuses to start in strict mode when a feature is missing.
//! [`JobRunner::features`] states what jobs really get, and every signed
//! result states what that job really got (Landlock as reported from inside
//! the job, right before `execve`).
#![deny(unsafe_code)]
pub mod brokered;
pub mod confine;
pub mod exec;
pub mod genprof;
pub mod iolimit;
pub mod kernel;
pub mod learn;
pub mod profile;
pub mod shim;
pub mod spec;
mod sys;
pub mod workspace;
use std::io::{Read, Write};
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use heimdall_core::netevent::{NetEvent, NetObserver};
use heimdall_core::{
codes, BootKey, ExitInfo, Features, Job, JobKind, JobResult, JobRunner, KillReason, PinnedKey,
Protection, RunError, SeccompProfileRef, SignedResult, Verdict, VerifyingKey,
};
use heimdall_plain::idmap;
pub use heimdall_plain::idmap::HostIds;
use heimdall_plain::image::OciLayout;
use heimdall_plain::outputs;
pub use heimdall_plain::ShimCommand;
use crate::brokered::BrokerRequest;
pub use crate::brokered::BrokerSetting;
use crate::genprof::GeneratedPlan;
use crate::kernel::KernelReport;
use crate::shim::{ShimRequest, ShimResponse};
/// The runner name in [`Features::runner`].
pub const RUNNER_NAME: &str = "heimdall";
/// The pinned youki `libcontainer` version the shim is built with.
pub const LIBCONTAINER_VERSION: &str = "0.7.0";
/// Extra time the shim gets beyond the wall limit before the runner kills it.
pub const SHIM_GRACE: Duration = Duration::from_secs(30);
/// The cgroup v2 `io.weight` jobs get by default (the kernel default is 100).
pub const DEFAULT_IO_WEIGHT: u16 = 50;
/// Whether jobs get Landlock.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum LandlockMode {
/// Landlock whenever the kernel has it (strict mode: required).
Auto,
/// No Landlock (tests and diagnosis; strict mode refuses this).
Off,
}
/// A mount the job gets that Landlock does NOT list. Only for containment
/// tests (feature `containment-tests`): it proves that Landlock, not the
/// mount, refuses the access.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct UnlistedMount {
pub source: PathBuf,
pub destination: String,
pub writable: bool,
}
/// Configuration of a [`HeimdallRunner`].
#[derive(Clone, Debug, PartialEq)]
pub struct HeimdallConfig {
/// A local OCI image layout holding the images jobs pin by digest.
pub image_layout: PathBuf,
/// Per-job bundles (`jobs/<id>/`) and libcontainer state (`state/`).
pub work_dir: PathBuf,
pub shim: ShimCommand,
/// The cgroup v2 mount the shim sees.
pub cgroup_root: PathBuf,
/// Jobs' cgroups go below `<cgroup_root>/<cgroup_parent>/`.
pub cgroup_parent: String,
/// Refuse to start when a kernel feature of spec §10 is missing.
pub strict: bool,
pub landlock: LandlockMode,
/// cgroup v2 `io.weight` of every job, 1..=10000 (written by the shim and
/// checked when the io controller is there).
pub io_weight: u16,
/// `io.max` limits of every job ([`iolimit`]), resolved to whole disks
/// at start. Not empty and no io controller: the runner refuses to start.
pub io_max: Vec<iolimit::IoMax>,
/// Where sysfs is mounted (the io limits' devices are resolved there).
pub sysfs_root: PathBuf,
/// Jobs may ask for an EXECUTABLE workspace (`Workspace::exec`): build and
/// trace runners only. Off by default: a scanner never gets exec.
pub allow_workspace_exec: bool,
/// The generated seccomp profile every job gets unless its policy names
/// its own ([`genprof`]); `None`: the fixed profile. Resolved when the
/// runner starts (a missing or tampered one refuses the start, unless it
/// allows the fallback).
pub seccomp: Option<SeccompProfileRef>,
/// The syscall broker ([`brokered`]); default [`BrokerSetting::Auto`].
pub broker: BrokerSetting,
#[cfg(feature = "containment-tests")]
pub unlisted_mounts: Vec<UnlistedMount>,
}
impl HeimdallConfig {
/// Defaults: located shim, `/sys/fs/cgroup`, parent `heimdall`, strict,
/// Landlock on.
pub fn new(image_layout: impl Into<PathBuf>, work_dir: impl Into<PathBuf>) -> Self {
Self {
image_layout: image_layout.into(),
work_dir: work_dir.into(),
shim: locate_shim(),
cgroup_root: "/sys/fs/cgroup".into(),
cgroup_parent: "heimdall".into(),
strict: true,
landlock: LandlockMode::Auto,
io_weight: DEFAULT_IO_WEIGHT,
io_max: vec![],
sysfs_root: "/sys".into(),
allow_workspace_exec: false,
seccomp: None,
broker: BrokerSetting::Auto,
#[cfg(feature = "containment-tests")]
unlisted_mounts: vec![],
}
}
fn unlisted(&self) -> &[UnlistedMount] {
#[cfg(feature = "containment-tests")]
return &self.unlisted_mounts;
#[cfg(not(feature = "containment-tests"))]
return &[];
}
}
/// `HEIMDALL_CONTAINER_SHIM` if set, else `heimdall-container-shim` next to
/// the current executable (or its parent dir, for test binaries in `deps/`).
pub fn locate_shim() -> ShimCommand {
const NAME: &str = "heimdall-container-shim";
if let Some(p) = std::env::var_os("HEIMDALL_CONTAINER_SHIM") {
return ShimCommand {
program: p.into(),
args: vec![],
};
}
let exe = std::env::current_exe().unwrap_or_default();
let dir = exe.parent().map(Path::to_path_buf).unwrap_or_default();
let near = dir.join(NAME);
let program = if near.exists() {
near
} else {
dir.parent()
.map(|d| d.join(NAME))
.filter(|p| p.exists())
.unwrap_or(near)
};
ShimCommand {
program,
args: vec![],
}
}
/// The argument that makes a host binary act as the shim.
pub const SHIM_ARG: &str = "--heimdall-container-shim";
/// What a runner with this config on this kernel gives every job, or why it
/// refuses to start (spec §10).
pub fn admit(cfg: &HeimdallConfig, kernel: &KernelReport) -> Result<Features, RunError> {
let refuse = |what: String| RunError::Unsupported { what };
if !kernel.seccomp_filter {
return Err(refuse(format!(
"heimdall refuses to start: no seccomp filters with SECCOMP_RET_KILL_PROCESS on kernel {}",
kernel.release
)));
}
let mut missing: Vec<String> = Vec::new();
if cfg.strict {
missing.extend(kernel.missing_for_strict().into_iter().map(|m| m.0));
if cfg.landlock == LandlockMode::Off {
missing.push("Landlock is switched off in the config".into());
}
}
// A configured io limit that the kernel cannot apply would be a lie in
// every result: refused, strict or not.
if !(1..=10000).contains(&cfg.io_weight) {
return Err(refuse(format!(
"heimdall refuses to start: io_weight {} is outside 1..=10000",
cfg.io_weight
)));
}
if !cfg.io_max.is_empty() && !kernel.has_controller("io") {
return Err(refuse(format!(
"heimdall refuses to start on kernel {}: the config sets io limits (io.max) and the cgroup v2 io controller is not delegated to the jobs' parent cgroup",
kernel.release
)));
}
if !missing.is_empty() {
return Err(refuse(format!(
"heimdall refuses to start in strict mode on kernel {}: missing {}",
kernel.release,
missing.join("; ")
)));
}
Ok(Features {
runner: RUNNER_NAME.into(),
container: true,
seccomp: Protection::Default,
landlock: cfg.landlock == LandlockMode::Auto && kernel.landlock_abi.is_some(),
broker: brokered::mode_for(cfg.broker, kernel) == Some(heimdall_broker::Mode::Full),
wasm: false,
image_verified_each_start: true,
})
}
/// The protected runner. See the crate docs.
pub struct HeimdallRunner {
cfg: HeimdallConfig,
key: BootKey,
kernel: KernelReport,
features: Features,
/// The config's generated profile, resolved at start.
default_plan: Option<GeneratedPlan>,
host_ids: Option<HostIds>,
/// The jobs' `RLIMIT_NOFILE` ([`HeimdallRunner::with_nofile`]).
nofile: u64,
/// The config's io limits, resolved to whole disks at start.
io_max: Vec<iolimit::ResolvedIoMax>,
/// The network observer of every job ([`HeimdallRunner::with_observer`]).
observer: Option<Arc<dyn NetObserver>>,
}
impl std::fmt::Debug for HeimdallRunner {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HeimdallRunner")
.field("cfg", &self.cfg)
.field("key_id", &self.key.key_id())
.field("kernel", &self.kernel)
.field("features", &self.features)
.field("host_ids", &self.host_ids)
.finish()
}
}
/// Check what can be checked before anything is created.
pub fn check_job(job: &Job) -> Result<(), RunError> {
let JobKind::Native { image, argv } = &job.kind else {
return Err(RunError::Unsupported {
what: "wasm jobs: heimdall-container runs native OCI images (wasm: heimdall-run)"
.into(),
});
};
if let Some(n) = &job.policy.net {
n.check().map_err(invalid)?;
}
job.policy.check_job_ref().map_err(invalid)?;
if let Some(ws) = &job.policy.workspace {
if ws.bytes == 0 {
return Err(invalid("policy.workspace.bytes is zero"));
}
}
if image.sha256().is_none() {
return Err(invalid(format!(
"image digest `{}` is not sha256:<64 lower-case hex>",
image.digest
)));
}
if argv.is_empty() || argv[0].is_empty() {
return Err(invalid("argv is empty"));
}
let l = &job.policy.limits;
if l.wall.is_zero() {
return Err(invalid("limits.wall is zero"));
}
if !(l.cpus.is_finite() && l.cpus > 0.0 && l.cpus <= 1024.0) {
return Err(invalid(format!(
"limits.cpus {} is not in (0, 1024]",
l.cpus
)));
}
if l.pids == 0 {
return Err(invalid("limits.pids is zero"));
}
if l.memory_bytes < 4 * 1024 * 1024 {
return Err(invalid("limits.memory_bytes is below 4 MiB"));
}
for p in job.policy.read.iter().chain(&job.policy.write) {
if !p.is_absolute() || p.components().any(|c| c == std::path::Component::ParentDir) {
return Err(invalid(format!(
"policy path {} must be absolute without `..`",
p.display()
)));
}
// A policy path may not cover or sit inside the places Landlock and
// the mounts treat on their own.
let first = p.components().nth(1);
let reserved = match first {
None => true,
Some(std::path::Component::Normal(n)) => {
["in", "out", "proc", "dev", "sys", "work"].contains(&n.to_string_lossy().as_ref())
}
Some(_) => true,
};
if reserved {
return Err(invalid(format!(
"policy path {} is `/` or under /in, /out, /proc, /dev or /sys",
p.display()
)));
}
}
Ok(())
}
fn invalid(reason: impl Into<String>) -> RunError {
RunError::InvalidJob {
reason: reason.into(),
}
}
fn io(e: impl std::fmt::Display) -> RunError {
RunError::Io {
reason: e.to_string(),
}
}
fn setup(reason: impl Into<String>) -> RunError {
RunError::Setup {
reason: reason.into(),
}
}
/// Remove everything in `dir` (never following a symlink), keeping `dir`.
fn empty_dir(dir: &Path) -> std::io::Result<()> {
for e in std::fs::read_dir(dir)? {
let p = e?.path();
if std::fs::symlink_metadata(&p)?.is_dir() {
std::fs::remove_dir_all(&p)?;
} else {
std::fs::remove_file(&p)?;
}
}
Ok(())
}
fn now_unix() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn existing_dir(what: &str, p: &Path) -> Result<PathBuf, RunError> {
let c = p
.canonicalize()
.map_err(|e| invalid(format!("{what} {}: {e}", p.display())))?;
if !c.is_dir() {
return Err(invalid(format!(
"{what} {} is not a directory",
p.display()
)));
}
Ok(c)
}
impl HeimdallRunner {
/// Probe the kernel and start with a fresh per-boot key; refuses to start
/// as [`admit`] says.
pub fn new(cfg: HeimdallConfig) -> Result<Self, RunError> {
let k = KernelReport::probe(&cfg.cgroup_root, &cfg.cgroup_parent);
Self::with_kernel(cfg, BootKey::generate(), k)
}
/// A runner with a given key and kernel report (tests and fixtures).
pub fn with_kernel(
cfg: HeimdallConfig,
key: BootKey,
kernel: KernelReport,
) -> Result<Self, RunError> {
let mut features = admit(&cfg, &kernel)?;
let default_plan = match &cfg.seccomp {
None => None,
Some(r) => genprof::resolve(r).map_err(|e| RunError::Unsupported {
what: format!("heimdall refuses to start: {e}"),
})?,
};
let io_max =
iolimit::resolve(&cfg.io_max, &cfg.sysfs_root).map_err(|e| RunError::Unsupported {
what: format!("heimdall refuses to start: {e}"),
})?;
if let Some(p) = &default_plan {
features.seccomp = Protection::Generated {
profile_digest: p.digest.clone(),
};
}
Ok(Self {
cfg,
key,
kernel,
features,
default_plan,
host_ids: None,
nofile: spec::NOFILE,
io_max,
observer: None,
})
}
/// Map container root to the host id range `base..base + count` instead
/// of the runner's own uid/gid (see [`heimdall_plain::idmap`]): for a
/// runner that runs as host root, so that no job's root is host root.
/// The unpacked root is shifted into the range and `input_dir` /
/// `output_dir` are given to the job's root before it starts. Refused
/// (naming `host_ids`) when the range is invalid, the runner is not root,
/// or the range holds the runner's own uid or gid.
pub fn with_host_ids(mut self, base: u32, count: u32) -> Result<Self, RunError> {
self.host_ids = Some(HostIds::for_this_runner(base, count)?);
Ok(self)
}
/// The host id range container root maps to, if one was set.
pub fn host_ids(&self) -> Option<HostIds> {
self.host_ids
}
/// Give every job `RLIMIT_NOFILE` = `n` (soft and hard) instead of
/// [`spec::NOFILE`] (1024): for jobs that open many files at once (a
/// decompressor fanning out, a build). Refused (naming `nofile`) below
/// [`spec::NOFILE_MIN`], above [`spec::NOFILE_MAX`], or above this
/// runner's own hard `RLIMIT_NOFILE`, which no job can exceed.
pub fn with_nofile(mut self, n: u64) -> Result<Self, RunError> {
self.nofile = heimdall_core::check_nofile("nofile", n, own_nofile_hard())
.map_err(|what| RunError::Unsupported { what })?;
Ok(self)
}
/// The `RLIMIT_NOFILE` one job gets: its [`Limits::nofile`]
/// (heimdall_core::Limits) when set, else this runner's
/// [`HeimdallRunner::nofile`]. Refused (InvalidJob, naming
/// `limits.nofile`) outside [`spec::NOFILE_MIN`]`..=`[`spec::NOFILE_MAX`]
/// or above this runner's own hard limit.
pub fn nofile_for(&self, limits: &heimdall_core::Limits) -> Result<u64, RunError> {
match limits.nofile {
None => Ok(self.nofile),
Some(n) => {
heimdall_core::check_nofile("limits.nofile", n, own_nofile_hard()).map_err(invalid)
}
}
}
/// Hand every network decision of every job (each `socket` and
/// `connect` the broker decides) to `observer`, as [`NetEvent`]s (0.4).
/// It runs on a thread of its own per job, behind bounded queues: it
/// never changes a decision and never stalls a job; when it falls
/// behind, events are dropped and one `dropped` event says how many. A
/// panic in it is caught. `run` waits for it at most
/// [`OBSERVER_GRACE`] after the job ended.
pub fn with_observer(mut self, observer: Arc<dyn NetObserver>) -> Self {
self.observer = Some(observer);
self
}
/// The jobs' `RLIMIT_NOFILE`.
pub fn nofile(&self) -> u64 {
self.nofile
}
/// The kernel features this runner found at start (spec §10).
/// The config's io limits, resolved to whole disks at start.
pub fn io_max(&self) -> &[iolimit::ResolvedIoMax] {
&self.io_max
}
pub fn kernel(&self) -> &KernelReport {
&self.kernel
}
pub fn pinned(&self) -> PinnedKey {
self.key.pinned()
}
pub fn config(&self) -> &HeimdallConfig {
&self.cfg
}
/// Start the shim with `req` on its stdin; its stdout is read on a
/// thread (so a chatty shim never blocks on a full pipe).
fn spawn_shim(
&self,
req: &ShimRequest,
events: Option<heimdall_broker::EventSink>,
) -> Result<(std::process::Child, std::thread::JoinHandle<Vec<u8>>), RunError> {
// Exclusive against unpacks: the shim never inherits a write fd of
// another job's program (ETXTBSY, see heimdall_plain::forklock).
let spawn_guard = heimdall_plain::forklock::spawning();
let mut child = Command::new(&self.cfg.shim.program)
.args(&self.cfg.shim.args)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(if std::env::var_os("HEIMDALL_CONTAINER_DEBUG").is_some() {
Stdio::inherit()
} else {
Stdio::null()
})
.spawn()
.map_err(|e| {
setup(format!(
"cannot start the shim {}: {e}",
self.cfg.shim.program.display()
))
})?;
drop(spawn_guard);
{
let mut stdin = child.stdin.take().expect("piped");
let body = serde_json::to_vec(req).expect("request serializes");
stdin
.write_all(&body)
.map_err(|e| setup(format!("shim stdin: {e}")))?;
}
let stdout = child.stdout.take().expect("piped");
let reader = std::thread::spawn(move || read_shim_stdout(stdout, events.as_ref()));
Ok((child, reader))
}
fn run_shim(
&self,
req: &ShimRequest,
wall: Duration,
events: Option<heimdall_broker::EventSink>,
) -> Result<ShimResponse, RunError> {
let (mut child, reader) = self.spawn_shim(req, events)?;
let deadline = Instant::now() + wall + SHIM_GRACE;
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) if Instant::now() < deadline => {
std::thread::sleep(Duration::from_millis(10))
}
Ok(None) => {
let _ = child.kill();
let _ = child.wait();
// The job's processes outlive a killed shim: end them by cgroup.
let _ = std::fs::write(req.cgroup_dir.join("cgroup.kill"), "1");
return Err(setup("the shim did not finish in time"));
}
Err(e) => return Err(setup(format!("waiting for the shim: {e}"))),
}
}
let out = reader.join().unwrap_or_default();
let out = String::from_utf8_lossy(&out);
serde_json::from_str(out.trim())
.map_err(|e| setup(format!("bad shim response ({e}): {}", out.trim())))
}
/// What this job really got, from the executor's report inside the job;
/// anything less than [`JobRunner::features`] promises is a setup error
/// and nothing is signed.
fn job_features(
&self,
resp: &ShimResponse,
plan: Option<&GeneratedPlan>,
) -> Result<Features, RunError> {
let Some(inside) = &resp.inside else {
return Err(setup(format!(
"no report from inside the job (the executor never ran){}",
resp.error
.as_deref()
.map(|e| format!(": {e}"))
.unwrap_or_default()
)));
};
if let Some(e) = &inside.error {
return Err(setup(format!(
"inside the job, before execve: {e}{}{}",
resp.error
.as_deref()
.map(|e| format!(" (shim: {e})"))
.unwrap_or_default(),
resp.broker
.as_ref()
.and_then(|b| b.events.last())
.map(|ev| format!(
" (last broker decision: {} {:?} {})",
ev.call,
ev.decision,
ev.args.join(" ")
))
.unwrap_or_default()
)));
}
if !inside.no_new_privs {
return Err(setup("no-new-privs was not set inside the job"));
}
if inside.seccomp_mode != 2 {
return Err(setup(format!(
"the seccomp filter was not on inside the job (mode {})",
inside.seccomp_mode
)));
}
if self.features.landlock && inside.landlock.is_none() {
return Err(setup("Landlock was not applied inside the job"));
}
let want = self.broker_mode();
if inside.broker != want {
return Err(setup(format!(
"the broker filter inside the job is {:?}, not {want:?}",
inside.broker
)));
}
if let Some(m) = want {
if m != heimdall_broker::Mode::DenyOnly {
match &resp.broker {
Some(r) if r.mode == m => {}
Some(r) => {
return Err(setup(format!("the broker ran in {:?}, not {m:?}", r.mode)))
}
None => {
return Err(setup(format!(
"the broker never served the job{}",
resp.error
.as_deref()
.map(|e| format!(": {e}"))
.unwrap_or_default()
)))
}
}
}
}
if let Some(s) = &resp.swap_max {
if s != "0" {
return Err(setup(format!("memory.swap.max is {s}, not 0")));
}
}
if !self.io_max.is_empty() {
let Some(got) = &resp.io_max else {
return Err(setup("io.max was not read back from the job's cgroup"));
};
iolimit::check(got, &self.io_max).map_err(setup)?;
}
let seccomp = match plan {
None => Protection::Default,
Some(p) if inside.generated.as_deref() == Some(p.digest.as_str()) => {
Protection::Generated {
profile_digest: p.digest.clone(),
}
}
Some(p) => {
return Err(setup(format!(
"the generated seccomp profile {} was not confirmed from inside the job",
p.digest
)))
}
};
Ok(Features {
landlock: inside.landlock.is_some(),
seccomp,
broker: want == Some(heimdall_broker::Mode::Full),
..self.features.clone()
})
}
/// One fresh bundle dir (root, config) per job, removed after it.
fn new_bundle(&self, job: &Job) -> Result<PathBuf, RunError> {
let bundle = self.cfg.work_dir.join("jobs").join(job.id.to_hex());
std::fs::create_dir_all(bundle.parent().expect("has parent")).map_err(io)?;
if let Err(e) = std::fs::create_dir(&bundle) {
return Err(if e.kind() == std::io::ErrorKind::AlreadyExists {
invalid(format!("job id {} is already running or ran", job.id))
} else {
io(e)
});
}
Ok(bundle)
}
/// The broker mode jobs get (`None`: no broker).
pub fn broker_mode(&self) -> Option<heimdall_broker::Mode> {
brokered::mode_for(self.cfg.broker, &self.kernel)
}
/// The generated profile this job gets: its policy's, else the config's;
/// `None` = the fixed profile.
fn plan_for(&self, job: &Job) -> Result<Option<GeneratedPlan>, RunError> {
match &job.policy.seccomp {
Some(r) => genprof::resolve(r),
None => Ok(self.default_plan.clone()),
}
}
fn run_in(&self, job: &Job, bundle: &Path) -> Result<SignedResult, RunError> {
if job.policy.workspace.is_some_and(|w| w.exec) && !self.cfg.allow_workspace_exec {
return Err(RunError::Unsupported {
what: "an executable workspace: this runner does not allow exec (HeimdallConfig::allow_workspace_exec, for build and trace runners only)".into(),
});
}
let JobKind::Native { image, argv } = &job.kind else {
unreachable!("checked")
};
let plan = self.plan_for(job)?;
let nofile = self.nofile_for(&job.policy.limits)?;
let input_dir = existing_dir("input_dir", &job.input_dir)?;
let output_dir = existing_dir("output_dir", &job.output_dir)?;
if std::fs::read_dir(&output_dir).map_err(io)?.next().is_some() {
return Err(invalid(format!(
"output_dir {} is not empty",
job.output_dir.display()
)));
}
for p in job.policy.read.iter().chain(&job.policy.write) {
if !p.exists() {
return Err(invalid(format!(
"policy path {} does not exist",
p.display()
)));
}
}
// Verified (manifest, config, every layer by sha256) and unpacked
// fresh for this start.
let rootfs = bundle.join("rootfs");
std::fs::create_dir_all(&rootfs).map_err(io)?;
let unpacked = OciLayout::new(&self.cfg.image_layout)
.unpack(image, &rootfs)
.map_err(|e| RunError::Image {
reason: e.to_string(),
})?;
let landlock = if self.features.landlock {
Some(
confine::plan_with_workspace(
&rootfs,
&job.policy.read,
&job.policy.write,
job.policy.workspace.as_ref(),
)
.map_err(io)?,
)
} else {
None
};
let me = std::fs::metadata("/proc/self").map_err(io)?;
let cgroup_path = format!("/{}/job-{}", self.cfg.cgroup_parent, job.id);
let limits = job.policy.limits;
// A fetch job learns its declared peers, host names included (for
// SNI and the certificate check): HEIMDALL_NET_PEERS.
let mut job_env = unpacked.env.clone();
if let Some(n) = &job.policy.net {
job_env.retain(|e| !e.starts_with(&format!("{}=", brokered::PEERS_ENV)));
job_env.push(brokered::peers_env(n));
}
let mut cfg = spec::build(
&heimdall_plain::spec::SpecInput {
argv,
env: &job_env,
input_dir: &input_dir,
output_dir: &output_dir,
read: &job.policy.read,
write: &job.policy.write,
limits: &limits,
uid: me.uid(),
gid: me.gid(),
cgroup_path: &cgroup_path,
},
self.cfg.unlisted(),
plan.is_some(),
);
if let Some(ids) = self.host_ids {
idmap::apply(&mut cfg, ids);
idmap::shift_tree(&rootfs, ids).map_err(|e| io(format!("host_ids: rootfs: {e}")))?;
for d in [&input_dir, &output_dir] {
idmap::give_tree(d, ids)
.map_err(|e| io(format!("host_ids: {}: {e}", d.display())))?;
}
}
iolimit::apply(&mut cfg, &self.io_max);
spec::set_nofile(&mut cfg, nofile);
// The workspace: its mount, and a file size limit that lets a
// workspace file grow to the workspace's cap (`/out` is held to
// max_output_file_bytes by the output scan either way).
let mut ws_watch = None;
if let Some(ws) = &job.policy.workspace {
let host = match ws.kind() {
heimdall_core::WorkspaceKind::Tmpfs => None,
heimdall_core::WorkspaceKind::HostDir => {
let dir = workspace::host_dir(&self.cfg.work_dir, &job.id);
workspace::create_host_dir(&dir).map_err(io)?;
// With a host id range the job runs as host uid `base`: the folder is
// given to it like /in and /out, or the job cannot write its workspace.
if let Some(ids) = self.host_ids {
idmap::give_tree(&dir, ids)
.map_err(|e| io(format!("host_ids: {}: {e}", dir.display())))?;
}
ws_watch = Some(shim::WorkspaceWatch {
dir: dir.clone(),
cap: ws.bytes,
});
Some(dir)
}
};
let m = workspace::mount(ws, host.as_deref()).map_err(setup)?;
if let Some(ms) = cfg["mounts"].as_array_mut() {
ms.push(m);
}
let fsize = ws.bytes.max(limits.max_output_file_bytes);
if let Some(rl) = cfg["process"]["rlimits"].as_array_mut() {
for r in rl.iter_mut() {
if r["type"] == "RLIMIT_FSIZE" {
r["hard"] = fsize.into();
r["soft"] = fsize.into();
}
}
}
}
std::fs::write(
bundle.join("config.json"),
serde_json::to_vec_pretty(&cfg).expect("config serializes"),
)
.map_err(io)?;
let req = ShimRequest {
id: format!("heimdall-{}", job.id),
bundle: bundle.to_path_buf(),
state_root: self.cfg.work_dir.join("state"),
wall_ms: limits.wall_ms(),
cgroup_dir: self
.cfg
.cgroup_root
.join(cgroup_path.trim_start_matches('/')),
landlock,
generated: plan.clone(),
learning: None,
broker: self.broker_mode().map(|mode| BrokerRequest {
mode,
policy: brokered::job_policy(job),
observe: self.observer.is_some(),
}),
io_weight: self
.kernel
.has_controller("io")
.then_some(self.cfg.io_weight),
workspace: ws_watch,
};
let started_unix = now_unix();
// The jobs' parent cgroup, created here (create_dir_all tolerates a
// parallel job creating it too): libcgroups 0.7.0 checks exists()
// then mkdirs, so two parallel jobs racing on a missing parent fail
// EEXIST. libcgroups still enables the controllers on it.
if let Some(parent) = req.cgroup_dir.parent() {
let _ = std::fs::create_dir_all(parent);
}
// The observer: the shim's event lines are relayed into a bounded
// queue that the observer's own thread drains.
let observing = self.observer.as_ref().map(|obs| {
let (sink, rx) = heimdall_broker::EventSink::channel(
job.policy.event_job_id(&job.id),
heimdall_broker::observe::DEFAULT_QUEUE,
);
let stop = Arc::new(std::sync::atomic::AtomicBool::new(false));
let h = heimdall_broker::spawn_observer(Arc::clone(obs), rx, Arc::clone(&stop));
(sink, stop, h)
});
let resp = self.run_shim(
&req,
limits.wall,
observing.as_ref().map(|(s, _, _)| s.clone()),
);
if let Some((sink, stop, h)) = observing {
drop(sink);
stop.store(true, std::sync::atomic::Ordering::Release);
// A slow observer keeps its thread; the job's result does not wait.
let deadline = Instant::now() + OBSERVER_GRACE;
while !h.is_finished() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(5));
}
}
let resp = resp?;
let finished_unix = now_unix().max(started_unix);
if let Some(e) = &resp.error {
if resp.exit_code.is_none() && resp.signal.is_none() {
return Err(setup(e.clone()));
}
}
let runner = self.job_features(&resp, plan.as_ref())?;
if let Some(w) = req.io_weight {
let want = format!("default {w}");
if resp.io_weight.as_deref() != Some(want.as_str()) {
return Err(setup(format!(
"io.weight is {:?}, not {want:?}",
resp.io_weight
)));
}
}
let killed = if resp.wall_killed {
Some(KillReason::Wall)
} else if resp.workspace_killed {
Some(KillReason::Workspace)
} else if resp.oom_kills > 0 {
Some(KillReason::Memory)
} else {
None
};
let exit = ExitInfo {
code: resp.exit_code,
signal: resp.signal,
killed,
};
let scanned = outputs::scan(&output_dir, &limits);
// The rule: when the output scan refuses `output_dir` (out-1, out-2)
// the signed list is empty, so Heimdall empties `output_dir` too and
// the result stays self-consistent (`check_outputs` passes on it).
if scanned.is_err() {
empty_dir(&output_dir).map_err(|e| {
io(format!(
"emptying the refused output_dir {}: {e}",
output_dir.display()
))
})?;
}
let verdict = match (&killed, &scanned) {
(Some(KillReason::Wall), _) => Verdict::refused(
codes::LIMIT_WALL,
format!("killed at the wall-clock limit of {} ms", limits.wall_ms()),
),
(Some(KillReason::Workspace), _) => Verdict::refused(
codes::LIMIT_WORKSPACE,
format!(
"killed at the workspace cap of {} bytes (it held {} bytes)",
job.policy.workspace.map_or(0, |w| w.bytes),
resp.workspace_bytes
),
),
(Some(KillReason::Memory), _) => Verdict::refused(
codes::LIMIT_MEMORY,
format!(
"killed at the memory limit of {} bytes ({} OOM kills)",
limits.memory_bytes, resp.oom_kills
),
),
_ if resp.pids_max > 0 && resp.signal != Some(libc::SIGSYS) => Verdict::refused(
codes::LIMIT_PIDS,
format!(
"the job hit the process limit of {} ({} refused forks), then {}",
limits.pids,
resp.pids_max,
match (resp.exit_code, resp.signal) {
(Some(c), _) => format!("exited with code {c}"),
(None, Some(s)) => format!("died from signal {s}"),
(None, None) => "left no exit status".to_string(),
}
),
),
(_, Err((code, reason))) => Verdict::refused(*code, reason.clone()),
_ => match (resp.exit_code, resp.signal) {
(Some(0), _) => Verdict::Accepted,
(Some(c), _) => Verdict::refused(
codes::EXIT_NONZERO,
format!("the job exited with code {c}"),
),
(None, Some(s)) if s == libc::SIGSYS => Verdict::refused(
codes::SECCOMP_KILL,
format!(
"the job died from signal {s} (SIGSYS): a system call its seccomp profile ({}) kills",
match &runner.seccomp {
Protection::Generated { profile_digest } =>
format!("generated, {profile_digest}"),
_ => "the fixed profile".to_string(),
}
),
),
(None, Some(s)) => {
let mut why = format!("the job died from signal {s}");
if s == libc::SIGXFSZ {
why.push_str(&format!(
" (a file over the size limit of {} bytes)",
limits.max_output_file_bytes
));
}
Verdict::refused(codes::EXIT_SIGNAL, why)
}
(None, None) => Verdict::refused(codes::EXIT_SIGNAL, "the job left no exit status"),
},
};
let result = JobResult {
job_id: job.id,
nonce: job.nonce,
verdict,
outputs: scanned.unwrap_or_default(),
exit,
started_unix,
finished_unix,
events: match self.broker_mode() {
Some(m) => brokered::events(m, resp.broker.as_ref()),
None => vec![],
},
runner,
kernel: Some(self.kernel.features()),
};
Ok(self.key.sign(result)?)
}
}
impl JobRunner for HeimdallRunner {
fn run(&self, job: Job) -> Result<SignedResult, RunError> {
check_job(&job)?;
if job.policy.net.is_some() && self.broker_mode() != Some(heimdall_broker::Mode::Full) {
return Err(RunError::Unsupported {
what: format!(
"a fetch job needs the broker in full mode (USER_NOTIF + ADDFD); this runner has {:?}",
self.broker_mode()
),
});
}
let bundle = self.new_bundle(&job)?;
let r = self.run_in(&job, &bundle);
let _ = std::fs::remove_dir_all(&bundle);
// A host-folder workspace is deleted when its job ends, whatever the end.
let _ = std::fs::remove_dir_all(workspace::host_dir(&self.cfg.work_dir, &job.id));
r
}
fn verifying_key(&self) -> VerifyingKey {
self.key.verifying_key()
}
fn features(&self) -> Features {
self.features.clone()
}
}
/// How long `run` waits after a job for its observer to take the job's last
/// events (a slower observer gets them later, or loses them, counted).
pub const OBSERVER_GRACE: Duration = Duration::from_secs(2);
/// The start of every network event line on the shim's stdout.
const EVENT_LINE: &[u8] = b"{\"schema\":\"heimdall.net-event/1\"";
/// Reads the shim's stdout: network event lines go to `events` as they come
/// (relayed, never blocking: a full queue drops and counts), everything
/// else is the response (at most 16 MiB).
fn read_shim_stdout(stdout: impl Read, events: Option<&heimdall_broker::EventSink>) -> Vec<u8> {
use std::io::BufRead;
let mut r = std::io::BufReader::new(stdout);
let mut out = Vec::new();
let mut line = Vec::new();
loop {
line.clear();
match (&mut r).take(1 << 20).read_until(b'\n', &mut line) {
Ok(0) | Err(_) => break,
Ok(_) => {}
}
if line.starts_with(EVENT_LINE) {
if let Some(sink) = events {
if let Ok(ev) = NetEvent::from_json_line(&String::from_utf8_lossy(&line)) {
sink.relay(ev);
}
}
continue;
}
if out.len() + line.len() <= 16 << 20 {
out.extend_from_slice(&line);
}
}
out
}
/// This process's hard `RLIMIT_NOFILE`. A job can never get more than this.
fn own_nofile_hard() -> u64 {
heimdall_plain::spec::own_nofile_hard()
}