crates/heimdall-container/src/learn.rs
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Ignalina ApS and Vetra Automation AB
//! **Learning inside the protected container** (`heimdall profile
//! probe-container`): run ONE program as an ordinary [`crate::HeimdallRunner`]
//! job — user, mount, pid, ipc, uts and cgroup namespaces, an EMPTY network
//! namespace, the read-only root of an image pinned by digest and verified at
//! the start, Landlock, no capabilities, rlimits, cgroup v2 limits — while
//! Heimdall records every system call it makes, so a workload driver OUTSIDE
//! the program's process tree (TPC-DS queries, HTTP load, scripted clients)
//! can drive it.
//!
//! Heimdall is a defensive sandbox. The two deviations from a normal job,
//! stated truthfully in what this path reports and never signed:
//!
//! 1. **seccomp**: instead of the fixed profile ([`crate::profile`]) or a
//! generated one ([`crate::genprof`]), the executor installs the LEARNING
//! filter of heimdall-probe (default `SECCOMP_RET_USER_NOTIF`, the same
//! filter and prelude as `heimdall profile probe-exec`) right before the
//! program starts, and hands its listener to the runner. Every call is
//! recorded and allowed (`FLAG_CONTINUE`). An enforcing filter would kill
//! exactly the calls this path is meant to learn. No signed result comes
//! out of a learning run, and nothing here is a
//! [`heimdall_core::Protection::Generated`] claim.
//! 2. **the channel** (optional): one host directory bind-mounted read-write
//! (`nosuid`, `nodev`, `noexec`) at [`CHANNEL_DIR`], where Landlock allows
//! read, write and making unix sockets ([`crate::confine::rights::channel`]),
//! so a driver outside connects to the sockets the program creates there
//! (`postgres -k /run/heimdall-channel`, `curl --unix-socket`). The network
//! namespace stays empty. Without a channel there is no read-write mount
//! besides `/out` and `/tmp`.
//!
//! The listener path: the shim gives the job's init one end of a socketpair;
//! the executor (inside the job, after Landlock) installs the learning filter
//! with `NEW_LISTENER`, sends the listener over that end (`sendmsg` on
//! heimdall-seccomp's `HANDOFF_FD`, a prelude call) and reports on its report
//! fd; the shim forwards the listener and the program's host pid to the
//! runner over an abstract unix socket (the runner checks the peer's pid is
//! its shim; the job itself, in its empty network namespace and Landlock
//! scope, can reach no abstract socket outside). The per-boot key stays in the runner; the shim
//! never sees it.
//!
//! Signals: the runner forwards each signal its caller hands it to the shim,
//! the shim forwards it to the program (the container's init; as pid 1 of
//! its namespace the program sees only the signals it has a handler for).
//! After [`STOP_SIGNALS`] the shim waits `term_grace` and then kills the job.
use std::collections::VecDeque;
use std::ffi::OsString;
use std::io::{BufRead, Read};
use std::os::fd::{AsFd, OwnedFd};
use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
use std::os::unix::net::{UnixListener, UnixStream};
use std::path::{Component, Path, PathBuf};
use std::process::Child;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use heimdall_core::{Job, JobKind, RunError};
use heimdall_plain::image::OciLayout;
use heimdall_seccomp::Libc;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::confine::{self, LandlockApplied};
use crate::shim::{ShimRequest, ShimResponse};
use crate::{check_job, existing_dir, invalid, io, setup, spec, HeimdallRunner, SHIM_GRACE};
/// Where the channel directory is mounted inside the job.
pub const CHANNEL_DIR: &str = "/run/heimdall-channel";
/// The signals that stop the program: forwarded, then a kill after the grace.
pub const STOP_SIGNALS: [i32; 3] = [libc::SIGTERM, libc::SIGINT, libc::SIGQUIT];
/// Every signal a caller may hand on to the program (the stop signals plus
/// HUP, USR1, USR2, which a daemon may use to reload or rotate).
pub const FORWARDED: [i32; 6] = [
libc::SIGTERM,
libc::SIGINT,
libc::SIGQUIT,
libc::SIGHUP,
libc::SIGUSR1,
libc::SIGUSR2,
];
/// The learning part of a shim request.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LearnPlan {
/// The name of the runner's ABSTRACT unix socket the shim hands the
/// listener and pid to (no path, so no length limit from the work dir;
/// the runner accepts only its own shim, by `SO_PEERCRED`).
pub handoff: String,
/// How long the shim waits after a stop signal before it kills the job.
pub term_grace_ms: u64,
/// A host file the program's stdout and stderr are appended to
/// (`None`: `/dev/null`, as for every job).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub log: Option<PathBuf>,
/// Run the program as this uid and gid (0: as root of the job's user
/// namespace, like every job). Not 0: the executor, before Landlock,
/// moves the program into a NESTED user namespace where the job's root
/// (the runner's own uid on the host) is this id ([`LearnSpec::uid`]).
/// The same host uid, and a program that refuses to run as root
/// (postgres) runs. youki itself cannot start a rootless job whose user
/// namespace has no root mapped.
#[serde(default)]
pub uid: u32,
}
/// How the job's OCI config differs from a normal job's.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct LearnSpec {
/// The host channel directory (mounted at [`CHANNEL_DIR`]).
pub channel: Option<PathBuf>,
/// The size of the `/tmp` tmpfs in bytes (`None`: heimdall-plain's).
pub tmp_bytes: Option<u64>,
/// The uid the program runs as ([`LearnPlan::uid`]). Not 0: the job's
/// init keeps ONE capability, `CAP_SETFCAP` (effective and permitted,
/// never bounding, inheritable or ambient), which the kernel demands to
/// map the job's root into the nested namespace; the executor drops
/// every capability right after the maps, and the program, a non-root
/// uid with an empty bounding set, holds none.
pub uid: u32,
}
/// Turn a normal job's `config.json` (from [`crate::spec::build`]) into the
/// learning job's: no seccomp profile for the runtime to install (the
/// executor installs the learning filter), the channel mount when there is
/// one, the `/tmp` size and, for another uid, `CAP_SETFCAP` until the maps.
pub fn learning_config(cfg: &mut Value, s: &LearnSpec) {
if s.uid != 0 {
let c = &mut cfg["process"]["capabilities"];
c["effective"] = json!(["CAP_SETFCAP"]);
c["permitted"] = json!(["CAP_SETFCAP"]);
}
if let Some(l) = cfg["linux"].as_object_mut() {
l.remove("seccomp");
}
if let Some(m) = cfg["mounts"].as_array_mut() {
if let Some(t) = s.tmp_bytes {
for x in m.iter_mut().filter(|x| x["destination"] == "/tmp") {
if let Some(o) = x["options"].as_array_mut() {
o.retain(|o| !o.as_str().is_some_and(|o| o.starts_with("size=")));
o.push(json!(format!("size={t}")));
}
}
}
if let Some(c) = &s.channel {
m.push(json!({
"destination": CHANNEL_DIR,
"type": "bind",
"source": c,
"options": ["rbind", "rw", "nosuid", "nodev", "noexec"],
}));
}
}
}
fn components(p: &Path) -> VecDeque<OsString> {
p.components()
.filter_map(|c| match c {
Component::Normal(n) => Some(n.to_os_string()),
Component::ParentDir => Some("..".into()),
_ => None,
})
.collect()
}
/// Resolve `path` inside `rootfs` without ever leaving it: every symlink on
/// the way (absolute or relative) is resolved against `rootfs`. `None` when
/// a component is missing or there are more than 40 links.
pub fn resolve_in(rootfs: &Path, path: &Path) -> Option<PathBuf> {
let mut pending = components(path);
let mut cur: Vec<OsString> = Vec::new();
let mut links = 0;
while let Some(c) = pending.pop_front() {
if c == ".." {
cur.pop();
continue;
}
let mut cand = rootfs.to_path_buf();
cand.extend(&cur);
cand.push(&c);
let md = std::fs::symlink_metadata(&cand).ok()?;
if md.file_type().is_symlink() {
links += 1;
if links > 40 {
return None;
}
let t = std::fs::read_link(&cand).ok()?;
if t.is_absolute() {
cur.clear();
}
let mut next = components(&t);
next.extend(pending.drain(..));
pending = next;
} else {
cur.push(c);
}
}
let mut p = rootfs.to_path_buf();
p.extend(&cur);
Some(p)
}
/// The program `argv0` inside `rootfs` (a path, or the first regular file
/// of that name in the `PATH` of `env`).
fn find_program(rootfs: &Path, argv0: &str, env: &[String]) -> Option<PathBuf> {
if argv0.contains('/') {
return resolve_in(rootfs, Path::new(argv0)).filter(|p| p.is_file());
}
let path = env
.iter()
.find_map(|e| e.strip_prefix("PATH="))
.unwrap_or("/usr/local/bin:/usr/bin:/bin");
path.split(':')
.filter_map(|d| resolve_in(rootfs, &Path::new(d).join(argv0)))
.find(|p| p.is_file())
}
/// The largest program file [`detect_libc`] reads.
const MAX_PROGRAM_BYTES: u64 = 256 << 20;
fn detect_at(rootfs: &Path, argv0: &str, env: &[String], depth: u8) -> Option<Libc> {
let p = find_program(rootfs, argv0, env)?;
let f = std::fs::OpenOptions::new()
.read(true)
.custom_flags(libc::O_NOFOLLOW)
.open(&p)
.ok()?;
let mut bytes = Vec::new();
f.take(MAX_PROGRAM_BYTES).read_to_end(&mut bytes).ok()?;
if let Some(rest) = bytes.strip_prefix(b"#!") {
if depth >= 2 {
return None;
}
let first = rest.split(|b| *b == b'\n').next()?;
let first = String::from_utf8_lossy(first);
let interp = first.split_whitespace().next()?;
return detect_at(rootfs, interp, env, depth + 1);
}
Libc::detect(&bytes)
}
/// The libc of the program `argv0` (a path, or a name looked up in the
/// job's `PATH`) inside `rootfs`: its ELF (heimdall-seccomp's
/// [`Libc::detect`]), or for a `#!` script its interpreter's. `None` when it
/// does not show (e.g. a static binary without a libc mark).
pub fn detect_libc(rootfs: &Path, argv0: &str, env: &[String]) -> Option<Libc> {
detect_at(rootfs, argv0, env, 0)
}
/// One learning run.
#[derive(Clone, Debug)]
pub struct LearnJob {
/// A native job (image pinned by digest, argv, input/output dirs, policy
/// limits); its `policy.seccomp` must be `None`.
pub job: Job,
/// `KEY=value` added to the image's environment (replacing the same key).
pub env: Vec<String>,
pub spec: LearnSpec,
/// The program's libc; `None`: detected from the image ([`detect_libc`]).
pub libc: Option<Libc>,
pub term_grace: Duration,
/// See [`LearnPlan::log`].
pub log: Option<PathBuf>,
}
/// The program is about to start: its learning listener, its host pid and
/// its libc. Every call of the program blocks until the listener answers.
#[derive(Debug)]
pub struct Started {
pub listener: OwnedFd,
pub pid: i32,
pub libc: Libc,
}
/// How a learning run ended (unsigned: a learning run is never a result).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct LearnOutcome {
pub exit_code: Option<i32>,
pub signal: Option<i32>,
/// Killed at the wall-clock limit.
pub wall_killed: bool,
/// Killed after a stop signal and the grace.
pub term_killed: bool,
pub oom_kills: u64,
pub pids_max: u64,
/// Landlock as the executor reported it from inside the job.
pub landlock: Option<LandlockApplied>,
/// The program started under the learning filter (its listener came).
pub started: bool,
pub ran_ms: u64,
}
/// The image's env with `extra` (`KEY=value`) added, replacing the same keys.
fn merged_env(image: &[String], extra: &[String]) -> Vec<String> {
let key = |e: &str| e.split('=').next().unwrap_or("").to_string();
let mut env: Vec<String> = image.to_vec();
for kv in extra {
let k = key(kv);
env.retain(|e| key(e) != k);
env.push(kv.clone());
}
env
}
/// Take the listener and pid the shim (and only the shim) hands over.
fn take_handoff(s: &UnixStream, shim_pid: i32) -> Result<(OwnedFd, i32), String> {
let peer = crate::sys::peer_pid(s.as_fd()).map_err(|e| format!("peer: {e}"))?;
if peer != shim_pid {
return Err(format!("a process other than the shim ({peer}) connected"));
}
s.set_nonblocking(false).map_err(|e| e.to_string())?;
s.set_read_timeout(Some(Duration::from_secs(10)))
.map_err(|e| e.to_string())?;
let fd = heimdall_seccomp::notif::recv_fd(s.as_fd()).map_err(|e| format!("listener: {e}"))?;
let mut line = String::new();
std::io::BufReader::new(s.take(32))
.read_line(&mut line)
.map_err(|e| format!("pid: {e}"))?;
let pid = line
.trim()
.parse::<i32>()
.map_err(|e| format!("pid {line:?}: {e}"))?;
Ok((fd, pid))
}
impl HeimdallRunner {
fn learn_in(
&self,
lj: &LearnJob,
bundle: &Path,
signals: &AtomicU64,
on_start: impl FnOnce(Started),
) -> Result<LearnOutcome, RunError> {
let job = &lj.job;
let JobKind::Native { image, argv } = &job.kind else {
unreachable!("checked")
};
let input_dir = existing_dir("input_dir", &job.input_dir)?;
let output_dir = existing_dir("output_dir", &job.output_dir)?;
if std::fs::read_dir(&output_dir).map_err(io)?.next().is_some() {
return Err(invalid(format!(
"output_dir {} is not empty",
job.output_dir.display()
)));
}
for p in job.policy.read.iter().chain(&job.policy.write) {
if !p.exists() {
return Err(invalid(format!(
"policy path {} does not exist",
p.display()
)));
}
}
let channel = match &lj.spec.channel {
Some(c) => Some(existing_dir("channel", c)?),
None => None,
};
for kv in &lj.env {
if kv.split_once('=').is_none_or(|(k, _)| k.is_empty()) || kv.contains('\0') {
return Err(invalid(format!("env {kv:?} is not KEY=value")));
}
}
// Verified (manifest, config, every layer by sha256) and unpacked
// fresh for this start.
let rootfs = bundle.join("rootfs");
std::fs::create_dir_all(&rootfs).map_err(io)?;
let unpacked = OciLayout::new(&self.cfg.image_layout)
.unpack(image, &rootfs)
.map_err(|e| RunError::Image {
reason: e.to_string(),
})?;
let env = merged_env(&unpacked.env, &lj.env);
let libc = match lj.libc {
Some(l) => l,
None => detect_libc(&rootfs, &argv[0], &env).ok_or_else(|| {
invalid(format!(
"cannot tell the libc of {} from the image (a static binary?): name it (--libc glibc|musl)",
argv[0]
))
})?,
};
if channel.is_some() {
// The mount point, in this start's own copy of the image.
let run = rootfs.join("run");
match std::fs::symlink_metadata(&run) {
Ok(m) if m.file_type().is_symlink() || !m.is_dir() => {
return Err(invalid(
"the image's /run is not a directory: no place for the channel",
))
}
Ok(_) => {}
Err(_) => std::fs::create_dir(&run).map_err(io)?,
}
let ch = run.join("heimdall-channel");
if std::fs::symlink_metadata(&ch).is_ok_and(|m| !m.is_dir()) {
return Err(invalid(
"the image's /run/heimdall-channel is not a directory",
));
}
std::fs::create_dir_all(&ch).map_err(io)?;
}
let landlock = if self.features.landlock {
let mut p = confine::plan(&rootfs, &job.policy.read, &job.policy.write).map_err(io)?;
if channel.is_some() {
p.channel.push(CHANNEL_DIR.into());
}
Some(p)
} else {
None
};
let me = std::fs::metadata("/proc/self").map_err(io)?;
let cgroup_path = format!("/{}/job-{}", self.cfg.cgroup_parent, job.id);
let limits = job.policy.limits;
let mut cfg = spec::build(
&heimdall_plain::spec::SpecInput {
argv,
env: &env,
input_dir: &input_dir,
output_dir: &output_dir,
read: &job.policy.read,
write: &job.policy.write,
limits: &limits,
uid: me.uid(),
gid: me.gid(),
cgroup_path: &cgroup_path,
},
self.cfg.unlisted(),
false,
);
crate::iolimit::apply(&mut cfg, &self.io_max);
// The job's RLIMIT_NOFILE (Limits::nofile, else the runner's), as for every job.
spec::set_nofile(&mut cfg, self.nofile_for(&limits)?);
learning_config(
&mut cfg,
&LearnSpec {
channel: channel.clone(),
..lj.spec.clone()
},
);
std::fs::write(
bundle.join("config.json"),
serde_json::to_vec_pretty(&cfg).expect("config serializes"),
)
.map_err(io)?;
let name = format!("heimdall-learn-{}-{}", std::process::id(), job.id);
let handoff = {
use std::os::linux::net::SocketAddrExt;
let addr =
std::os::unix::net::SocketAddr::from_abstract_name(name.as_bytes()).map_err(io)?;
UnixListener::bind_addr(&addr).map_err(io)?
};
handoff.set_nonblocking(true).map_err(io)?;
let req = ShimRequest {
id: format!("heimdall-{}", job.id),
bundle: bundle.to_path_buf(),
state_root: self.cfg.work_dir.join("state"),
wall_ms: limits.wall_ms(),
cgroup_dir: self
.cfg
.cgroup_root
.join(cgroup_path.trim_start_matches('/')),
landlock,
generated: None,
// The learning filter holds the job's one USER_NOTIF listener.
broker: None,
learning: Some(LearnPlan {
handoff: name,
term_grace_ms: u64::try_from(lj.term_grace.as_millis()).unwrap_or(u64::MAX),
log: lj.log.clone(),
uid: lj.spec.uid,
}),
io_weight: self
.kernel
.has_controller("io")
.then_some(self.cfg.io_weight),
workspace: None,
};
let (mut child, reader) = self.spawn_shim(&req, None)?;
let shim_pid = child.id() as i32;
let deadline = Instant::now() + limits.wall + SHIM_GRACE + lj.term_grace;
let mut on_start = Some(on_start);
let kill_all = |child: &mut Child| {
let _ = child.kill();
let _ = child.wait();
// The job's processes outlive a killed shim: end them by cgroup.
let _ = std::fs::write(req.cgroup_dir.join("cgroup.kill"), "1");
};
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) => {}
Err(e) => {
kill_all(&mut child);
return Err(setup(format!("waiting for the shim: {e}")));
}
}
if on_start.is_some() {
match handoff.accept() {
Ok((s, _)) => match take_handoff(&s, shim_pid) {
Ok((listener, pid)) => {
if let Some(f) = on_start.take() {
f(Started {
listener,
pid,
libc,
});
}
}
Err(e) => {
if std::env::var_os("HEIMDALL_CONTAINER_DEBUG").is_some() {
eprintln!("heimdall: learning hand-off refused: {e}");
}
}
},
Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {}
Err(_) => {}
}
}
let pending = signals.swap(0, Ordering::SeqCst);
for sig in FORWARDED {
if pending & (1u64 << sig) != 0 {
let _ = crate::sys::kill(shim_pid, sig);
}
}
if Instant::now() >= deadline {
kill_all(&mut child);
return Err(setup("the shim did not finish in time"));
}
std::thread::sleep(Duration::from_millis(10));
}
let out = reader.join().unwrap_or_default();
let out = String::from_utf8_lossy(&out);
let resp: ShimResponse = match serde_json::from_str(out.trim()) {
Ok(r) => r,
Err(e) => {
let _ = std::fs::write(req.cgroup_dir.join("cgroup.kill"), "1");
return Err(setup(format!("bad shim response ({e}): {}", out.trim())));
}
};
if let Some(e) = &resp.error {
if resp.exit_code.is_none() && resp.signal.is_none() {
return Err(setup(e.clone()));
}
}
let Some(inside) = &resp.inside else {
return Err(setup(format!(
"no report from inside the job (the executor never ran){}",
resp.error
.as_deref()
.map(|e| format!(": {e}"))
.unwrap_or_default()
)));
};
if let Some(e) = &inside.error {
return Err(setup(format!("inside the job, before execve: {e}")));
}
if !inside.no_new_privs {
return Err(setup("no-new-privs was not set inside the job"));
}
if !inside.learning {
return Err(setup(
"the learning filter was not confirmed from inside the job",
));
}
if self.features.landlock && inside.landlock.is_none() {
return Err(setup("Landlock was not applied inside the job"));
}
if let Some(s) = &resp.swap_max {
if s != "0" {
return Err(setup(format!("memory.swap.max is {s}, not 0")));
}
}
if let Some(w) = req.io_weight {
let want = format!("default {w}");
if resp.io_weight.as_deref() != Some(want.as_str()) {
return Err(setup(format!(
"io.weight is {:?}, not {want:?}",
resp.io_weight
)));
}
}
if !self.io_max.is_empty() {
let Some(got) = &resp.io_max else {
return Err(setup("io.max was not read back from the job's cgroup"));
};
crate::iolimit::check(got, &self.io_max).map_err(setup)?;
}
Ok(LearnOutcome {
exit_code: resp.exit_code,
signal: resp.signal,
wall_killed: resp.wall_killed,
term_killed: resp.term_killed,
oom_kills: resp.oom_kills,
pids_max: resp.pids_max,
landlock: inside.landlock,
started: on_start.is_none(),
ran_ms: resp.ran_ms,
})
}
/// Run `lj` in learning mode (see the module docs). `on_start` gets the
/// listener once the program is about to start; `signals` is a bit set
/// (`1 << signo`) the caller fills (e.g. from its signal handlers) and
/// this call drains, forwarding each signal to the program. Returns when
/// the program and every process it started are gone.
pub fn learn(
&self,
lj: LearnJob,
signals: &AtomicU64,
on_start: impl FnOnce(Started),
) -> Result<LearnOutcome, RunError> {
check_job(&lj.job)?;
if lj.job.policy.net.is_some() {
return Err(invalid(
"a learning job has no network: its netns is empty and no broker injects a peer (drive it through --channel)",
));
}
if lj.job.policy.workspace.is_some() {
return Err(invalid(
"a learning job has no workspace (use --channel or /tmp)",
));
}
if self.host_ids.is_some() {
return Err(RunError::Unsupported {
what: "learning runs on a runner with host_ids (use the default single-id mapping)"
.into(),
});
}
if lj.job.policy.seccomp.is_some() {
return Err(invalid(
"a learning job runs under the learning filter: policy.seccomp must be None",
));
}
let bundle = self.new_bundle(&lj.job)?;
let r = self.learn_in(&lj, &bundle, signals, on_start);
let _ = std::fs::remove_dir_all(&bundle);
r
}
}