crates/heimdall-trace/src/tracer.rs
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Ignalina ApS and Vetra Automation AB
//! The in-job tracer (`heimdall-trace`, PID 1 of a trace job). Heimdall is
//! a defensive sandbox: the tracer starts the command under heimdall-probe's
//! learning filter (seccomp USER_NOTIF for every call, the same filter as
//! `heimdall profile probe`), answers every call itself and records:
//!
//! - every call with its argument facts, per program (as the probe does);
//! - `execve`/`execveat`: the path and the argument vector;
//! - `connect`/`bind`/`sendto`/`sendmsg` with an address: the address, and it
//! refuses all but AF_UNIX (`EACCES`);
//! - file calls (opens, creates, removes, renames, links, mode/owner changes,
//! truncates): the path and flags, and it refuses every write outside the
//! writable paths, into `/out`, or whose path it could not read (`EACCES`);
//! - calls of a foreign arch or an unknown number: refused (`ENOSYS`).
//!
//! When the command ends (or the deadline passes) the tracer kills every
//! process left (it is PID 1: `kill(-1, SIGKILL)`), reaps them all, and only
//! then writes the trace into the file it created before the command
//! started; nothing of the package is alive while it writes. As PID 1 it
//! cannot be signalled from inside the job.
//!
//! What it cannot see: calls the job's own seccomp profile answers before
//! the listener does (`EPERM` or kill: the stricter action wins), the
//! vDSO's time calls, and a `sendmsg` on fd 1021 (heimdall-seccomp's launch
//! prelude lets it through for the listener hand-off; the fd is closed at
//! exec, and the job has no network). The command is started with a plain
//! `execve` under the filter (`Launch::exec_by_path`: the job's fixed
//! profile refuses `execveat`), so its first exec is in the trace too, from
//! the program `/heimdall/heimdall-trace`. A path or address is read from the
//! caller's memory when the call stops; a second thread can change it before
//! the kernel reads it again (`Continue`). Such a race can make the trace
//! show another path than the kernel used, but never widen what the job may
//! do: Landlock and the read-only root still decide. The TOCTOU-safe way is
//! the broker's (open on the caller's behalf, `NOTIF_ADDFD`).
use std::ffi::{OsStr, OsString};
use std::fs::File;
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::{FileExt, OpenOptionsExt};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use heimdall_probe::facts::fixed_rules;
use heimdall_probe::learn::call_of;
use heimdall_seccomp::launch::{Exit, Launch, Stdio};
use heimdall_seccomp::notif::{Listener, Notif, Readiness, Response};
use heimdall_seccomp::{Action, Arch};
use crate::classify::{
file_call, join, net_call, open_writes, outside_tmp, refuse_net, refuse_write, sockaddr,
};
use crate::doc::{cut, ExecEvent, FileEvent, NetEvent, SyscallFact, Trace, MAX_ARGS, MAX_STR};
use crate::record::Recorder;
/// Exit code: a trace was written (complete, truncated, or incomplete with
/// its reason: the caller reads the status, so the reason reaches it).
pub const EXIT_TRACED: i32 = 0;
/// Exit code: no trace could be written (usage, the out file).
pub const EXIT_NO_TRACE: i32 = 71;
/// How long the tracer waits for killed processes to go.
const DRAIN: Duration = Duration::from_secs(10);
/// The tracer's options.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Config {
pub out: PathBuf,
pub timeout: Duration,
pub max_bytes: u64,
pub writable: Vec<String>,
pub env: Vec<(OsString, OsString)>,
pub cwd: Option<PathBuf>,
pub argv: Vec<OsString>,
}
/// The usage line.
pub const USAGE: &str = "heimdall-trace --out <file> [--timeout-ms <n>] [--max-bytes <n>] \
[--writable <dir>]... [--env K=V]... [--cwd <dir>] -- <program> [args...]";
/// Parse the tracer's arguments (without argv\[0\]).
pub fn parse_args<I: IntoIterator<Item = OsString>>(args: I) -> Result<Config, String> {
let mut it = args.into_iter();
let mut out = None;
let mut timeout = Duration::from_secs(60);
let mut max_bytes = crate::doc::DEFAULT_MAX_BYTES;
let mut writable = Vec::new();
let mut env = Vec::new();
let mut cwd = None;
let val = |it: &mut I::IntoIter, k: &str| -> Result<OsString, String> {
it.next().ok_or_else(|| format!("{k} needs a value"))
};
loop {
let Some(a) = it.next() else {
return Err("no command after --".into());
};
let s = a.to_string_lossy().into_owned();
match s.as_str() {
"--" => break,
"--out" => out = Some(PathBuf::from(val(&mut it, &s)?)),
"--timeout-ms" => {
let v = val(&mut it, &s)?;
let ms: u64 = v
.to_string_lossy()
.parse()
.map_err(|_| format!("--timeout-ms {v:?} is not a number"))?;
timeout = Duration::from_millis(ms);
}
"--max-bytes" => {
let v = val(&mut it, &s)?;
max_bytes = v
.to_string_lossy()
.parse()
.map_err(|_| format!("--max-bytes {v:?} is not a number"))?;
if max_bytes < crate::doc::MIN_MAX_BYTES {
return Err(format!(
"--max-bytes must be at least {}",
crate::doc::MIN_MAX_BYTES
));
}
}
"--writable" => {
let v = val(&mut it, &s)?.to_string_lossy().into_owned();
if !v.starts_with('/') {
return Err(format!("--writable {v:?} is not absolute"));
}
writable.push(crate::classify::normalize(&v));
}
"--env" => {
let v = val(&mut it, &s)?;
let b = v.as_bytes();
let eq = b
.iter()
.position(|&c| c == b'=')
.filter(|&i| i > 0)
.ok_or_else(|| format!("--env {v:?} is not K=V"))?;
env.push((
OsStr::from_bytes(&b[..eq]).to_os_string(),
OsStr::from_bytes(&b[eq + 1..]).to_os_string(),
));
}
"--cwd" => cwd = Some(PathBuf::from(val(&mut it, &s)?)),
other => return Err(format!("unknown option {other:?}")),
}
}
let argv: Vec<OsString> = it.collect();
if argv.is_empty() || argv[0].is_empty() {
return Err("no command after --".into());
}
if writable.is_empty() {
writable.push("/tmp".into());
}
Ok(Config {
out: out.ok_or("--out is required")?,
timeout,
max_bytes,
writable,
env,
cwd,
argv,
})
}
/// Run the tracer: create the out file, trace, write. The exit code.
pub fn main_with(cfg: &Config) -> i32 {
let out = match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.custom_flags(libc::O_NOFOLLOW)
.open(&cfg.out)
{
Ok(f) => f,
Err(e) => {
eprintln!("heimdall-trace: {}: {e}", cfg.out.display());
return EXIT_NO_TRACE;
}
};
let (trace, bytes) = trace(cfg);
let w = out
.set_len(0)
.and_then(|()| out.write_all_at(&bytes, 0))
.and_then(|()| out.set_len(bytes.len() as u64));
if let Err(e) = w {
eprintln!("heimdall-trace: writing {}: {e}", cfg.out.display());
return EXIT_NO_TRACE;
}
if !trace.is_usable() {
eprintln!("heimdall-trace: incomplete: {}", trace.incomplete);
}
EXIT_TRACED
}
fn lossy(s: &OsStr) -> String {
s.to_string_lossy().into_owned()
}
/// Trace the command; the finished document (incomplete on any failure).
pub fn trace(cfg: &Config) -> (Trace, Vec<u8>) {
let argv = cfg.argv.iter().map(|a| cut(lossy(a))).collect();
let mut rec = Recorder::new(argv, cfg.max_bytes);
// SAFETY: getpid cannot fail.
if unsafe { libc::getpid() } != 1 {
rec.incomplete("the tracer is not PID 1 of the job's pid namespace");
return rec.finish();
}
let Some(arch) = Arch::native() else {
rec.incomplete("this arch is not supported");
return rec.finish();
};
if let Err(e) = Listener::check_kernel() {
rec.incomplete(&format!("seccomp USER_NOTIF: {e}"));
return rec.finish();
}
let program = match find_program(&cfg.argv[0], &cfg.env) {
Some(p) => p,
None => {
rec.incomplete(&format!("no program {:?} in the image", cfg.argv[0]));
return rec.finish();
}
};
let mut l = Launch::new(&program)
.arg0(&cfg.argv[0])
.args(&cfg.argv[1..])
.rules(arch, fixed_rules(), Action::UserNotif)
.listener(true)
.exec_by_path(true)
.stdin(Stdio::Null)
.stdout(Stdio::Null)
.stderr(Stdio::Null);
for (k, v) in &cfg.env {
l = l.env(k, v);
}
if let Some(d) = &cfg.cwd {
l = l.cwd(d);
}
let mut child = match l.spawn() {
Ok(c) => c,
Err(e) => {
// EBUSY: a listener sits on the job already (a broker).
rec.incomplete(&format!("the command did not start under the tracer: {e}"));
return rec.finish();
}
};
let pid = child.pid();
let Some(listener) = child.take_listener() else {
rec.incomplete("no listener");
kill_all();
reap_all(Instant::now() + DRAIN);
return rec.finish();
};
// The tracer reaps every process itself (it is PID 1), the command too.
std::mem::forget(child);
let mut t = Tracer {
arch,
writable: &cfg.writable,
rec,
listener,
};
t.run(pid, cfg.timeout);
t.rec.finish()
}
struct Tracer<'a> {
arch: Arch,
writable: &'a [String],
rec: Recorder,
listener: Listener,
}
impl Tracer<'_> {
fn run(&mut self, pid: libc::pid_t, timeout: Duration) {
let deadline = Instant::now() + timeout;
let mut main_exit: Option<String> = None;
let mut killed_at: Option<Instant> = None;
let mut timed_out = false;
let mut leftovers = 0u64;
let mut hung_up = false;
loop {
for (p, st) in reap_now() {
if p == pid {
main_exit = Some(describe(st));
} else if killed_at.is_some() {
leftovers += 1;
}
}
if killed_at.is_none() && (main_exit.is_some() || Instant::now() >= deadline) {
timed_out = main_exit.is_none();
kill_all();
killed_at = Some(Instant::now());
}
if let Some(k) = killed_at {
if hung_up {
break;
}
if k.elapsed() > DRAIN {
self.rec
.incomplete("processes of the job did not end after SIGKILL");
break;
}
}
match self.listener.wait(Some(Duration::from_millis(50))) {
Ok(Readiness::Hangup) => {
hung_up = true;
if killed_at.is_none() {
// Every traced process has exited; reap the command.
std::thread::sleep(Duration::from_millis(1));
}
}
Ok(Readiness::Timeout) => {}
Ok(Readiness::Ready) => match self.listener.recv() {
Ok(n) => {
let r = self.handle(&n);
match self.listener.respond(n.id, r) {
Ok(()) => {}
Err(e) if e.raw_os_error() == Some(libc::ENOENT) => {}
Err(e) => {
self.rec.incomplete(&format!("answering a call: {e}"));
break;
}
}
}
Err(e) if e.raw_os_error() == Some(libc::ENOENT) => {}
Err(e) => {
self.rec.incomplete(&format!("receiving a call: {e}"));
break;
}
},
Err(e) => {
self.rec.incomplete(&format!("waiting for calls: {e}"));
break;
}
}
}
kill_all();
let (n, ended) = reap_all(Instant::now() + DRAIN);
for (p, st) in ended.iter().copied() {
if p == pid {
main_exit = Some(describe(st));
}
}
leftovers += n - u64::from(ended.iter().any(|(p, _)| *p == pid));
let t = self.rec.trace_mut();
t.timed_out = timed_out;
t.leftover_killed = leftovers;
t.exit = if timed_out {
"timed out".into()
} else {
main_exit.unwrap_or_else(|| "no exit status".into())
};
}
/// Record one call and decide its answer.
fn handle(&mut self, n: &Notif) -> Response {
self.rec.notification();
let call = match call_of(self.arch, n) {
Ok(c) => c,
Err(u) => {
self.rec.unknown(u);
return Response::Errno(libc::ENOSYS);
}
};
let tid = n.pid;
let program = link(tid, "exe").unwrap_or_else(|| "unknown".into());
self.rec.syscall(&program, SyscallFact::of(&call));
let a = n.data.args;
let name = call.syscall.as_str();
let mut mem = Mem::new(tid);
if name == "execve" || name == "execveat" {
let (dirfd, path_i, argv_i, flags) = if name == "execve" {
(None, 0, 1, 0)
} else {
(Some(a[0] as i32), 1, 2, a[4])
};
let path = mem
.cstr(a[path_i])
.map(|p| resolve(tid, dirfd, &p, flags & libc::AT_EMPTY_PATH as u64 != 0));
let (argv, argv_cut) = mem.argv(a[argv_i]);
if self.valid(n) {
self.rec.exec(ExecEvent {
program,
path: path.unwrap_or_else(|| "unreadable".into()),
argv,
argv_cut,
});
}
return Response::Continue;
}
if let Some((addr_i, len_i)) = net_call(name) {
let (ptr, len) = if name == "sendmsg" {
match mem.read(a[1], 16) {
Some(h) if h.len() == 16 => (
u64::from_ne_bytes(h[0..8].try_into().expect("8 bytes")),
u64::from(u32::from_ne_bytes(h[8..12].try_into().expect("4 bytes"))),
),
_ => (u64::MAX, 0),
}
} else {
(a[usize::from(addr_i)], a[usize::from(len_i)])
};
if ptr == 0 && name != "connect" && name != "bind" {
// A send on a connected socket: no address, no event.
return Response::Continue;
}
let (family, address) = match mem.read(ptr, len.min(128) as usize) {
Some(b) if ptr != u64::MAX => sockaddr(&b),
_ => ("unreadable".into(), String::new()),
};
let mut refused = refuse_net(&family);
if family == "unix"
&& name == "bind"
&& !address.is_empty()
&& !address.starts_with('@')
{
// A unix socket file is a write.
let p = resolve(tid, None, &address, false);
refused |= refuse_write(Some(&p), self.writable);
}
if !self.valid(n) {
return Response::Continue;
}
self.rec.net(NetEvent {
program,
op: name.to_string(),
family,
address,
refused,
});
return if refused {
Response::Errno(libc::EACCES)
} else {
Response::Continue
};
}
if let Some(fc) = file_call(name) {
let flags = if let Some(f) = fc.fixed_flags {
Some(f)
} else if let Some(i) = fc.flags {
if fc.flags_in_struct {
mem.read(a[usize::from(i)], 8)
.filter(|b| b.len() == 8)
.map(|b| u64::from_ne_bytes(b[..8].try_into().expect("8 bytes")))
} else {
Some(a[usize::from(i)] & 0xffff_ffff)
}
} else {
None
};
// Unreadable open flags count as a write (refused).
let write = fc.always_write || flags.is_none_or(open_writes);
let empty_ok = fc
.empty_path_flag
.is_some_and(|i| a[usize::from(i)] & libc::AT_EMPTY_PATH as u64 != 0);
let mut events = Vec::new();
let mut refuse = false;
for pa in &fc.paths {
let dirfd = pa.dirfd.map(|i| a[usize::from(i)] as i32);
let path = mem.cstr(a[usize::from(pa.path)]).map(|p| {
let r = resolve(tid, dirfd, &p, empty_ok);
if write {
real_parent(&r)
} else {
r
}
});
let refused = write && refuse_write(path.as_deref(), self.writable);
refuse |= refused;
let path = path.unwrap_or_else(|| "unreadable".into());
events.push(FileEvent {
program: program.clone(),
op: name.to_string(),
outside_tmp: write && outside_tmp(&path),
path,
flags: flags.map(|f| format!("{f:#x}")).unwrap_or_default(),
write,
refused,
});
}
if !self.valid(n) {
return Response::Continue;
}
for e in events {
self.rec.file(e);
}
return if refuse {
Response::Errno(libc::EACCES)
} else {
Response::Continue
};
}
Response::Continue
}
/// The call still waits (its memory was the caller's, not a recycled
/// tid's).
fn valid(&self, n: &Notif) -> bool {
self.listener.id_valid(n.id).unwrap_or(false)
}
}
/// `/proc/<tid>/<what>` as a string.
fn link(tid: u32, what: &str) -> Option<String> {
std::fs::read_link(format!("/proc/{tid}/{what}"))
.ok()
.map(|p| lossy(p.as_os_str()))
}
/// The absolute path a call means: `p` against its dirfd (or the cwd); an
/// empty `p` with `empty_ok` names the dirfd itself.
fn resolve(tid: u32, dirfd: Option<i32>, p: &str, empty_ok: bool) -> String {
let base = || match dirfd {
Some(fd) if fd != libc::AT_FDCWD => link(tid, &format!("fd/{fd}")),
_ => link(tid, "cwd"),
};
if p.is_empty() && empty_ok {
return base()
.map(|b| crate::classify::normalize(&b))
.unwrap_or_else(|| "unreadable".into());
}
if p.starts_with('/') {
return cut(crate::classify::normalize(p));
}
match base() {
Some(b) if b.starts_with('/') => cut(join(&b, p)),
_ => format!("unreadable/{}", cut(p.to_string())),
}
}
/// `p` with its parent directory's symlinks resolved (as the tracer sees the
/// job's root, which is the caller's), when the parent exists.
fn real_parent(p: &str) -> String {
let path = Path::new(p);
let (Some(parent), Some(name)) = (path.parent(), path.file_name()) else {
return p.to_string();
};
match std::fs::canonicalize(parent) {
Ok(r) => cut(lossy(r.join(name).as_os_str())),
Err(_) => p.to_string(),
}
}
/// The caller's memory (`/proc/<tid>/mem`, opened on first use).
struct Mem {
tid: u32,
f: Option<Option<File>>,
}
impl Mem {
fn new(tid: u32) -> Mem {
Mem { tid, f: None }
}
fn file(&mut self) -> Option<&File> {
if self.f.is_none() {
self.f = Some(File::open(format!("/proc/{}/mem", self.tid)).ok());
}
self.f.as_ref().and_then(|f| f.as_ref())
}
/// Up to `len` bytes at `addr` (fewer at the end of a mapping).
fn read(&mut self, addr: u64, len: usize) -> Option<Vec<u8>> {
if addr == 0 {
return None;
}
let f = self.file()?;
let mut b = vec![0u8; len];
let mut got = 0;
while got < len {
match f.read_at(&mut b[got..], addr + got as u64) {
Ok(0) => break,
Ok(n) => got += n,
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
Err(_) => break,
}
}
if got == 0 && len > 0 {
return None;
}
b.truncate(got);
Some(b)
}
/// A NUL-terminated string at `addr` (cut at [`MAX_STR`]).
fn cstr(&mut self, addr: u64) -> Option<String> {
let mut out = Vec::new();
let mut at = addr;
while out.len() <= MAX_STR {
// Never read across a page end in one go: the next page may be
// unmapped.
let to_page = (4096 - (at % 4096)) as usize;
let chunk = self.read(at, to_page.min(256))?;
if let Some(i) = chunk.iter().position(|&c| c == 0) {
out.extend_from_slice(&chunk[..i]);
return Some(cut(String::from_utf8_lossy(&out).into_owned()));
}
out.extend_from_slice(&chunk);
at += chunk.len() as u64;
}
Some(cut(String::from_utf8_lossy(&out).into_owned()))
}
/// A NULL-terminated pointer array of strings (argv); (args, cut).
fn argv(&mut self, addr: u64) -> (Vec<String>, bool) {
let mut v = Vec::new();
if addr == 0 {
return (v, false);
}
for i in 0..=MAX_ARGS {
let Some(p) = self.read(addr + 8 * i as u64, 8).filter(|b| b.len() == 8) else {
return (v, true);
};
let ptr = u64::from_ne_bytes(p[..8].try_into().expect("8 bytes"));
if ptr == 0 {
return (v, false);
}
if i == MAX_ARGS {
return (v, true);
}
match self.cstr(ptr) {
Some(s) => v.push(s),
None => return (v, true),
}
}
(v, true)
}
}
/// `argv0` itself when it holds a `/`, else the first executable file of
/// that name on the PATH (the `--env` PATH, else the tracer's).
fn find_program(argv0: &OsStr, env: &[(OsString, OsString)]) -> Option<PathBuf> {
use std::os::unix::fs::PermissionsExt;
if argv0.as_bytes().contains(&b'/') {
return Some(PathBuf::from(argv0));
}
let path = env
.iter()
.rev()
.find(|(k, _)| k == "PATH")
.map(|(_, v)| v.clone())
.or_else(|| std::env::var_os("PATH"))
.unwrap_or_else(|| "/usr/local/bin:/usr/bin:/bin".into());
std::env::split_paths(&path)
.map(|d| d.join(argv0))
.find(|p| {
std::fs::metadata(p)
.map(|m| m.is_file() && m.permissions().mode() & 0o111 != 0)
.unwrap_or(false)
})
}
fn describe(st: libc::c_int) -> String {
if libc::WIFEXITED(st) {
Exit::Code(libc::WEXITSTATUS(st)).to_string()
} else {
Exit::Signal {
signal: libc::WTERMSIG(st),
core: libc::WCOREDUMP(st),
}
.to_string()
}
}
/// SIGKILL to every process of the pid namespace but the tracer (PID 1).
fn kill_all() {
// SAFETY: kill(-1) from PID 1 signals every other process of its pid
// namespace; the tracer itself is never included.
unsafe { libc::kill(-1, libc::SIGKILL) };
}
/// Reap whatever has ended, without waiting.
fn reap_now() -> Vec<(libc::pid_t, libc::c_int)> {
let mut v = Vec::new();
loop {
let mut st = 0;
// SAFETY: waitpid on any child, no hang.
let r = unsafe { libc::waitpid(-1, &mut st, libc::WNOHANG) };
if r <= 0 {
return v;
}
v.push((r, st));
}
}
/// Reap until no child is left or `until`: (how many, the ended ones).
fn reap_all(until: Instant) -> (u64, Vec<(libc::pid_t, libc::c_int)>) {
let mut all = Vec::new();
loop {
let mut st = 0;
// SAFETY: waitpid on any child, no hang.
let r = unsafe { libc::waitpid(-1, &mut st, libc::WNOHANG) };
if r > 0 {
all.push((r, st));
continue;
}
if r < 0 || Instant::now() >= until {
// ECHILD: nothing left.
return (all.len() as u64, all);
}
kill_all();
std::thread::sleep(Duration::from_millis(5));
}
}