crates/heimdall-broker/src/bin/heimdall-broker-selftest.rs
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Ignalina ApS and Vetra Automation AB
//! heimdall-broker-selftest: the worker the broker's kernel tests run under
//! the broker. Heimdall is a defensive sandbox; every scenario here attacks
//! only the broker that supervises this very process.
//!
//! Each scenario prints one line on stdout: `ok ...` or `err <errno>`.
//! `heimdall-broker-selftest probe` prints what this kernel gives the broker.
use std::ffi::CString;
use std::io::{Read, Write};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Instant;
fn errno() -> i32 {
std::io::Error::last_os_error().raw_os_error().unwrap_or(0)
}
fn cstr(s: &str) -> CString {
CString::new(s).expect("no NUL in argument")
}
/// Raw openat (never through std's path handling).
fn openat(path: &str, flags: i32, mode: u32) -> Result<i32, i32> {
let c = cstr(path);
// SAFETY: valid C string.
let fd = unsafe { libc::openat(libc::AT_FDCWD, c.as_ptr(), flags, mode) };
if fd < 0 {
Err(errno())
} else {
Ok(fd)
}
}
fn read_fd(fd: i32) -> Vec<u8> {
let mut out = Vec::new();
let mut buf = [0u8; 256];
loop {
// SAFETY: reading into our buffer.
let n = unsafe { libc::read(fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) };
if n <= 0 {
break;
}
out.extend_from_slice(&buf[..n as usize]);
if out.len() > 4096 {
break;
}
}
out
}
fn close(fd: i32) {
// SAFETY: closing our own fd.
unsafe {
libc::close(fd);
}
}
fn print_result(r: Result<String, i32>) {
match r {
Ok(s) => println!("ok {s}"),
Err(e) => println!("err {e}"),
}
}
fn open_scenario(path: &str, how: &str) -> Result<String, i32> {
let flags = match how {
"r" => libc::O_RDONLY | libc::O_CLOEXEC,
"w" => libc::O_WRONLY | libc::O_CLOEXEC,
"c" => libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC,
"x" => libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_CLOEXEC,
"rw" => libc::O_RDWR | libc::O_CLOEXEC,
"d" => libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC,
"nofollow" => libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_CLOEXEC,
"trunc" => libc::O_RDONLY | libc::O_TRUNC | libc::O_CLOEXEC,
other => panic!("open mode {other}"),
};
let fd = openat(path, flags, 0o644)?;
let res = match how {
"r" | "nofollow" => {
let data = read_fd(fd);
format!("fd={fd} {}", String::from_utf8_lossy(&data).trim_end())
}
"w" | "c" | "x" | "rw" => {
let msg = b"written by the worker\n";
// SAFETY: writing our buffer.
let n = unsafe { libc::write(fd, msg.as_ptr() as *const libc::c_void, msg.len()) };
format!("fd={fd} wrote={n}")
}
"d" => {
// A usable directory fd: fstat it.
// SAFETY: zeroed stat filled by fstat.
let mut st: libc::stat = unsafe { std::mem::zeroed() };
// SAFETY: as above.
let r = unsafe { libc::fstat(fd, &mut st) };
format!(
"fd={fd} dir={}",
r == 0 && st.st_mode & libc::S_IFMT == libc::S_IFDIR
)
}
_ => format!("fd={fd}"),
};
close(fd);
Ok(res)
}
fn raw_syscall_open(path: &str) -> Result<String, i32> {
let c = cstr(path);
#[cfg(target_arch = "x86_64")]
// SAFETY: plain system call with a valid C string.
let fd = unsafe { libc::syscall(libc::SYS_open, c.as_ptr(), libc::O_RDONLY) } as i32;
#[cfg(not(target_arch = "x86_64"))]
// SAFETY: plain system call with a valid C string.
let fd = unsafe { libc::syscall(libc::SYS_openat, libc::AT_FDCWD, c.as_ptr(), libc::O_RDONLY) }
as i32;
if fd < 0 {
return Err(errno());
}
let d = read_fd(fd);
close(fd);
Ok(format!(
"fd={fd} {}",
String::from_utf8_lossy(&d).trim_end()
))
}
fn openat2_scenario(path: &str) -> Result<String, i32> {
#[repr(C)]
struct How {
flags: u64,
mode: u64,
resolve: u64,
}
let how = How {
flags: (libc::O_RDONLY | libc::O_CLOEXEC) as u64,
mode: 0,
resolve: 0,
};
let c = cstr(path);
// SAFETY: valid string and struct.
let fd = unsafe {
libc::syscall(
libc::SYS_openat2,
libc::AT_FDCWD,
c.as_ptr(),
&how as *const How,
24usize,
)
} as i32;
if fd < 0 {
return Err(errno());
}
let d = read_fd(fd);
close(fd);
Ok(format!(
"fd={fd} {}",
String::from_utf8_lossy(&d).trim_end()
))
}
/// Thread A opens `good` while thread B flips the very buffer between `good`
/// and `bad` (same length). Counts what A actually got.
fn race(good: &str, bad: &str, iters: usize) -> Result<String, i32> {
assert_eq!(
good.len(),
bad.len(),
"race paths must have the same length"
);
let n = good.len();
let buf: Arc<Vec<std::sync::atomic::AtomicU8>> = Arc::new(
(0..n + 1)
.map(|_| std::sync::atomic::AtomicU8::new(0))
.collect(),
);
for (i, b) in good.bytes().enumerate() {
buf[i].store(b, Ordering::Relaxed);
}
let stop = Arc::new(AtomicBool::new(false));
let flipper = {
let buf = Arc::clone(&buf);
let stop = Arc::clone(&stop);
let (g, b) = (good.as_bytes().to_vec(), bad.as_bytes().to_vec());
std::thread::spawn(move || {
let mut flip = false;
while !stop.load(Ordering::Relaxed) {
let src = if flip { &g } else { &b };
for (i, c) in src.iter().enumerate() {
buf[i].store(*c, Ordering::Relaxed);
}
flip = !flip;
}
})
};
let (mut good_n, mut bad_n, mut refused, mut other) = (0u64, 0u64, 0u64, 0u64);
let ptr = buf.as_ptr() as *const libc::c_char;
for _ in 0..iters {
// SAFETY: the buffer lives (Arc) and is NUL-terminated at n.
let fd = unsafe { libc::openat(libc::AT_FDCWD, ptr, libc::O_RDONLY | libc::O_CLOEXEC) };
if fd < 0 {
if errno() == libc::EACCES {
refused += 1;
} else {
other += 1;
}
continue;
}
let d = read_fd(fd);
close(fd);
match d.as_slice() {
b"GOOD" => good_n += 1,
b"BAD" => bad_n += 1,
_ => other += 1,
}
}
stop.store(true, Ordering::Relaxed);
let _ = flipper.join();
Ok(format!(
"good={good_n} bad={bad_n} refused={refused} other={other}"
))
}
fn connect(addr: &str) -> Result<String, i32> {
let a: std::net::SocketAddr = addr.parse().expect("socket address");
match std::net::TcpStream::connect(a) {
Ok(mut s) => {
let mut line = String::new();
let _ = s.read_to_string(&mut line);
Ok(format!("connected {}", line.trim_end()))
}
Err(e) => Err(e.raw_os_error().unwrap_or(0)),
}
}
/// connect() with a sockaddr the worker keeps rewriting after the call
/// started (allowed <-> forbidden): the broker must connect only where it
/// checked.
fn connect_race(allowed: &str, forbidden: &str, iters: usize) -> Result<String, i32> {
let a: std::net::SocketAddrV4 = allowed.parse().expect("v4 address");
let f: std::net::SocketAddrV4 = forbidden.parse().expect("v4 address");
let mk = |x: std::net::SocketAddrV4| {
// SAFETY: all-zero is a valid sockaddr_in.
let mut s: libc::sockaddr_in = unsafe { std::mem::zeroed() };
s.sin_family = libc::AF_INET as u16;
s.sin_port = x.port().to_be();
s.sin_addr.s_addr = u32::from_ne_bytes(x.ip().octets());
s
};
let (sa, sf) = (mk(a), mk(f));
// Leaked: the flipper thread writes through a raw pointer until joined.
let shared: &'static std::cell::UnsafeCell<libc::sockaddr_in> =
Box::leak(Box::new(std::cell::UnsafeCell::new(sa)));
struct P(*mut libc::sockaddr_in);
unsafe impl Send for P {}
let stop = Arc::new(AtomicBool::new(false));
let p = P(shared.get());
let st = Arc::clone(&stop);
let flipper = std::thread::spawn(move || {
let p = p;
let mut flip = false;
while !st.load(Ordering::Relaxed) {
// SAFETY: deliberate racy writes into the worker's own buffer.
unsafe { std::ptr::write_volatile(p.0, if flip { sa } else { sf }) };
flip = !flip;
}
});
let (mut ok_allowed, mut ok_forbidden, mut refused, mut other) = (0, 0, 0, 0);
for _ in 0..iters {
// SAFETY: plain socket calls on our own fds.
unsafe {
let s = libc::socket(libc::AF_INET, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0);
if s < 0 {
other += 1;
continue;
}
let r = libc::connect(
s,
shared.get() as *const libc::sockaddr,
std::mem::size_of::<libc::sockaddr_in>() as u32,
);
if r == 0 {
let d = read_fd(s);
match d.as_slice() {
b"ALLOWED" => ok_allowed += 1,
b"FORBIDDEN" => ok_forbidden += 1,
_ => other += 1,
}
} else if errno() == libc::EACCES {
refused += 1;
} else {
other += 1;
}
libc::close(s);
}
}
stop.store(true, Ordering::Relaxed);
let _ = flipper.join();
Ok(format!(
"allowed={ok_allowed} forbidden={ok_forbidden} refused={refused} other={other}"
))
}
fn socket_scenario(kind: &str) -> Result<String, i32> {
let (d, t) = match kind {
"unix" => (libc::AF_UNIX, libc::SOCK_STREAM),
"udp" => (libc::AF_INET, libc::SOCK_DGRAM),
"tcp4" => (libc::AF_INET, libc::SOCK_STREAM),
"tcp6" => (libc::AF_INET6, libc::SOCK_STREAM),
"raw" => (libc::AF_INET, libc::SOCK_RAW),
"netlink" => (libc::AF_NETLINK, libc::SOCK_RAW),
other => panic!("socket kind {other}"),
};
// SAFETY: plain system call.
let s = unsafe { libc::socket(d, t | libc::SOCK_CLOEXEC, 0) };
if s < 0 {
return Err(errno());
}
close(s);
Ok(format!("fd={s}"))
}
fn bind_scenario(addr: &str) -> Result<String, i32> {
std::net::TcpListener::bind(addr)
.map(|_| "bound".to_string())
.map_err(|e| e.raw_os_error().unwrap_or(0))
}
fn listen_scenario() -> Result<String, i32> {
// SAFETY: plain system calls.
unsafe {
let s = libc::socket(libc::AF_INET, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0);
if s < 0 {
return Err(errno());
}
let r = libc::listen(s, 1);
let e = errno();
close(s);
if r < 0 {
Err(e)
} else {
Ok("listening".into())
}
}
}
fn fastopen(addr: &str) -> Result<String, i32> {
let a: std::net::SocketAddrV4 = addr.parse().expect("v4 address");
// SAFETY: plain system calls.
unsafe {
let s = libc::socket(libc::AF_INET, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0);
if s < 0 {
return Err(errno());
}
let mut sa: libc::sockaddr_in = std::mem::zeroed();
sa.sin_family = libc::AF_INET as u16;
sa.sin_port = a.port().to_be();
sa.sin_addr.s_addr = u32::from_ne_bytes(a.ip().octets());
let msg = b"hi";
let r = libc::sendto(
s,
msg.as_ptr() as *const libc::c_void,
msg.len(),
libc::MSG_FASTOPEN,
&sa as *const _ as *const libc::sockaddr,
std::mem::size_of::<libc::sockaddr_in>() as u32,
);
let e = errno();
close(s);
if r < 0 {
Err(e)
} else {
Ok(format!("sent={r}"))
}
}
}
fn exec_scenario(path: &str, at: bool) -> Result<String, i32> {
let c = cstr(path);
let argv = [c.as_ptr(), std::ptr::null()];
let envp = [std::ptr::null::<libc::c_char>()];
// SAFETY: valid argv/envp arrays.
unsafe {
if at {
libc::syscall(
libc::SYS_execveat,
libc::AT_FDCWD,
c.as_ptr(),
argv.as_ptr(),
envp.as_ptr(),
0,
);
} else {
libc::execve(c.as_ptr(), argv.as_ptr(), envp.as_ptr());
}
}
Err(errno())
}
fn uring() -> Result<String, i32> {
let mut params = [0u8; 120];
// SAFETY: io_uring_setup with a zeroed params struct.
let r = unsafe { libc::syscall(libc::SYS_io_uring_setup, 4u32, params.as_mut_ptr()) };
if r < 0 {
Err(errno())
} else {
close(r as i32);
Ok("ring".into())
}
}
fn fork_open(path: &str) -> Result<String, i32> {
// SAFETY: the child makes system calls only and _exits.
let pid = unsafe { libc::fork() };
if pid == 0 {
let c = cstr(path);
// SAFETY: as above.
unsafe {
let fd = libc::openat(libc::AT_FDCWD, c.as_ptr(), libc::O_RDONLY);
let mut b = [0u8; 4];
let ok = fd >= 0
&& libc::read(fd, b.as_mut_ptr() as *mut libc::c_void, 4) == 4
&& &b == b"GOOD";
libc::_exit(if ok { 0 } else { 1 });
}
}
let mut st = 0;
// SAFETY: waiting for our child.
unsafe { libc::waitpid(pid, &mut st, 0) };
if libc::WIFEXITED(st) && libc::WEXITSTATUS(st) == 0 {
Ok("child read GOOD".into())
} else {
Err(libc::EIO)
}
}
fn thread_open(path: &str) -> Result<String, i32> {
let p = path.to_string();
std::thread::spawn(move || open_scenario(&p, "r"))
.join()
.unwrap()
}
/// A thread makes a brokered call; the process exits while it is pending.
fn die_mid_call(path: &str) -> ! {
let p = path.to_string();
std::thread::spawn(move || {
let _ = openat(&p, libc::O_RDONLY, 0);
});
std::thread::sleep(std::time::Duration::from_millis(150));
println!("dying");
let _ = std::io::stdout().flush();
// SAFETY: ends this process.
unsafe { libc::_exit(7) }
}
fn bench(path: &str, n: usize) -> Result<String, i32> {
let t = Instant::now();
for _ in 0..n {
let fd = openat(path, libc::O_RDONLY | libc::O_CLOEXEC, 0)?;
close(fd);
}
let el = t.elapsed();
Ok(format!(
"opens={n} ns_per_open={} opens_per_s={:.0}",
el.as_nanos() / n as u128,
n as f64 / el.as_secs_f64()
))
}
fn fifo(dir: &str) -> Result<String, i32> {
let p = format!("{dir}/fifo");
let c = cstr(&p);
// SAFETY: valid string.
if unsafe { libc::mkfifo(c.as_ptr(), 0o600) } != 0 {
return Err(errno());
}
open_scenario(&p, "r")
}
fn symlink(target: &str, link: &str, open: &str) -> Result<String, i32> {
std::os::unix::fs::symlink(target, link).map_err(|e| e.raw_os_error().unwrap_or(0))?;
open_scenario(open, "r")
}
/// openat2 with the worker's own `RESOLVE_*` flags (decimal).
fn openat2_resolve(path: &str, resolve: u64) -> Result<String, i32> {
let how: [u64; 3] = [(libc::O_RDONLY | libc::O_CLOEXEC) as u64, 0, resolve];
let c = cstr(path);
// SAFETY: valid string and a 24-byte open_how.
let fd = unsafe {
libc::syscall(
libc::SYS_openat2,
libc::AT_FDCWD,
c.as_ptr(),
how.as_ptr(),
24usize,
)
} as i32;
if fd < 0 {
return Err(errno());
}
let d = read_fd(fd);
close(fd);
Ok(format!(
"fd={fd} {}",
String::from_utf8_lossy(&d).trim_end()
))
}
/// Opens `path` until an open fails; prints how many succeeded and the errno.
/// Then closes one and checks the broker still answers.
fn fd_exhaust(path: &str) -> Result<String, i32> {
let lim = libc::rlimit {
rlim_cur: 64,
rlim_max: 64,
};
// SAFETY: lowers this process's own limit.
unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &lim) };
let mut fds = Vec::new();
let e = loop {
match openat(path, libc::O_RDONLY | libc::O_CLOEXEC, 0) {
Ok(fd) => fds.push(fd),
Err(e) => break e,
}
if fds.len() > 100_000 {
break 0;
}
};
let n = fds.len();
if let Some(fd) = fds.pop() {
close(fd);
}
let again = openat(path, libc::O_RDONLY | libc::O_CLOEXEC, 0).is_ok();
for fd in fds {
close(fd);
}
Ok(format!("opened={n} errno={e} again={again}"))
}
/// Opens `path` for writing, then tries to read it back by reopening the
/// magic link /proc/self/fd/<n>.
fn proc_fd_reopen(path: &str) -> Result<String, i32> {
let fd = openat(
path,
libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC,
0o644,
)?;
let msg = b"secret-out\n";
// SAFETY: writing our buffer.
unsafe { libc::write(fd, msg.as_ptr() as *const libc::c_void, msg.len()) };
let r = open_scenario(&format!("/proc/self/fd/{fd}"), "r");
close(fd);
r
}
/// chdir into `dir`, then open `name` relative to the cwd.
fn rel_open(dir: &str, name: &str, how: &str) -> Result<String, i32> {
let c = cstr(dir);
// SAFETY: valid string.
if unsafe { libc::chdir(c.as_ptr()) } != 0 {
return Err(errno());
}
open_scenario(name, how)
}
/// Open `dir` (O_PATH|O_DIRECTORY), then `name` relative to that dirfd.
fn dirfd_open(dir: &str, name: &str) -> Result<String, i32> {
let d = openat(dir, libc::O_PATH | libc::O_DIRECTORY | libc::O_CLOEXEC, 0)?;
let c = cstr(name);
// SAFETY: valid string, our fd.
let fd = unsafe { libc::openat(d, c.as_ptr(), libc::O_RDONLY | libc::O_CLOEXEC) };
let e = errno();
close(d);
if fd < 0 {
return Err(e);
}
let data = read_fd(fd);
close(fd);
Ok(format!(
"fd={fd} {}",
String::from_utf8_lossy(&data).trim_end()
))
}
fn run(args: &[String]) -> Result<String, i32> {
let a = |i: usize| args.get(i).map(String::as_str).unwrap_or("");
match a(0) {
"noop" => Ok("noop".into()),
"open" => open_scenario(a(1), if a(2).is_empty() { "r" } else { a(2) }),
"open-syscall" => raw_syscall_open(a(1)),
"openat2" => openat2_scenario(a(1)),
"openat2-resolve" => openat2_resolve(a(1), a(2).parse().unwrap()),
"fd-exhaust" => fd_exhaust(a(1)),
"proc-fd" => proc_fd_reopen(a(1)),
"rel" => rel_open(a(1), a(2), if a(3).is_empty() { "r" } else { a(3) }),
"dirfd" => dirfd_open(a(1), a(2)),
"race" => race(a(1), a(2), a(3).parse().unwrap()),
"connect" => connect(a(1)),
"connect-race" => connect_race(a(1), a(2), a(3).parse().unwrap()),
"socket" => socket_scenario(a(1)),
"bind" => bind_scenario(a(1)),
"listen" => listen_scenario(),
"fastopen" => fastopen(a(1)),
"exec" => exec_scenario(a(1), false),
"execveat" => exec_scenario(a(1), true),
"uring" => uring(),
"fork-open" => fork_open(a(1)),
"thread-open" => thread_open(a(1)),
"die-mid-call" => die_mid_call(a(1)),
"bench" => bench(a(1), a(2).parse().unwrap()),
"fifo" => fifo(a(1)),
"symlink" => symlink(a(1), a(2), a(3)),
"deny-only" => {
let p = heimdall_broker::filter::program(
heimdall_broker::FilterMode::Deny,
heimdall_broker::ExecPolicy::Allow,
)
.expect("the deny filter builds");
heimdall_seccomp::launch::install(&p, 0).map_err(|e| e.raw_os_error().unwrap_or(0))?;
run(&args[1..])
}
"probe" => {
let s = heimdall_broker::Support::probe();
Ok(format!("{s:?} mode={:?}", s.mode()))
}
"seq" => {
// Several scenarios separated by `--`, one line each.
let mut last = Ok(String::new());
for part in args[1..].split(|x| x == "--") {
last = run(part);
print_result(last.clone());
}
last.map(|_| "seq done".into())
}
other => panic!("unknown scenario {other}"),
}
}
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
print_result(run(&args));
}