crates/heimdall-cli/src/profile.rs
// SPDX-License-Identifier: AGPL-3.0-only
//! `heimdall profile probe|gen|check <workload>` (spec §3.1, §7): the
//! syscall profile verbs, over `profiles/<workload>/workload.toml`.
//! **Heimdall is a defensive sandbox.** (Module of lane m3; heimdall-cli
//! itself belongs to lane api02.)
//!
//! - `probe`: run the corpus in learning mode, write `<arch>-<libc>.probe.json`.
//! - `gen`: from the probe file write `<arch>-<libc>.rs`, `.json`, `.oci.json`.
//! - `check`: probe again and fail (exit 1) when a call is outside the
//! checked-in profile, naming the call and the test that made it, or when
//! `.rs`/`.oci.json` do not match `.json`.
use std::path::PathBuf;
use clap::{Parser, Subcommand};
use heimdall_probe::{check_workload, gen_and_write, probe_and_write, ProbeOptions, Workload};
use heimdall_seccomp::{Arch, Libc};
use crate::{EXIT_HOST, EXIT_OK, EXIT_REJECTED, EXIT_USAGE};
#[derive(Parser, Debug)]
#[command(
name = "heimdall profile",
about = "Heimdall is a defensive sandbox: probe a workload, generate its syscall profile, check it for drift."
)]
struct ProfileCli {
#[command(subcommand)]
verb: Verb,
}
#[derive(Subcommand, Debug)]
enum Verb {
/// Run the workload's corpus in learning mode; write <arch>-<libc>.probe.json.
Probe(Common),
/// Generate <arch>-<libc>.rs, .json and .oci.json from the probe file.
Gen(Common),
/// Probe again; exit 1 when a call is outside the checked-in profile.
Check(Common),
/// Probe ONE program where it really runs (a daemon in a booted machine, a
/// plugin its host executes): stdio, environment, cwd and argv0 are its
/// own; what it calls is merged into --out; its exit is this command's.
#[command(name = "probe-exec")]
ProbeExec(ProbeExec),
/// Probe ONE program inside Heimdall's own protected container (a native
/// job: namespaces, empty netns, read-only root pinned by digest,
/// Landlock, cgroups) while a driver outside drives it, e.g. through
/// unix sockets in --channel. The seccomp layer is the LEARNING filter
/// (log and allow) instead of an enforcing profile; nothing is signed.
/// What it calls is merged into --out; its exit is this command's.
#[command(name = "probe-container")]
ProbeContainer(Box<ProbeContainer>),
/// Merge evidence files (probe-exec --out, probe) into one.
Merge(MergeArgs),
}
#[derive(clap::Args, Debug)]
struct ProbeContainer {
/// The evidence file (merged into, created when missing).
#[arg(long)]
out: PathBuf,
/// The workload the evidence belongs to.
#[arg(long)]
workload: String,
/// The run name every call is recorded under (e.g. tpcds:q1-q99).
#[arg(long)]
run: String,
/// The run drives an error path.
#[arg(long)]
error_path: bool,
/// A local OCI image layout (index.json, blobs/sha256/...) holding the image.
#[arg(long)]
image: PathBuf,
/// The image manifest's digest, sha256:<64 hex>; verified at the start.
#[arg(long)]
digest: String,
/// A host dir mounted read-write at /run/heimdall-channel (unix sockets
/// the program makes there are reachable from outside). Default: none.
#[arg(long)]
channel: Option<PathBuf>,
/// KEY=VALUE added to the image's environment; may repeat.
#[arg(long = "env", value_parser = parse_env)]
env: Vec<String>,
/// The program's libc (glibc, musl); default: read from its ELF in the image.
#[arg(long, value_parser = parse_libc)]
libc: Option<Libc>,
/// The wall-clock limit; the job is killed when it is reached.
#[arg(long, default_value_t = 86_400)]
wall_secs: u64,
/// The cgroup memory limit (swap 0; /tmp is counted in it).
#[arg(long, default_value_t = 1024)]
memory_mib: u64,
/// The cgroup pids limit.
#[arg(long, default_value_t = 256)]
pids: u32,
/// The CPU quota in CPUs.
#[arg(long, default_value_t = 1.0)]
cpus: f32,
/// The size of the /tmp tmpfs (default: a normal job's 64 MiB).
#[arg(long)]
tmp_mib: Option<u64>,
/// The uid and gid the program runs as inside (a nested user namespace
/// maps it to the job's root, i.e. your uid; default 0). For programs
/// that refuse root, e.g. postgres.
#[arg(long, default_value_t = 0)]
uid: u32,
/// After SIGTERM/SIGINT/SIGQUIT: how long the program may take before it is killed.
#[arg(long, default_value_t = 10)]
term_grace_secs: u64,
/// The shortest time between two rewrites of --out while the program runs.
#[arg(long, default_value_t = 5)]
flush_secs: u64,
/// Append the program's stdout and stderr to this host file (default: /dev/null).
#[arg(long)]
log: Option<PathBuf>,
/// The heimdall-container-shim binary (default: $HEIMDALL_CONTAINER_SHIM,
/// else next to this binary).
#[arg(long)]
shim: Option<PathBuf>,
/// Bundles and runtime state (default: $XDG_RUNTIME_DIR/heimdall, else
/// /tmp/heimdall-<uid>).
#[arg(long)]
work_dir: Option<PathBuf>,
/// The cgroup v2 mount.
#[arg(long, default_value = "/sys/fs/cgroup")]
cgroup_root: PathBuf,
/// Jobs' cgroups go below <cgroup-root>/<cgroup-parent>/ (delegated to you).
#[arg(long, default_value = "heimdall")]
cgroup_parent: String,
/// The program, then its arguments (argv inside the image).
#[arg(required = true, trailing_var_arg = true, allow_hyphen_values = true)]
command: Vec<String>,
}
fn parse_env(s: &str) -> Result<String, String> {
match s.split_once('=') {
Some((k, _)) if !k.is_empty() && !k.contains('\0') => Ok(s.to_string()),
_ => Err(format!("{s:?} is not KEY=VALUE")),
}
}
#[derive(clap::Args, Debug)]
struct ProbeExec {
/// The evidence file (merged into, created when missing).
#[arg(long)]
out: PathBuf,
/// The workload the evidence belongs to.
#[arg(long)]
workload: String,
/// The run name every call is recorded under (e.g. kvm:cp:rusternetes).
#[arg(long)]
run: String,
/// The program's libc (glibc, musl).
#[arg(long, value_parser = parse_libc)]
libc: Libc,
/// argv[0] for the program (default: the program path).
#[arg(long)]
arg0: Option<String>,
/// The run drives an error path.
#[arg(long)]
error_path: bool,
/// Record only the program's own process, not the processes it starts.
#[arg(long)]
self_only: bool,
/// The shortest time between two rewrites of --out while the program runs.
#[arg(long, default_value_t = 5)]
flush_secs: u64,
/// The program, then its arguments.
#[arg(required = true, trailing_var_arg = true, allow_hyphen_values = true)]
command: Vec<std::ffi::OsString>,
}
#[derive(clap::Args, Debug)]
struct MergeArgs {
/// The merged file.
#[arg(long)]
out: PathBuf,
/// The workload name of the merged file (default: the first input's).
#[arg(long)]
workload: Option<String>,
/// Evidence files of one arch and libc.
#[arg(required = true)]
inputs: Vec<PathBuf>,
}
#[derive(clap::Args, Debug)]
struct Common {
/// The workload: a dir under --profiles holding workload.toml.
workload: String,
/// The profiles dir.
#[arg(long, default_value = "profiles")]
profiles: PathBuf,
/// The libc (glibc, musl): picks the binary in workload.toml [binary].
#[arg(long, value_parser = parse_libc)]
libc: Option<Libc>,
/// Probe this binary instead of the one in workload.toml.
#[arg(long)]
binary: Option<PathBuf>,
/// Only the runs whose name contains this; may repeat (probe, check).
#[arg(long = "only")]
only: Vec<String>,
}
fn parse_libc(s: &str) -> Result<Libc, String> {
s.parse()
}
/// Entry from `heimdall profile ...` with the words after `profile`.
pub fn main(rest: Vec<String>) -> u8 {
let cli = match ProfileCli::try_parse_from(
std::iter::once("heimdall profile".to_string()).chain(rest),
) {
Ok(c) => c,
Err(e) => {
let _ = e.print();
return match e.kind() {
clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion => {
EXIT_OK
}
_ => EXIT_USAGE,
};
}
};
let (verb, c) = match cli.verb {
Verb::Probe(c) => ("probe", c),
Verb::Gen(c) => ("gen", c),
Verb::Check(c) => ("check", c),
Verb::ProbeExec(p) => return probe_exec(p),
Verb::ProbeContainer(p) => return probe_container(*p),
Verb::Merge(m) => return merge(m),
};
let w = match Workload::load(&c.profiles, &c.workload) {
Ok(w) => w,
Err(e) => {
eprintln!("heimdall profile {verb}: {e}");
return EXIT_HOST;
}
};
let opts = ProbeOptions {
libc: c.libc,
binary: c.binary.clone(),
only: c.only.clone(),
};
match verb {
"probe" => probe(&w, &opts),
"gen" => gen(&w, c.libc),
_ => check(&w, &opts),
}
}
fn probe_exec(p: ProbeExec) -> u8 {
let mut command = p.command.into_iter();
let Some(program) = command.next() else {
eprintln!("heimdall profile probe-exec: name the program");
return EXIT_USAGE;
};
let run = heimdall_probe::LiveRun {
workload: p.workload,
run: p.run,
error_path: p.error_path,
program: PathBuf::from(program),
arg0: p.arg0.map(Into::into),
args: command.collect(),
env: Vec::new(),
libc: p.libc,
out: p.out,
flush: std::time::Duration::from_secs(p.flush_secs),
self_only: p.self_only,
};
match heimdall_probe::learn_live(&run) {
// The program's own exit, so a host that runs it through the probe sees no difference.
Ok(heimdall_seccomp::launch::Exit::Code(c)) => u8::try_from(c & 0xff).unwrap_or(1),
Ok(heimdall_seccomp::launch::Exit::Signal { signal, .. }) => {
// SAFETY: back to the default disposition, then the same signal to this process.
unsafe {
libc::signal(signal, libc::SIG_DFL);
libc::kill(libc::getpid(), signal);
}
u8::try_from(128 + signal).unwrap_or(1)
}
Err(e) => {
eprintln!(
"heimdall profile probe-exec: {}: {e}",
run.program.display()
);
EXIT_HOST
}
}
}
/// Signals `probe-container` got (bit `1 << signo`), handed on to the program.
static SIGNALS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
extern "C" fn caught(sig: libc::c_int) {
if (0..64).contains(&sig) {
SIGNALS.fetch_or(1u64 << sig, std::sync::atomic::Ordering::SeqCst);
}
}
/// `$XDG_RUNTIME_DIR/heimdall`, else `/tmp/heimdall-<uid>`.
fn default_work_dir() -> PathBuf {
match std::env::var_os("XDG_RUNTIME_DIR") {
Some(d) if !d.is_empty() => PathBuf::from(d).join("heimdall"),
// SAFETY: getuid has no failure mode and takes no pointers.
_ => std::env::temp_dir().join(format!("heimdall-{}", unsafe { libc::getuid() })),
}
}
/// One stdout status line (a JSON object), flushed at once.
fn status(v: serde_json::Value) {
use std::io::Write as _;
let mut out = std::io::stdout().lock();
let _ = writeln!(out, "{v}");
let _ = out.flush();
}
fn probe_container(p: ProbeContainer) -> u8 {
use heimdall_container::learn::{LearnJob, LearnSpec, Started, CHANNEL_DIR, FORWARDED};
use heimdall_core::{ImageRef, Job, JobKind, Policy};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
const ME: &str = "heimdall profile probe-container";
let image = ImageRef {
name: p.image.display().to_string(),
digest: p.digest.clone(),
};
if image.sha256().is_none() {
eprintln!(
"{ME}: --digest {} is not sha256:<64 lower-case hex>",
p.digest
);
return EXIT_USAGE;
}
let Some(arch) = Arch::native() else {
eprintln!("{ME}: this arch is not supported");
return EXIT_HOST;
};
let work = p.work_dir.clone().unwrap_or_else(default_work_dir);
{
use std::os::unix::fs::DirBuilderExt;
if let Err(e) = std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(&work)
{
eprintln!("{ME}: work dir {}: {e}", work.display());
return EXIT_HOST;
}
}
let mut cfg = heimdall_container::HeimdallConfig::new(&p.image, &work);
if let Some(s) = &p.shim {
cfg.shim = heimdall_container::ShimCommand {
program: s.clone(),
args: vec![],
};
}
cfg.cgroup_root = p.cgroup_root.clone();
cfg.cgroup_parent = p.cgroup_parent.clone();
let runner = match heimdall_container::HeimdallRunner::new(cfg) {
Ok(r) => r,
Err(e) => {
eprintln!("{ME}: {e}");
return EXIT_HOST;
}
};
// The job's own (empty) /in and /out.
let dirs = work.join(format!("probe-{}", heimdall_core::JobId::random()));
let (input, output) = (dirs.join("in"), dirs.join("out"));
if let Err(e) = std::fs::create_dir_all(&input).and_then(|()| std::fs::create_dir_all(&output))
{
eprintln!("{ME}: {}: {e}", dirs.display());
return EXIT_HOST;
}
let tmp_bytes = p.tmp_mib.map(|m| m.saturating_mul(1 << 20));
let mut policy = Policy::default();
let l = &mut policy.limits;
l.wall = Duration::from_secs(p.wall_secs.max(1));
l.memory_bytes = p.memory_mib.saturating_mul(1 << 20);
l.pids = p.pids;
l.cpus = p.cpus;
// RLIMIT_FSIZE covers every file the program writes (/tmp included):
// the tmpfs size bounds them, not a job's output cap.
l.max_output_file_bytes = tmp_bytes.unwrap_or(64 << 20).max(64 << 20);
let job = Job::new(
JobKind::Native {
image,
argv: p.command.clone(),
},
input,
output,
serde_json::Value::Null,
policy,
);
let lj = LearnJob {
job,
env: p.env.clone(),
spec: LearnSpec {
channel: p.channel.clone(),
tmp_bytes,
uid: p.uid,
},
libc: p.libc,
term_grace: Duration::from_secs(p.term_grace_secs),
log: p.log.clone(),
};
let ev = heimdall_probe::Evidence {
workload: p.workload.clone(),
run: p.run.clone(),
error_path: p.error_path,
out: p.out.clone(),
flush: Duration::from_secs(p.flush_secs),
};
for sig in FORWARDED {
// SAFETY: a handler that only does an atomic fetch_or.
unsafe { libc::signal(sig, caught as *const () as libc::sighandler_t) };
}
let done = Arc::new(AtomicBool::new(false));
type Recorded = (heimdall_probe::Observations, std::io::Result<()>);
let mut recorder: Option<std::thread::JoinHandle<Recorded>> = None;
let channel = p.channel.is_some();
let r = runner.learn(lj, &SIGNALS, |s: Started| {
status(serde_json::json!({
"event": "started",
"pid": s.pid,
"seccomp": "learning",
"net": "empty",
"channel": if channel { Some(CHANNEL_DIR) } else { None },
"libc": s.libc.as_str(),
}));
let (ev, done) = (ev.clone(), done.clone());
recorder = Some(std::thread::spawn(move || {
let mut obs = heimdall_probe::Observations::new(&ev.workload, arch, s.libc);
let l = heimdall_seccomp::notif::Listener::from_fd(s.listener);
let r = heimdall_probe::record(&l, &ev, &mut obs, &|_| true, &done);
(obs, r)
}));
});
done.store(true, Ordering::SeqCst);
let recorded = recorder.map(|h| h.join().expect("the recorder never panics"));
let _ = std::fs::remove_dir_all(&dirs);
let o = match r {
Ok(o) => o,
Err(e) => {
if let Some((obs, _)) = &recorded {
let _ = ev.write_merged(obs, "heimdall error");
}
eprintln!("{ME}: {e}");
return if matches!(e, heimdall_core::RunError::InvalidJob { .. }) {
EXIT_USAGE
} else {
EXIT_HOST
};
}
};
let Some((obs, rec)) = recorded else {
eprintln!("{ME}: the program never started under the learning filter");
return EXIT_HOST;
};
let exit = match (o.exit_code, o.signal) {
(Some(c), _) => heimdall_seccomp::launch::Exit::Code(c),
(None, Some(signal)) => heimdall_seccomp::launch::Exit::Signal {
signal,
core: false,
},
(None, None) => {
eprintln!("{ME}: the program left no exit status");
return EXIT_HOST;
}
};
if let Err(e) = ev.write_merged(&obs, &exit.to_string()) {
eprintln!("{ME}: {}: {e}", ev.out.display());
return EXIT_HOST;
}
if let Err(e) = &rec {
eprintln!("{ME}: recording stopped early: {e}");
}
status(serde_json::json!({
"event": "exited",
"exit": exit.to_string(),
"calls": obs.calls.len(),
"unknown": obs.unknown.len(),
"out": ev.out,
"seccomp": "learning",
"landlock": o.landlock.map(|l| match l {
heimdall_container::confine::LandlockApplied::Full => "full",
heimdall_container::confine::LandlockApplied::Partial => "partial",
}),
"term_killed": o.term_killed,
"wall_killed": o.wall_killed,
"oom_kills": o.oom_kills,
}));
if rec.is_err() {
return EXIT_HOST;
}
match exit {
// The program's own exit, as probe-exec does.
heimdall_seccomp::launch::Exit::Code(c) => u8::try_from(c & 0xff).unwrap_or(1),
heimdall_seccomp::launch::Exit::Signal { signal, .. } => {
// SAFETY: back to the default disposition, then the same signal to this process.
unsafe {
libc::signal(signal, libc::SIG_DFL);
libc::kill(libc::getpid(), signal);
}
u8::try_from(128 + signal).unwrap_or(1)
}
}
}
fn merge(m: MergeArgs) -> u8 {
let mut all: Option<heimdall_probe::Observations> = None;
for p in &m.inputs {
let o = match std::fs::read_to_string(p)
.map_err(|e| e.to_string())
.and_then(|t| heimdall_probe::Observations::from_json(&t))
{
Ok(o) => o,
Err(e) => {
eprintln!("heimdall profile merge: {}: {e}", p.display());
return EXIT_HOST;
}
};
match &mut all {
None => all = Some(o),
Some(a) if (a.arch, a.libc) != (o.arch, o.libc) => {
eprintln!(
"heimdall profile merge: {}: {} {}, the others are {} {}",
p.display(),
o.arch,
o.libc.as_str(),
a.arch,
a.libc.as_str()
);
return EXIT_USAGE;
}
Some(a) => a.merge(&o),
}
}
let Some(mut all) = all else {
return EXIT_USAGE;
};
if let Some(w) = m.workload {
all.workload = w;
}
match std::fs::write(&m.out, all.to_json()) {
Ok(()) => {
println!(
"merge: {} file(s), {} run(s), {} distinct call(s) -> {}",
m.inputs.len(),
all.runs.len(),
all.calls.len(),
m.out.display()
);
EXIT_OK
}
Err(e) => {
eprintln!("heimdall profile merge: {}: {e}", m.out.display());
EXIT_HOST
}
}
}
fn probe(w: &Workload, opts: &ProbeOptions) -> u8 {
match probe_and_write(w, opts) {
Ok((obs, path)) => {
println!(
"probe: {} run(s), {} distinct call(s), {} unknown -> {}",
obs.runs.len(),
obs.calls.len(),
obs.unknown.len(),
path.display()
);
EXIT_OK
}
Err(e) => {
eprintln!("heimdall profile probe: {e}");
EXIT_HOST
}
}
}
fn gen(w: &Workload, libc: Option<Libc>) -> u8 {
let Some(arch) = Arch::native() else {
eprintln!("heimdall profile gen: this arch is not supported");
return EXIT_HOST;
};
let libc = match libc {
Some(l) => l,
None => {
let found: Vec<Libc> = [Libc::Glibc, Libc::Musl]
.into_iter()
.filter(|l| w.paths(arch, *l).probe.exists())
.collect();
match found.as_slice() {
[l] => *l,
[] => {
eprintln!(
"heimdall profile gen: no {arch}-*.probe.json in {}: run `heimdall profile probe {}` first",
w.dir.display(),
w.manifest.workload
);
return EXIT_HOST;
}
_ => {
eprintln!("heimdall profile gen: probe files for both libcs: pass --libc");
return EXIT_USAGE;
}
}
}
};
match gen_and_write(w, arch, libc) {
Ok((g, p)) => {
println!(
"gen: {} rule(s) for {} on {arch} {libc}",
g.profile.rules.len(),
g.profile.workload
);
for f in [&p.rs, &p.json, &p.oci] {
println!(" wrote {}", f.display());
}
let r = &g.review;
println!(
"review: {} near-miss, {} W|X left out, {} bare, {} unknown; commit only after a person reviewed the diff",
r.near_miss.len(),
r.wx.len(),
r.bare.len(),
r.unknown.len()
);
EXIT_OK
}
Err(e) => {
eprintln!("heimdall profile gen: {e}");
EXIT_HOST
}
}
}
fn check(w: &Workload, opts: &ProbeOptions) -> u8 {
match check_workload(w, opts) {
Ok(r) => {
for v in &r.violations {
eprintln!("heimdall profile check: {v}");
}
for p in &r.out_of_sync {
eprintln!(
"heimdall profile check: {} does not match {} (hand-edited or stale)",
p.display(),
r.paths.json.display()
);
}
if r.ok() {
println!(
"check: OK, every call is in {} ({} near-miss)",
r.paths.json.display(),
r.near_miss
);
EXIT_OK
} else {
eprintln!(
"fix: heimdall profile probe {0} && heimdall profile gen {0}, then review the diff (never widen a profile by hand)",
w.manifest.workload
);
EXIT_REJECTED
}
}
Err(e) => {
eprintln!("heimdall profile check: {e}");
EXIT_HOST
}
}
}