server/sandbox/tests/remote_sidecar.rs
//! **The `Remote` engine and its sidecar, end to end** (lane B4, A31 §7.3): the
//! server's [`Sandbox`] with `engine: remote` asks a sidecar over a unix socket,
//! the sidecar ([`remote::serve`]) runs the worker over the job directory the
//! server wrote into the shared directory, and the report comes back.
//!
//! The worker here is a small shell program, so these tests need no image and
//! no container engine: what is tested is the seam (the wire, the shared
//! directory, the mounts copied in, the limits, the refusals by name).
use std::os::unix::fs::PermissionsExt as _;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use holger_sandbox::pids::PidsBound;
use holger_sandbox::remote::{self, Ask, Op, ServeConfig};
use holger_sandbox::{EngineKind, Limits, Mount, Request, Sandbox, SandboxConfig};
/// A worker that echoes its job, the copied YARA rule, and its rlimits.
const ECHO: &str = r#"#!/bin/sh
if [ "$1" = probe ]; then
printf '{"probe":"%s","nnp":"%s","core":"%s","as_kib":"%s","home":"%s"}' "$2" \
"$(grep NoNewPrivs /proc/self/status | tr -dc 0-9)" "$(ulimit -c)" "$(ulimit -v)" "$HOME"
exit 0
fi
printf '{"job":%s,"yara":"%s","artifact":"%s"}' "$(cat "$1/job.json")" "$(cat "$1/d-yara/r.yar")" "$(cat "$1/artifact")"
"#;
struct Side {
_tmp: tempfile::TempDir,
share: PathBuf,
socket: PathBuf,
}
fn program(dir: &Path, name: &str, body: &str) -> PathBuf {
let p = dir.join(name);
std::fs::write(&p, body).unwrap();
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
p
}
/// Start a sidecar over `worker_body` (in a thread of this test process).
fn sidecar(worker_body: &str, ceiling: Limits) -> Side {
let tmp = tempfile::tempdir().unwrap();
let share = tmp.path().join("share");
let scratch = tmp.path().join("scratch");
std::fs::create_dir_all(&share).unwrap();
std::fs::create_dir_all(&scratch).unwrap();
let worker = program(tmp.path(), "worker.sh", worker_body);
let socket = share.join("worker.sock");
let cfg = ServeConfig {
socket: socket.clone(),
share: share.clone(),
worker,
scratch,
ceiling,
max_jobs: 4,
cgroup_fs: None,
refuse_unbounded: false,
pod_pids_limit: None,
};
let listener = remote::bind(&cfg).expect("the sidecar binds its socket");
std::thread::spawn(move || remote::serve(listener, cfg));
Side {
_tmp: tmp,
share,
socket,
}
}
/// Start a sidecar under a pids `bound` already decided (RA5-27); `release` refuses an
/// unbounded one, as a release build does.
fn sidecar_bounded(worker_body: &str, ceiling: Limits, release: bool, bound: PidsBound) -> Side {
let tmp = tempfile::tempdir().unwrap();
let share = tmp.path().join("share");
let scratch = tmp.path().join("scratch");
std::fs::create_dir_all(&share).unwrap();
std::fs::create_dir_all(&scratch).unwrap();
let worker = program(tmp.path(), "worker.sh", worker_body);
let socket = share.join("worker.sock");
let cfg = ServeConfig {
socket: socket.clone(),
share: share.clone(),
worker,
scratch,
ceiling,
max_jobs: 4,
cgroup_fs: None,
refuse_unbounded: release,
pod_pids_limit: None,
};
let listener = remote::bind(&cfg).expect("the sidecar binds its socket");
std::thread::spawn(move || remote::serve_bounded(listener, cfg, bound));
Side {
_tmp: tmp,
share,
socket,
}
}
fn ping() -> Ask {
Ask {
schema: remote::SCHEMA.into(),
op: Op::Ping,
dir: String::new(),
command: vec![],
limits: Limits::default(),
}
}
fn remote_cfg(side: &Side, limits: Limits) -> SandboxConfig {
SandboxConfig {
engine: EngineKind::Remote,
socket: Some(side.socket.display().to_string()),
work_dir: Some(side.share.display().to_string()),
limits,
..Default::default()
}
}
fn run(
sb: &Sandbox,
mounts: Vec<Mount>,
command: Vec<String>,
) -> Result<holger_sandbox::Output, holger_sandbox::Refusal> {
sb.run(&Request {
artifact: b"the-bytes",
job: br#"{"schema":"t"}"#,
mounts,
sha256: "ab12",
command,
})
}
#[test]
fn a_scan_through_the_sidecar_hands_back_the_workers_report_with_its_mounts() {
let side = sidecar(ECHO, Limits::default());
let rules = tempfile::tempdir().unwrap();
std::fs::write(rules.path().join("r.yar"), "rule-text").unwrap();
let sb = Sandbox::from_config(remote_cfg(&side, Limits::default())).unwrap();
assert!(
sb.engine_words().contains("sidecar"),
"{}",
sb.engine_words()
);
sb.ready().expect("the sidecar answers its ping");
let out = run(&sb, vec![Mount::new(rules.path(), "d-yara")], vec![]).expect("the scan runs");
let v: serde_json::Value = serde_json::from_slice(&out.report).unwrap();
assert_eq!(v["job"]["schema"], "t", "the job the server wrote: {v}");
assert_eq!(
v["yara"], "rule-text",
"a mount outside the share is copied into the job directory: {v}"
);
assert_eq!(v["artifact"], "the-bytes", "{v}");
let left: Vec<_> = std::fs::read_dir(&side.share)
.unwrap()
.filter_map(|e| e.ok())
.filter(|e| e.file_name() != "worker.sock")
.collect();
assert!(
left.is_empty(),
"the job directory is removed after the scan: {left:?}"
);
}
#[test]
fn the_worker_runs_with_no_new_privileges_no_core_a_memory_ceiling_and_its_own_home() {
let ceiling = Limits {
memory_mib: 512,
..Default::default()
};
let side = sidecar(ECHO, ceiling);
// The server asks for more than the sidecar allows: the ceiling wins.
let sb = Sandbox::from_config(remote_cfg(
&side,
Limits {
memory_mib: 4096,
..Default::default()
},
))
.unwrap();
let out = run(&sb, vec![], vec!["probe".into(), "p1".into()]).expect("the probe runs");
let v: serde_json::Value = serde_json::from_slice(&out.report).unwrap();
assert_eq!(v["probe"], "p1", "{v}");
assert_eq!(v["nnp"], "1", "no_new_privs is set: {v}");
assert_eq!(v["core"], "0", "no core dumps: {v}");
let as_kib: u64 = v["as_kib"]
.as_str()
.unwrap()
.parse()
.expect("a finite address-space limit");
assert!(
as_kib <= (512 + 128) * 1024,
"the sidecar's ceiling (512 MiB) bounds the ask (4096): {as_kib} KiB"
);
assert!(
!v["home"].as_str().unwrap().is_empty() && v["home"] != "/",
"HOME is the job's scratch: {v}"
);
}
#[test]
fn a_sidecar_that_is_not_there_is_no_engine_and_is_asked_again_once_it_is() {
let tmp = tempfile::tempdir().unwrap();
let share = tmp.path().join("share");
std::fs::create_dir_all(&share).unwrap();
let socket = share.join("worker.sock");
let cfg = SandboxConfig {
engine: EngineKind::Remote,
socket: Some(socket.display().to_string()),
work_dir: Some(share.display().to_string()),
..Default::default()
};
let sb = Sandbox::from_config(cfg).unwrap();
let e = sb.ready().unwrap_err();
assert_eq!(e.code().code, "holger-sandbox-1", "{e}");
assert!(e.to_string().contains("sidecar"), "{e}");
// The sidecar starts after the server (a pod's containers start together):
// readiness is asked again, never remembered as a refusal.
let scratch = tmp.path().join("scratch");
std::fs::create_dir_all(&scratch).unwrap();
let worker = program(tmp.path(), "w.sh", ECHO);
let sc = ServeConfig {
socket,
share,
worker,
scratch,
ceiling: Limits::default(),
max_jobs: 1,
cgroup_fs: None,
refuse_unbounded: false,
pod_pids_limit: None,
};
let l = remote::bind(&sc).unwrap();
std::thread::spawn(move || remote::serve(l, sc));
sb.ready().expect("the sidecar is asked again and answers");
}
#[test]
fn a_worker_past_its_wall_clock_is_a_timeout() {
let side = sidecar("#!/bin/sh\nsleep 30\n", Limits::default());
let sb = Sandbox::from_config(remote_cfg(
&side,
Limits {
timeout_secs: 1,
..Default::default()
},
))
.unwrap();
let t = Instant::now();
let e = run(&sb, vec![], vec![]).unwrap_err();
assert_eq!(e.code().code, "holger-sandbox-2", "{e}");
assert!(
t.elapsed() < Duration::from_secs(10),
"killed at the deadline, not after: {:?}",
t.elapsed()
);
}
#[test]
fn a_report_past_its_cap_is_refused_by_name() {
let side = sidecar("#!/bin/sh\nhead -c 100000 /dev/zero\n", Limits::default());
let sb = Sandbox::from_config(remote_cfg(
&side,
Limits {
report_max_bytes: 4096,
..Default::default()
},
))
.unwrap();
let e = run(&sb, vec![], vec![]).unwrap_err();
assert_eq!(e.code().code, "holger-sandbox-4", "{e}");
}
#[test]
fn a_worker_that_fails_without_a_report_is_refused_with_its_words() {
let side = sidecar(
"#!/bin/sh\necho 'the job: broken' >&2\nexit 2\n",
Limits::default(),
);
let sb = Sandbox::from_config(remote_cfg(&side, Limits::default())).unwrap();
let e = run(&sb, vec![], vec![]).unwrap_err();
assert_eq!(e.code().code, "holger-sandbox-9", "{e}");
assert!(e.to_string().contains("the job: broken"), "{e}");
}
/// The sidecar trusts nothing the socket says: a job directory outside its share
/// and a command other than the worker's own `probe` are refused.
#[test]
fn the_sidecar_refuses_a_directory_outside_its_share_and_a_foreign_command() {
let side = sidecar(ECHO, Limits::default());
for ask in [
Ask {
schema: remote::SCHEMA.into(),
op: Op::Run,
dir: "../..".into(),
command: vec![],
limits: Limits::default(),
},
Ask {
schema: remote::SCHEMA.into(),
op: Op::Run,
dir: "a/b".into(),
command: vec![],
limits: Limits::default(),
},
Ask {
schema: remote::SCHEMA.into(),
op: Op::Run,
dir: "missing".into(),
command: vec![],
limits: Limits::default(),
},
Ask {
schema: remote::SCHEMA.into(),
op: Op::Run,
dir: String::new(),
command: vec!["sh".into(), "-c".into(), "id".into()],
limits: Limits::default(),
},
Ask {
schema: "holger.scan-remote/v0".into(),
op: Op::Ping,
dir: String::new(),
command: vec![],
limits: Limits::default(),
},
] {
let a = remote::ask(&side.socket, &ask, Duration::from_secs(5), 1 << 20).unwrap_err();
assert_eq!(a.code().code, "holger-sandbox-10", "{ask:?}: {a}");
}
}
#[test]
fn a_remote_config_needs_an_absolute_shared_directory() {
let c: SandboxConfig = ron::from_str("(engine: remote, socket: Some(\"/run/holger-scan/worker.sock\"), work_dir: Some(\"/run/holger-scan\"))").unwrap();
c.validate().unwrap();
let none: SandboxConfig = ron::from_str("(engine: remote)").unwrap();
assert!(
none.validate()
.unwrap_err()
.to_string()
.contains("work_dir"),
"the shared directory is named"
);
let rel: SandboxConfig = ron::from_str("(engine: remote, work_dir: Some(\"scan\"))").unwrap();
assert_eq!(rel.validate().unwrap_err().code().code, "holger-sandbox-10");
let rel_sock: SandboxConfig =
ron::from_str("(engine: remote, work_dir: Some(\"/s\"), socket: Some(\"w.sock\"))")
.unwrap();
assert_eq!(
rel_sock.validate().unwrap_err().code().code,
"holger-sandbox-10"
);
}
// ── RA5-27: what bounds a job's processes ────────────────────────────────────
/// ★ **A release sidecar with nothing bounding a job's processes refuses its ping and every
/// job by name** (`holger-sandbox-13`), and the refusal says what to set.
#[test]
fn a_release_sidecar_with_no_pids_bound_refuses_its_ping_and_every_job_by_name() {
let side = sidecar_bounded(ECHO, Limits::default(), true, PidsBound::Unbounded);
let sb = Sandbox::from_config(remote_cfg(&side, Limits::default())).unwrap();
let e = sb.ready().unwrap_err();
assert_eq!(e.code().code, "holger-sandbox-13", "{e}");
assert!(
e.to_string().contains("podPidsLimit") && e.to_string().contains("--pids-limit"),
"{e}"
);
let e = run(&sb, vec![], vec!["probe".into(), "p".into()]).unwrap_err();
assert_eq!(
e.code().code,
"holger-sandbox-13",
"the job is refused too: {e}"
);
let e = run(&sb, vec![], vec![]).unwrap_err();
assert_eq!(e.code().code, "holger-sandbox-13", "and a scan: {e}");
}
/// ★ **The ping names the bound**: a dev sidecar with none says so and still runs; a
/// release one under the container's own `pids.max` names it and runs.
#[test]
fn the_ping_names_the_pids_bound_and_a_dev_sidecar_without_one_still_runs() {
let dev = sidecar_bounded(ECHO, Limits::default(), false, PidsBound::Unbounded);
let (a, _) = remote::ask(&dev.socket, &ping(), Duration::from_secs(5), 0).expect("the ping");
assert!(a.worker.contains("pids: NO LIMIT"), "{}", a.worker);
let sb = Sandbox::from_config(remote_cfg(&dev, Limits::default())).unwrap();
run(&sb, vec![], vec!["probe".into(), "p".into()]).expect("a dev build runs the job");
let held = sidecar_bounded(
ECHO,
Limits::default(),
true,
PidsBound::Container { max: 2048 },
);
let (a, _) = remote::ask(&held.socket, &ping(), Duration::from_secs(5), 0).expect("the ping");
assert!(
a.worker.contains("pids: the container's limit, 2048"),
"{}",
a.worker
);
let sb = Sandbox::from_config(remote_cfg(&held, Limits::default())).unwrap();
sb.ready()
.expect("a release sidecar under the container's limit is ready");
run(&sb, vec![], vec!["probe".into(), "p".into()]).expect("and runs the job");
}
/// ★ **In a pod the operator names the pod's limit** (`--pod-pids-limit`, the kubelet's
/// `podPidsLimit`, which sits above the container's cgroup namespace): a release sidecar
/// that reads no bound of its own takes it, names it in its ping, and runs.
#[test]
fn a_release_sidecar_takes_the_pods_limit_the_operator_names() {
let tmp = tempfile::tempdir().unwrap();
let share = tmp.path().join("share");
let scratch = tmp.path().join("scratch");
std::fs::create_dir_all(&share).unwrap();
std::fs::create_dir_all(&scratch).unwrap();
let worker = program(tmp.path(), "worker.sh", ECHO);
let socket = share.join("worker.sock");
let cfg = ServeConfig {
socket: socket.clone(),
share: share.clone(),
worker,
scratch,
ceiling: Limits::default(),
max_jobs: 2,
cgroup_fs: None,
refuse_unbounded: true,
pod_pids_limit: Some(1024),
};
let listener = remote::bind(&cfg).unwrap();
std::thread::spawn(move || remote::serve(listener, cfg));
let side = Side {
_tmp: tmp,
share,
socket,
};
let (a, _) = remote::ask(&side.socket, &ping(), Duration::from_secs(5), 0).expect("the ping");
assert!(
a.worker
.contains("pids: the pod's limit as the operator states it, 1024"),
"{}",
a.worker
);
let sb = Sandbox::from_config(remote_cfg(&side, Limits::default())).unwrap();
run(&sb, vec![], vec!["probe".into(), "p".into()]).expect("the job runs");
}
/// ★ **Under a delegated cgroup each job runs in its own, with its pids bound**: the files
/// the sidecar writes, in a cgroup tree in a tempdir. The job's `pids.max` is the ask under
/// the sidecar's ceiling, the worker joined it before exec, and it is killed after.
#[test]
fn under_a_delegated_cgroup_each_job_gets_its_own_with_its_pids_bound() {
let cg = tempfile::tempdir().unwrap();
let side = sidecar_bounded(
ECHO,
Limits {
pids: 32,
..Default::default()
},
true,
PidsBound::PerJob {
own: cg.path().to_path_buf(),
},
);
let sb = Sandbox::from_config(remote_cfg(
&side,
Limits {
pids: 4096,
..Default::default()
},
))
.unwrap();
sb.ready().expect("a delegated sidecar is ready");
run(&sb, vec![], vec!["probe".into(), "p".into()]).expect("the job runs");
let jobs: Vec<PathBuf> = std::fs::read_dir(cg.path())
.unwrap()
.filter_map(|e| e.ok())
.filter(|e| e.file_name().to_string_lossy().starts_with("job-"))
.map(|e| e.path())
.collect();
assert_eq!(jobs.len(), 1, "one cgroup for the one job: {jobs:?}");
assert_eq!(
std::fs::read_to_string(jobs[0].join("pids.max")).unwrap(),
"32",
"the sidecar's ceiling bounds the ask"
);
assert_eq!(
std::fs::read_to_string(jobs[0].join("cgroup.procs")).unwrap(),
"0",
"the worker joined it before exec"
);
assert_eq!(
std::fs::read_to_string(jobs[0].join("cgroup.kill")).unwrap(),
"1",
"what the job left is killed"
);
}
/// The live tests share the container's one delegated cgroup: one at a time, so neither
/// sees the other's job cgroups.
static LIVE: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// A worker that forks 64 sleepers (their output away from the report's pipe) and says how
/// many it started and where it ran.
const BOMB: &str = r#"#!/bin/sh
n=0
i=0
while [ $i -lt 64 ]; do
if sleep 30 >/dev/null 2>&1 & then n=$((n+1)); fi
i=$((i+1))
done
printf '{"started":%s,"cgroup":"%s"}' "$n" "$(cut -d: -f3 /proc/self/cgroup)"
"#;
/// ★ **LIVE: a fork bomb in a job stops at the job's `pids.max`**, under a REAL delegated
/// cgroup v2, and the sidecar and the next job are fine. Runs where
/// `HOLGER_TEST_DELEGATED_CGROUP=1`: a container that delegates its cgroup, e.g. podman
/// `--security-opt unmask=/sys/fs/cgroup` as root in its user namespace.
#[test]
fn live_a_fork_bomb_stops_at_the_jobs_pids_max_under_a_delegated_cgroup() {
if std::env::var("HOLGER_TEST_DELEGATED_CGROUP").as_deref() != Ok("1") {
eprintln!("skipped: HOLGER_TEST_DELEGATED_CGROUP=1 runs it in a container that delegates its cgroup");
return;
}
let _one = LIVE.lock().unwrap_or_else(|e| e.into_inner());
let bound = holger_sandbox::pids::establish(
Path::new("/sys/fs/cgroup"),
&std::fs::read_to_string("/proc/self/cgroup").unwrap(),
);
let PidsBound::PerJob { own } = bound.clone() else {
panic!("this container does not delegate its cgroup: {bound:?}")
};
let side = sidecar_bounded(
BOMB,
Limits {
pids: 16,
..Default::default()
},
true,
bound,
);
let sb = Sandbox::from_config(remote_cfg(
&side,
Limits {
timeout_secs: 20,
..Default::default()
},
))
.unwrap();
match run(&sb, vec![], vec![]) {
Ok(out) => {
let v: serde_json::Value = serde_json::from_slice(&out.report).unwrap();
eprintln!("the bomb reported {v}");
let started = v["started"].as_u64().unwrap();
assert!(
started < 16,
"the job's pids.max (16) stopped the bomb: {v}"
);
// Sealed (RA5-27): the job's cgroup is the root of the worker's own cgroup namespace.
assert_eq!(
v["cgroup"], "/",
"the worker ran sealed in its job's cgroup: {v}"
);
}
// A shell that cannot fork may give up instead of carrying on: the bomb still stopped.
Err(e) => {
eprintln!("the bomb was refused: {e}");
assert!(e.to_string().to_lowercase().contains("fork"), "{e}");
}
}
let left: Vec<String> = std::fs::read_dir(&own)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.starts_with("job-"))
.collect();
assert!(
left.is_empty(),
"the job's cgroup, with its sleepers, is gone after the job: {left:?}"
);
let ok = sidecar_bounded(ECHO, Limits::default(), true, PidsBound::PerJob { own });
let sb = Sandbox::from_config(remote_cfg(&ok, Limits::default())).unwrap();
run(&sb, vec![], vec!["probe".into(), "p".into()]).expect("the next job runs");
}
/// A worker that tries to leave its job's cgroup (RA5-27): it moves itself into the
/// sidecar's own `serve` cgroup and raises its own job's `pids.max`, and says what it got.
/// `@OWN@` is the sidecar's delegated cgroup directory.
const ESCAPE: &str = r#"#!/bin/sh
own='@OWN@'
job=''
for d in "$own"/job-*; do
if grep -qx "$$" "$d/cgroup.procs" 2>/dev/null; then job=$d; fi
done
raise=refused
if [ -n "$job" ] && (echo max > "$job/pids.max") 2>/dev/null; then raise=raised; fi
serve=refused
if (echo 0 > "$own/serve/cgroup.procs") 2>/dev/null; then serve=moved; fi
printf '{"serve":"%s","raise":"%s","found_job":"%s","sees":"%s"}' "$serve" "$raise" "$job" "$(cut -d: -f3 /proc/self/cgroup)"
"#;
/// ★ **LIVE: a job's worker cannot leave its job's cgroup nor raise its bound** (RA5-27),
/// under a REAL delegated cgroup v2. The worker runs as the sidecar's own user, so the
/// cgroup files are its to write; what refuses it is the job's own cgroup namespace (the
/// job's cgroup is the namespace's root, and the hierarchy is mounted `nsdelegate`): a
/// move to a cgroup outside it and a write to the root's own `pids.max` are refused.
/// Runs where `HOLGER_TEST_DELEGATED_CGROUP=1`, as the fork-bomb test above.
#[test]
fn live_a_jobs_worker_cannot_leave_its_cgroup_nor_raise_its_bound() {
if std::env::var("HOLGER_TEST_DELEGATED_CGROUP").as_deref() != Ok("1") {
eprintln!("skipped: HOLGER_TEST_DELEGATED_CGROUP=1 runs it in a container that delegates its cgroup");
return;
}
let _one = LIVE.lock().unwrap_or_else(|e| e.into_inner());
let bound = holger_sandbox::pids::establish(
Path::new("/sys/fs/cgroup"),
&std::fs::read_to_string("/proc/self/cgroup").unwrap(),
);
let PidsBound::PerJob { own } = bound.clone() else {
panic!("this container does not delegate its cgroup (or a job cannot be sealed in it): {bound:?}")
};
let side = sidecar_bounded(
&ESCAPE.replace("@OWN@", &own.display().to_string()),
Limits {
pids: 16,
..Default::default()
},
true,
bound,
);
let sb = Sandbox::from_config(remote_cfg(
&side,
Limits {
timeout_secs: 20,
..Default::default()
},
))
.unwrap();
let out = run(&sb, vec![], vec![]).expect("the job runs and reports");
let v: serde_json::Value = serde_json::from_slice(&out.report).unwrap();
eprintln!("the worker reported {v}");
assert!(
v["found_job"].as_str().unwrap().contains("/job-"),
"the worker found its job's cgroup on the mount: {v}"
);
assert_eq!(
v["serve"], "refused",
"the worker could move itself out of its job's cgroup: {v}"
);
assert_eq!(
v["raise"], "refused",
"the worker could raise its own job's pids.max: {v}"
);
assert_eq!(
v["sees"], "/",
"the worker sees its job's cgroup as the root of its own namespace: {v}"
);
let ok = sidecar_bounded(ECHO, Limits::default(), true, PidsBound::PerJob { own });
let sb = Sandbox::from_config(remote_cfg(&ok, Limits::default())).unwrap();
run(&sb, vec![], vec!["probe".into(), "p".into()]).expect("the next job runs");
}