crates/heimdall-container/tests/io.rs
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Ignalina ApS and Vetra Automation AB
//! io limits on a real kernel: a job's direct reads and writes on a
//! throttled disk run at the configured rate, and its cgroup's `io.max` is
//! what the config asks while it runs. Heimdall is a defensive sandbox; the
//! jobs here only read and write their own files.
//!
//! These need everything `tests/container.rs` needs plus the io controller
//! and a disk of their own: `scripts/io-tests.sh` runs them in a privileged
//! ROOTFUL podman container on a fresh loop device (ext2), with
//! `HEIMDALL_PLAIN_SCRATCH` (every job dir, bundle and file) on it and
//! `HEIMDALL_IO_TEST_DISK` set to that loop device's number.
#[path = "../../heimdall-plain/tests/common/mod.rs"]
mod common;
use std::path::PathBuf;
use std::time::{Duration, Instant};
use common::*;
use heimdall_container::iolimit::{DevNum, IoDevice, IoMax, IoRates};
use heimdall_container::*;
use heimdall_core::*;
const MIB: u64 = 1 << 20;
fn disk() -> DevNum {
let v = std::env::var("HEIMDALL_IO_TEST_DISK")
.expect("HEIMDALL_IO_TEST_DISK unset: run scripts/io-tests.sh");
DevNum::parse(&v).expect("HEIMDALL_IO_TEST_DISK is major:minor")
}
struct Fx {
base: PathBuf,
image: ImageRef,
runner: HeimdallRunner,
}
/// A runner whose jobs get `rates` on the test disk, named the way `by` says.
fn fx(name: &str, by_path: bool, rates: Option<IoRates>) -> Fx {
let base = scratch(&format!("hc-io-{name}"));
let layout = Layout::new(&base.join("layout"));
let image = busybox_image(&layout);
let mut c = HeimdallConfig::new(&layout.dir, base.join("work"));
if let Some(rates) = rates {
let device = if by_path {
IoDevice::Holding(base.clone())
} else {
IoDevice::Number(disk())
};
c.io_max = vec![IoMax { device, rates }];
}
let runner =
HeimdallRunner::new(c).expect("this kernel has every feature and the io controller");
assert!(
runner.kernel().has_controller("io"),
"{:?}",
runner.kernel()
);
Fx {
base,
image,
runner,
}
}
impl Fx {
/// Run `script` with `big` (bytes) in `/in/big`; the time the run took.
fn timed(&self, script: &str, big: u64) -> (Duration, JobResult) {
let d = job_dirs(&self.base);
let f = d.input.join("big");
std::fs::write(&f, vec![0x5au8; big as usize]).unwrap();
// On the disk before the job starts (it reads with O_DIRECT).
std::fs::File::open(&f).unwrap().sync_all().unwrap();
let mut j = native_job(&self.image, &sh(script), &d);
j.policy.limits.wall = Duration::from_secs(60);
let t0 = Instant::now();
let s = self.runner.run(j).expect("a signed result");
verify(&s, &self.runner.verifying_key()).expect("it verifies");
(t0.elapsed(), s.result)
}
}
const READ: &str = "dd if=/in/big of=/dev/null bs=1M iflag=direct";
const WRITE: &str = "dd if=/dev/zero of=/out/w bs=1M count=8 oflag=direct";
#[test]
#[ignore = "needs the io controller and a loop device: scripts/io-tests.sh"]
fn a_read_bps_limit_slows_direct_reads_to_the_rate() {
let (fast, r) = fx("rd-free", false, None).timed(READ, 8 * MIB);
assert!(r.verdict.is_accepted(), "{:?}", r.verdict);
// 8 MiB at 2 MiB/s: about 4 s (the first slice may run ahead).
let (slow, r) = fx(
"rd-2m",
false,
Some(IoRates {
rbps: Some(2 * MIB),
..IoRates::default()
}),
)
.timed(READ, 8 * MIB);
assert!(r.verdict.is_accepted(), "{:?}", r.verdict);
assert!(
slow >= Duration::from_secs(3),
"throttled read took {slow:?}"
);
assert!(
slow >= fast + Duration::from_secs(2),
"free {fast:?}, throttled {slow:?}"
);
}
#[test]
#[ignore = "needs the io controller and a loop device: scripts/io-tests.sh"]
fn a_write_bps_limit_on_the_disk_holding_the_job_dirs_slows_direct_writes() {
let rates = IoRates {
wbps: Some(2 * MIB),
..IoRates::default()
};
let (slow, r) = fx("wr-2m", true, Some(rates)).timed(WRITE, 1);
assert!(r.verdict.is_accepted(), "{:?}", r.verdict);
assert!(
slow >= Duration::from_secs(3),
"throttled write took {slow:?}"
);
let (fast, _) = fx("wr-free", true, None).timed(WRITE, 1);
assert!(
slow >= fast + Duration::from_secs(2),
"free {fast:?}, throttled {slow:?}"
);
}
#[test]
#[ignore = "needs the io controller and a loop device: scripts/io-tests.sh"]
fn a_read_iops_limit_slows_small_direct_reads() {
let rates = IoRates {
riops: Some(20),
..IoRates::default()
};
// 80 reads of 4 KiB at 20 per second: about 4 s.
let (slow, r) = fx("riops", false, Some(rates)).timed(
"dd if=/in/big of=/dev/null bs=4k count=80 iflag=direct",
4 * MIB,
);
assert!(r.verdict.is_accepted(), "{:?}", r.verdict);
assert!(
slow >= Duration::from_secs(3),
"throttled reads took {slow:?}"
);
}
#[test]
#[ignore = "needs the io controller and a loop device: scripts/io-tests.sh"]
fn the_job_cgroup_has_the_configured_io_max_while_the_job_runs() {
let f = fx(
"iomax",
false,
Some(IoRates {
rbps: Some(3 * MIB),
wiops: Some(77),
..IoRates::default()
}),
);
let d = job_dirs(&f.base);
let j = native_job(&f.image, &sh("sleep 3"), &d);
let cg = f
.runner
.config()
.cgroup_root
.join(&f.runner.config().cgroup_parent)
.join(format!("job-{}", j.id));
let seen = std::thread::spawn(move || {
for _ in 0..1000 {
if cg.join("io.max").exists() {
std::thread::sleep(Duration::from_millis(1000));
let r = |f: &str| std::fs::read_to_string(cg.join(f)).unwrap_or_default();
return Some((r("io.max"), r("io.weight")));
}
std::thread::sleep(Duration::from_millis(5));
}
None
});
let s = f.runner.run(j).unwrap();
assert!(s.result.verdict.is_accepted(), "{:?}", s.result.verdict);
let want = format!("{} rbps={} wbps=max riops=max wiops=77", disk(), 3 * MIB);
let (got, weight) = seen.join().unwrap().unwrap_or_default();
assert_eq!(
weight.lines().next(),
Some("default 50"),
"io.weight {weight:?}"
);
assert!(
got.lines().any(|l| l.trim() == want),
"io.max {got:?}, wanted {want:?}"
);
}