crates/heimdall-container/src/iolimit.rs
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Ignalina ApS and Vetra Automation AB
//! io limits of every job: cgroup v2 `io.max` (bytes and operations per
//! second, per block device), on top of `io.weight`.
//!
//! Heimdall is a defensive sandbox. A job that parses untrusted data must not
//! be able to starve the host's disks. The runner's config names the devices
//! ([`IoDevice`]): by number, or as "the disk that holds this path". Both are
//! resolved at start to the WHOLE disk (the kernel refuses `io.max` on a
//! partition), written into the job's OCI config as `blockIO` throttles
//! (libcgroups turns them into `io.max` lines), read back by the shim before
//! the job starts and checked by the runner: a job whose `io.max` is not
//! exactly what the config asks is never signed. A config with io limits on
//! a kernel without the io controller refuses to start, strict or not.
use std::collections::BTreeMap;
use std::fmt;
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
/// A block device number.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
pub struct DevNum {
pub major: u32,
pub minor: u32,
}
impl fmt::Display for DevNum {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}:{}", self.major, self.minor)
}
}
impl DevNum {
/// Parse `major:minor`.
pub fn parse(s: &str) -> Option<DevNum> {
let (a, b) = s.trim().split_once(':')?;
Some(DevNum {
major: a.parse().ok()?,
minor: b.parse().ok()?,
})
}
}
/// Which device a limit is for.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum IoDevice {
/// A block device by number (a partition means its whole disk).
Number(DevNum),
/// The disk holding this path (its file system's device; a partition
/// means its whole disk). Refused when the file system has no block
/// device (tmpfs, overlay, a btrfs subvolume): name the device by number.
Holding(PathBuf),
}
/// The limits for one device; `None` = no limit (`max`). Zero is refused: it
/// would stop every io of the job on that device.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct IoRates {
pub rbps: Option<u64>,
pub wbps: Option<u64>,
pub riops: Option<u64>,
pub wiops: Option<u64>,
}
impl IoRates {
fn pairs(&self) -> [(&'static str, Option<u64>); 4] {
[
("rbps", self.rbps),
("wbps", self.wbps),
("riops", self.riops),
("wiops", self.wiops),
]
}
}
/// One `io.max` limit of the runner's config.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct IoMax {
pub device: IoDevice,
pub rates: IoRates,
}
/// An [`IoMax`] resolved to its whole disk.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResolvedIoMax {
pub dev: DevNum,
pub rates: IoRates,
}
impl ResolvedIoMax {
/// The `io.max` line the kernel shows for this limit.
pub fn line(&self) -> String {
let mut s = self.dev.to_string();
for (k, v) in self.rates.pairs() {
match v {
Some(n) => s.push_str(&format!(" {k}={n}")),
None => s.push_str(&format!(" {k}=max")),
}
}
s
}
}
/// The device number of the file system holding `p`.
pub fn device_of(p: &Path) -> Result<DevNum, String> {
let m = std::fs::metadata(p).map_err(|e| format!("io limit path {}: {e}", p.display()))?;
let d = m.dev();
Ok(DevNum {
major: libc::major(d),
minor: libc::minor(d),
})
}
/// The whole disk of `dev` (itself unless it is a partition), from
/// `<sysfs>/dev/block/<dev>`.
pub fn whole_disk(sysfs: &Path, dev: DevNum) -> Result<DevNum, String> {
let entry = sysfs.join("dev/block").join(dev.to_string());
if !entry.exists() {
return Err(format!(
"device {dev} is not a block device ({} missing): name the device by number",
entry.display()
));
}
if !entry.join("partition").exists() {
return Ok(dev);
}
let real = entry
.canonicalize()
.map_err(|e| format!("{}: {e}", entry.display()))?;
let parent = real
.parent()
.ok_or_else(|| format!("partition {dev} has no parent disk"))?;
let s = std::fs::read_to_string(parent.join("dev"))
.map_err(|e| format!("the disk of partition {dev}: {e}"))?;
DevNum::parse(&s).ok_or_else(|| format!("the disk of partition {dev}: bad dev {s:?}"))
}
/// Resolve the config's limits to whole disks; refuses zero rates, a limit
/// without any rate and two limits on the same disk.
pub fn resolve(limits: &[IoMax], sysfs: &Path) -> Result<Vec<ResolvedIoMax>, String> {
let mut out: Vec<ResolvedIoMax> = Vec::with_capacity(limits.len());
for l in limits {
if l.rates.pairs().iter().all(|(_, v)| v.is_none()) {
return Err(format!("io limit for {:?} sets no rate", l.device));
}
if let Some((k, _)) = l.rates.pairs().iter().find(|(_, v)| *v == Some(0)) {
return Err(format!(
"io limit {k}=0 for {:?} would stop every io of the job",
l.device
));
}
let dev = match &l.device {
IoDevice::Number(d) => *d,
IoDevice::Holding(p) => device_of(p)?,
};
let dev =
whole_disk(sysfs, dev).map_err(|e| format!("io limit for {:?}: {e}", l.device))?;
if out.iter().any(|r| r.dev == dev) {
return Err(format!("two io limits for the disk {dev}"));
}
out.push(ResolvedIoMax {
dev,
rates: l.rates,
});
}
Ok(out)
}
/// Add the limits to the job's OCI config as `blockIO` throttles.
pub fn apply(config: &mut Value, limits: &[ResolvedIoMax]) {
if limits.is_empty() {
return;
}
let keys = [
("rbps", "throttleReadBpsDevice"),
("wbps", "throttleWriteBpsDevice"),
("riops", "throttleReadIOPSDevice"),
("wiops", "throttleWriteIOPSDevice"),
];
let b = &mut config["linux"]["resources"]["blockIO"];
if !b.is_object() {
*b = json!({});
}
for (rate, key) in keys {
let list: Vec<Value> = limits
.iter()
.filter_map(|l| {
let v = l.rates.pairs().into_iter().find(|(k, _)| *k == rate)?.1?;
Some(json!({"major": l.dev.major, "minor": l.dev.minor, "rate": v}))
})
.collect();
if !list.is_empty() {
b[key] = Value::Array(list);
}
}
}
/// Check the job cgroup's `io.max` as read back: every configured disk has
/// exactly the configured rates (`max` where none is set).
pub fn check(read_back: &str, limits: &[ResolvedIoMax]) -> Result<(), String> {
let mut seen: BTreeMap<DevNum, BTreeMap<String, String>> = BTreeMap::new();
for line in read_back.lines() {
let mut it = line.split_whitespace();
let Some(dev) = it.next().and_then(DevNum::parse) else {
continue;
};
let kv = it
.filter_map(|w| w.split_once('='))
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
seen.insert(dev, kv);
}
for l in limits {
let Some(kv) = seen.get(&l.dev) else {
return Err(format!(
"io.max has no line for {} (wanted {})",
l.dev,
l.line()
));
};
for (k, v) in l.rates.pairs() {
let want = v.map(|n| n.to_string()).unwrap_or_else(|| "max".into());
let got = kv.get(k).map(String::as_str).unwrap_or("max");
if got != want {
return Err(format!(
"io.max of {} has {k}={got}, not {want} (wanted {})",
l.dev,
l.line()
));
}
}
}
Ok(())
}