crates/mock-upcloud/src/container.rs
//! **The container half: a rootless container behind every server** (monetize v2.0 P1, the
//! fast test `cargo xtask fast`, A24 §8.2; CI lane 2026-10-23).
//!
//! The estate delegates each server's machine to a [`GuestEngine`]; `mock-upcloud-kvm` puts a
//! QEMU there. [`ContainerEngine`] puts a rootless podman container there (`mock-upcloud
//! --engine container`), so the ONE box lays, pairs and drives real product binaries in
//! seconds. It shells out to `podman` and links nothing, so it lives here, beside the seam:
//!
//! | the API says | the container gets |
//! |---|---|
//! | the server's medium (a storage uploaded or imported, then cloned) | a CONTAINER MEDIUM ([`Medium`], JSON): the program, its arguments, its environment and the files the installer row stages under `/data` |
//! | its `user_data` (the shop's pairing seed) | `/data/pairing/seed.toml`, where the medium's seed marker would have fetched it |
//! | its utility address and MAC | the same, on a podman network over the mock's utility pool |
//! | its loopback public address and the admitted installer ports | those ports published on that address (all of them are ≥ 1024, so rootless podman binds them) |
//! | power on, off, running | `podman run -d`, `podman stop`/`kill`, `podman inspect` |
//!
//! Every box gets `--cap-add NET_ADMIN --device /dev/net/tun` in its own network namespace:
//! a box raises its own WireGuard link, as it does on a VM (measured 2026-10-23: rootless
//! `ip link add wg0 type wireguard` succeeds on builder-64 and on dwarf).
//!
//! What a container is not, said: there is no korp, no installer pass, no kernel of the
//! medium. The program the installer row would boot is started directly from this build's
//! binaries (mounted read only at `/opt/bin`).
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::Mutex;
use crate::kvm::{GuestEngine, GuestOutcome, GuestSpec, Machine};
/// The key every container medium carries (`{"container_medium": 1, …}`): the engine runs a
/// storage whose bytes are a JSON object with it, and takes any other bytes (an ISO) for none.
pub const MAGIC: &str = "container_medium";
/// Where a box's durable root is in its container.
pub const DATA: &str = "/data";
/// Where the box's pairing seed is staged (holger, gunnar and the ONE box read it there).
pub const SEED_REL: &str = "pairing/seed.toml";
/// Where this build's binaries are mounted.
pub const BIN: &str = "/opt/bin";
/// **A container medium**: what the installer row of a product would boot, as data.
#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
pub struct Medium {
/// Always 1.
pub container_medium: u32,
/// The program under [`BIN`].
pub program: String,
#[serde(default)]
pub args: Vec<String>,
#[serde(default)]
pub env: BTreeMap<String, String>,
/// Files staged under [`DATA`], by relative path.
#[serde(default)]
pub files: BTreeMap<String, String>,
}
impl Medium {
/// A medium from its bytes: `None` when they are not a container medium (a real ISO).
pub fn parse(bytes: &[u8]) -> Result<Option<Medium>, String> {
let Ok(v) = serde_json::from_slice::<serde_json::Value>(bytes) else { return Ok(None) };
if v.get(MAGIC).is_none() {
return Ok(None);
}
serde_json::from_value(v).map(Some).map_err(|e| format!("[container-medium] {e}"))
}
/// The medium as its bytes.
pub fn to_bytes(&self) -> Vec<u8> {
serde_json::to_vec_pretty(self).unwrap_or_default()
}
}
/// The engine's settings.
#[derive(Debug, Clone)]
pub struct Settings {
/// The engine's own directory: media, each server's `/data`.
pub root: PathBuf,
/// The podman network over the mock's utility pool ([`network_args`]).
pub network: String,
/// The image every box runs in (the products' musl binaries run in any).
pub image: String,
/// This build's binaries, mounted read only at [`BIN`].
pub bin_dir: PathBuf,
/// A medium for a server whose disk carries none, by its hostname's prefix
/// (`holger-server-` → the main's medium): a lay that names no release medium.
pub by_hostname: Vec<(String, Medium)>,
/// A label every container carries (its run), so a run's leftovers are found by it.
pub label: String,
/// The engine program (`podman`).
pub engine: String,
/// The utility network: `X.Y` of `X.Y.8.0/21` (gateway `X.Y.15.254`). The mock's own pool is
/// [`MOCK_UTILITY`]; a rig on an UpCloud machine already has a `10.13.8.0/22` on its own
/// utility NIC (measured on builder-64: 10.13.10.7/22 on eth1), so a run there takes another.
pub utility: String,
}
/// The mock's own utility pool's `X.Y` (`10.13.8.96–120`, `10.13.12.96–120`).
pub const MOCK_UTILITY: &str = "10.13";
/// Pure: does this `fib_trie` text name an address or route inside `X.Y.8.0/21`?
pub fn trie_holds(trie: &str, utility: &str) -> bool {
(8..=15).any(|c| trie.contains(&format!(" {utility}.{c}.")))
}
/// **A utility `X.Y` this host does not already hold** (its own NICs and routes, read from
/// `/proc/net/fib_trie`): the mock's own first, then two more. `None`: all are taken.
pub fn free_utility_here() -> Option<String> {
let trie = std::fs::read_to_string("/proc/net/fib_trie").unwrap_or_default();
free_utility_for_slot(&trie, 0)
}
/// How many fast runs may share one host's rootless network namespace at once.
pub const FAST_SLOTS: u8 = 4;
/// Pure: **the utility `X.Y` of parallel slot `slot`** that `trie` (the host's `fib_trie`) does
/// not hold. Slot 0 is today's choice (the mock's own, then `10.213`, then `172.29`); slot `n`
/// takes `10.(213+n)`, then `172.(29+n)`, so two slots never share a network. The candidates
/// are per slot because another run's network lives inside the rootless namespace, where the
/// host's table cannot see it.
pub fn free_utility_for_slot(trie: &str, slot: u8) -> Option<String> {
if slot >= FAST_SLOTS {
return None;
}
let cands: Vec<String> = if slot == 0 {
vec![MOCK_UTILITY.into(), "10.213".into(), "172.29".into()]
} else {
vec![format!("10.{}", 213 + slot as u16), format!("172.{}", 29 + slot as u16)]
};
cands.into_iter().find(|u| !trie_holds(trie, u))
}
impl Settings {
/// The utility network's subnet.
pub fn subnet(&self) -> String {
format!("{}.8.0/21", self.utility)
}
/// Its gateway: inside podman's rootless network namespace, the ONE side's address.
pub fn gateway(&self) -> String {
format!("{}.15.254", self.utility)
}
}
/// **The engine.** One container per server.
pub struct ContainerEngine {
s: Settings,
media: Mutex<BTreeMap<String, Vec<u8>>>,
user_data: Mutex<BTreeMap<String, String>>,
started: Mutex<BTreeMap<String, String>>,
}
/// Pure: the arguments that make the utility network: the mock's pool is `X.Y.8.96–120` and
/// `X.Y.12.96–120`, two /22s inside one /21, and the gateway is outside the pool.
pub fn network_args(s: &Settings) -> Vec<String> {
["network", "create", "--subnet", &s.subnet(), "--gateway", &s.gateway(), "--label", &format!("mock-upcloud-container={}", s.label), &s.network].iter().map(|x| x.to_string()).collect()
}
/// Pure: the container's name for a server.
/// The run's label is part of it: a mock's server uuids are deterministic, so two parallel runs
/// would otherwise name their containers alike.
pub fn container_name(label: &str, server: &str) -> String {
let safe = |t: &str, n: usize| t.chars().filter(|c| c.is_ascii_alphanumeric()).take(n).collect::<String>();
format!("mu-{}-{}", safe(label, 24), safe(server, 16))
}
/// Pure: is a run LIVE, by its containers' `podman ps -a --format {{.Status}}` lines? Live while
/// one is `Up` or still `Stopping`; a run whose containers all exited (or were killed) is dead.
pub fn run_is_live(statuses: &str) -> bool {
statuses.lines().map(str::trim).any(|l| l.starts_with("Up") || l.starts_with("Stopping"))
}
/// **One rung of a run's bounded teardown** ([`next_sweep`]): what to do next with the
/// containers a run still has.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Sweep {
/// No container of the run is left.
Done,
/// The last step still has time: poll again.
Wait,
/// `stop -t` the live ones.
Stop,
/// `kill` the live ones a stop did not end.
Kill,
/// `rm -f` every one that is left.
Remove,
/// The bounded attempts are spent and containers are left: the run says so.
GiveUp,
}
/// Pure: **the next teardown step**, from the run's `{{.Status}}` lines as polled now (`None`:
/// the list could not be read), the step issued last and how long ago, and how many removes
/// were run.
///
/// The ladder is stop, then kill, then remove (run again up to [`SWEEP_REMOVES`] times while a
/// slow engine has not finished it); each step gets `patience` before the next. A container
/// already `Stopping` is killed, not stopped; one that exited is removed at once. A list that cannot be read is
/// never "none left".
pub fn next_sweep(statuses: Option<&str>, last: Option<Sweep>, since_last: std::time::Duration, patience: std::time::Duration, removes: usize) -> Sweep {
let Some(statuses) = statuses else {
return if since_last < patience { Sweep::Wait } else { Sweep::GiveUp };
};
if !statuses.lines().any(|l| !l.trim().is_empty()) {
return Sweep::Done;
}
let live = run_is_live(statuses);
let up = statuses.lines().any(|l| l.trim().starts_with("Up"));
match last {
// A stop is refused on a box already `Stopping`: such a box is killed.
None if up => Sweep::Stop,
None if live => Sweep::Kill,
None => Sweep::Remove,
Some(Sweep::Stop | Sweep::Kill) if !live => Sweep::Remove,
Some(_) if since_last < patience => Sweep::Wait,
Some(Sweep::Stop) => Sweep::Kill,
Some(Sweep::Kill) => Sweep::Remove,
Some(Sweep::Remove) if removes < SWEEP_REMOVES => Sweep::Remove,
Some(_) => Sweep::GiveUp,
}
}
/// How many times a teardown runs `rm -f` before it gives up ([`next_sweep`]).
pub const SWEEP_REMOVES: usize = 3;
/// How long each teardown step has before the next one ([`next_sweep`]).
pub const SWEEP_PATIENCE: std::time::Duration = std::time::Duration::from_secs(15);
/// How long one `rm -f` of the teardown may run before it is killed: longer than the other
/// calls, since a loaded engine was seen to need 96 s for one.
pub const SWEEP_REMOVE_LIMIT: std::time::Duration = std::time::Duration::from_secs(90);
/// How long one engine call of the teardown may run before it is killed.
pub const SWEEP_CALL_LIMIT: std::time::Duration = std::time::Duration::from_secs(60);
/// One teardown step that was run: what, on how many containers, how long the call took, and
/// whether the engine said yes.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SweepStep {
pub action: Sweep,
pub containers: usize,
pub took: std::time::Duration,
pub ok: bool,
}
/// What a bounded teardown did, and what is left (`None`: the list could not be read).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SweepReport {
pub steps: Vec<SweepStep>,
pub left: Option<usize>,
pub took: std::time::Duration,
}
impl SweepReport {
/// True only when the list was read and no container is left.
pub fn clean(&self) -> bool {
self.left == Some(0)
}
/// One line: each step with its time, then what is left.
pub fn words(&self) -> String {
let mut parts: Vec<String> = self.steps.iter().map(|s| format!("{:?} {} in {:.1} s{}", s.action, s.containers, s.took.as_secs_f64(), if s.ok { "" } else { " (refused)" })).collect();
parts.push(match self.left {
Some(n) => format!("{n} left after {:.1} s", self.took.as_secs_f64()),
None => format!("the list could not be read after {:.1} s", self.took.as_secs_f64()),
});
parts.join("; ")
}
}
/// Pure: `podman ps -a --format '{{.ID}} {{.Status}}'` as (id, status) rows.
pub fn parse_ps(out: &str) -> Vec<(String, String)> {
out.lines()
.filter_map(|l| {
let l = l.trim();
let (id, status) = l.split_once(' ').unwrap_or((l, ""));
(!id.is_empty()).then(|| (id.to_string(), status.trim().to_string()))
})
.collect()
}
/// Runs `engine args`, killed after `limit`: its stdout when it succeeded in time.
fn bounded_output(engine: &str, args: &[&str], limit: std::time::Duration) -> Option<String> {
use std::io::Read;
let mut child = Command::new(engine).args(args).stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::null()).spawn().ok()?;
let mut stdout = child.stdout.take()?;
let reader = std::thread::spawn(move || {
let mut out = String::new();
let _ = stdout.read_to_string(&mut out);
out
});
let t0 = std::time::Instant::now();
let status = loop {
match child.try_wait() {
Ok(Some(st)) => break Some(st),
Ok(None) if t0.elapsed() < limit => std::thread::sleep(std::time::Duration::from_millis(100)),
_ => {
let _ = child.kill();
let _ = child.wait();
break None;
}
}
};
let out = reader.join().unwrap_or_default();
status.filter(|s| s.success()).map(|_| out)
}
/// **A run's bounded teardown**: polls the containers labelled `label` and follows
/// [`next_sweep`] (stop, kill, remove up to [`SWEEP_REMOVES`] times; each step `patience`, each
/// call [`SWEEP_CALL_LIMIT`], a remove [`SWEEP_REMOVE_LIMIT`]),
/// logging the time each step took. It ends when none is left or the attempts are spent; it
/// never says "none left" for a container that is.
pub fn sweep_run(engine: &str, label: &str, patience: std::time::Duration) -> SweepReport {
let filter = format!("label=mock-upcloud-container={label}");
let t0 = std::time::Instant::now();
let (mut steps, mut last, mut at) = (Vec::new(), None, std::time::Instant::now());
loop {
let rows = bounded_output(engine, &["ps", "-a", "--filter", &filter, "--format", "{{.ID}} {{.Status}}"], SWEEP_CALL_LIMIT).map(|o| parse_ps(&o));
let statuses = rows.as_ref().map(|r| r.iter().map(|(_, st)| st.as_str()).collect::<Vec<_>>().join("\n"));
let action = next_sweep(statuses.as_deref(), last, at.elapsed(), patience, steps.iter().filter(|s: &&SweepStep| s.action == Sweep::Remove).count());
let ids: Vec<&str> = match action {
Sweep::Done | Sweep::GiveUp => {
let report = SweepReport { steps, left: rows.as_ref().map(Vec::len), took: t0.elapsed() };
eprintln!("mock-upcloud: teardown of {label}: {}", report.words());
return report;
}
Sweep::Wait => {
std::thread::sleep(std::time::Duration::from_millis(500));
continue;
}
Sweep::Stop => rows.iter().flatten().filter(|(_, st)| st.starts_with("Up")).map(|(id, _)| id.as_str()).collect(),
Sweep::Kill => rows.iter().flatten().filter(|(_, st)| run_is_live(st)).map(|(id, _)| id.as_str()).collect(),
Sweep::Remove => rows.iter().flatten().map(|(id, _)| id.as_str()).collect(),
};
let mut args: Vec<&str> = match action {
Sweep::Stop => vec!["stop", "-t", "5"],
Sweep::Kill => vec!["kill"],
_ => vec!["rm", "-f", "-t", "0"],
};
args.extend(&ids);
let started = std::time::Instant::now();
let limit = if action == Sweep::Remove { SWEEP_REMOVE_LIMIT } else { SWEEP_CALL_LIMIT };
let ok = bounded_output(engine, &args, limit).is_some();
let step = SweepStep { action, containers: ids.len(), took: started.elapsed(), ok };
eprintln!("mock-upcloud: teardown of {label}: {:?} on {} container(s) took {:.1} s{}", step.action, step.containers, step.took.as_secs_f64(), if ok { "" } else { " (the engine refused or timed out)" });
steps.push(step);
last = Some(action);
at = std::time::Instant::now();
}
}
/// Pure: in `podman network ls --format json`, the networks on `subnet` other than `ours`:
/// (STALE: a mock-upcloud run's, with no live container of that run, safe to remove;
/// HELD: a live run's or a foreign one, never touched). An unreadable list names none.
pub fn stale_networks(ls_json: &str, subnet: &str, ours: &str, live: impl Fn(&str) -> bool) -> (Vec<String>, Vec<String>) {
let Ok(v) = serde_json::from_str::<serde_json::Value>(ls_json) else { return (vec![], vec![]) };
let (mut stale, mut held) = (Vec::new(), Vec::new());
for n in v.as_array().into_iter().flatten() {
let name = n["name"].as_str().unwrap_or_default();
if name.is_empty() || name == ours {
continue;
}
let on_ours = n["subnets"].as_array().into_iter().flatten().any(|s| s["subnet"].as_str() == Some(subnet));
if !on_ours {
continue;
}
match n["labels"]["mock-upcloud-container"].as_str() {
Some(label) if !live(label) => stale.push(name.to_string()),
_ => held.push(name.to_string()),
}
}
(stale, held)
}
/// Pure: **the `podman run` argv of a machine** booting `medium`, its `/data` at `data`.
pub fn run_args(s: &Settings, m: &Machine, medium: &Medium, data: &Path) -> Vec<String> {
let mut v: Vec<String> = ["run", "-d", "--init", "--name"].iter().map(|x| x.to_string()).collect();
v.push(container_name(&s.label, &m.server));
v.extend(["--label".to_string(), format!("mock-upcloud-container={}", s.label)]);
v.extend(["--hostname".to_string(), m.hostname.clone()]);
v.extend(["--network".to_string(), s.network.clone()]);
if !m.utility_offer.address.is_empty() {
v.extend(["--ip".to_string(), m.utility_offer.address.clone()]);
}
if !m.utility_mac.is_empty() {
v.extend(["--mac-address".to_string(), m.utility_mac.clone()]);
}
v.extend(["--cap-add", "NET_ADMIN", "--device", "/dev/net/tun"].iter().map(|x| x.to_string()));
// The plan's shape, as the box measures it (cores through its cgroup: a product's election
// makes the weaker box the twin).
if m.cores > 0 {
v.extend(["--cpus".to_string(), m.cores.to_string()]);
}
if m.mem_mb > 0 {
v.extend(["--memory".to_string(), format!("{}m", m.mem_mb)]);
}
if crate::kvm::is_loopback_public(&m.public_ip) {
for (proto, port) in &m.public_admitted {
v.extend(["-p".to_string(), format!("{}:{port}:{port}/{proto}", m.public_ip)]);
}
}
v.extend(["-v".to_string(), format!("{}:{DATA}", data.display())]);
v.extend(["-v".to_string(), format!("{}:{BIN}:ro", s.bin_dir.display())]);
for (k, val) in &medium.env {
v.extend(["-e".to_string(), format!("{k}={val}")]);
}
v.push(s.image.clone());
v.push(format!("{BIN}/{}", medium.program));
v.extend(medium.args.iter().cloned());
v
}
impl ContainerEngine {
pub fn new(s: Settings) -> ContainerEngine {
ContainerEngine { s, media: Mutex::new(BTreeMap::new()), user_data: Mutex::new(BTreeMap::new()), started: Mutex::new(BTreeMap::new()) }
}
/// Make the utility network (idempotent: an existing one is kept).
pub fn ensure_network(&self) -> Result<(), String> {
let exists = Command::new(&self.s.engine).args(["network", "exists", &self.s.network]).status().map(|s| s.success()).unwrap_or(false);
if exists {
return Ok(());
}
// A run that died without its teardown leaves its network on this slot's subnet, and
// podman then refuses ours. Remove such a leftover (its run has no container left); a
// live run's or a foreign network on the subnet is refused by name, never removed.
let ls = Command::new(&self.s.engine).args(["network", "ls", "--format", "json"]).output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default();
let engine = self.s.engine.clone();
let live = |label: &str| {
Command::new(&engine)
.args(["ps", "-a", "--filter", &format!("label=mock-upcloud-container={label}"), "--format", "{{.Status}}"])
.output()
.map(|o| run_is_live(&String::from_utf8_lossy(&o.stdout)))
.unwrap_or(true)
};
let (stale, held) = stale_networks(&ls, &self.s.subnet(), &self.s.network, live);
let label_of = |net: &str| net.strip_prefix("fast-").map(str::to_string);
for n in &stale {
// The dead run's exited containers go first: a network with containers on it stays.
let lbl = serde_json::from_str::<serde_json::Value>(&ls)
.ok()
.and_then(|v| v.as_array().and_then(|a| a.iter().find(|x| x["name"].as_str() == Some(n.as_str())).and_then(|x| x["labels"]["mock-upcloud-container"].as_str().map(str::to_string))))
.or_else(|| label_of(n));
if let Some(l) = lbl {
let ids = Command::new(&self.s.engine).args(["ps", "-aq", "--filter", &format!("label=mock-upcloud-container={l}")]).output().map(|o| String::from_utf8_lossy(&o.stdout).to_string()).unwrap_or_default();
for id in ids.split_whitespace() {
let _ = Command::new(&self.s.engine).args(["rm", "-f", "-t", "2", id]).output();
}
}
let _ = Command::new(&self.s.engine).args(["network", "rm", "-f", n]).output();
eprintln!("mock-upcloud: removed {n}, a dead run's network on {} (and its exited containers)", self.s.subnet());
}
if !held.is_empty() {
return Err(format!("[container-network-held] {} is held by {}: another live run (or not ours) is on this subnet", self.s.subnet(), held.join(", ")));
}
let out = Command::new(&self.s.engine).args(network_args(&self.s)).output().map_err(|e| format!("{}: {e}", self.s.engine))?;
if out.status.success() { Ok(()) } else { Err(format!("[container-network] {}", String::from_utf8_lossy(&out.stderr).trim())) }
}
/// Stop and remove every container of this run ([`sweep_run`], bounded and escalating),
/// then the network. The report says what is left.
pub fn tear_down(&self) -> SweepReport {
let report = sweep_run(&self.s.engine, &self.s.label, SWEEP_PATIENCE);
let _ = bounded_output(&self.s.engine, &["network", "rm", "-f", &self.s.network], SWEEP_CALL_LIMIT);
report
}
/// The medium a machine boots: its tray's, else its system disk's, else its hostname's.
pub fn medium_of(&self, m: &Machine) -> Result<Medium, String> {
let media = self.media.lock().unwrap();
let mut ids: Vec<&str> = Vec::new();
if let Some(t) = &m.medium {
ids.push(t);
}
ids.extend(m.disks.iter().map(|d| d.storage.as_str()));
for id in ids {
if let Some(b) = media.get(id) {
if let Some(md) = Medium::parse(b)? {
return Ok(md);
}
}
}
self.s
.by_hostname
.iter()
.find(|(p, _)| m.hostname.starts_with(p.as_str()))
.map(|(_, md)| md.clone())
.ok_or_else(|| format!("[container-no-medium] server {} ({}) boots no container medium and its hostname names none", m.server, m.hostname))
}
/// The host directory of a server's `/data`.
pub fn data_dir(&self, server: &str) -> PathBuf {
self.s.root.join("servers").join(server).join("data")
}
/// The container's log (stdout and stderr), for a reading.
pub fn logs(&self, server: &str) -> String {
Command::new(&self.s.engine).args(["logs", &container_name(&self.s.label, server)]).output().map(|o| format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))).unwrap_or_default()
}
}
impl GuestEngine for ContainerEngine {
fn name(&self) -> &'static str {
"container"
}
fn boot(&self, _spec: &GuestSpec) -> GuestOutcome {
GuestOutcome::Refused { clause: "boot", why: "the container engine boots servers through power_on only".into() }
}
fn power_on(&self, m: &Machine) -> Result<(), String> {
let medium = self.medium_of(m)?;
let data = self.data_dir(&m.server);
std::fs::create_dir_all(data.join("pairing")).map_err(|e| format!("{}: {e}", data.display()))?;
for (rel, text) in &medium.files {
let p = data.join(rel);
if let Some(parent) = p.parent() {
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
}
std::fs::write(&p, text).map_err(|e| format!("{}: {e}", p.display()))?;
}
let seed = m.user_data.clone().or_else(|| self.user_data.lock().unwrap().get(&m.server).cloned());
if let Some(seed) = seed {
std::fs::write(data.join(SEED_REL), seed).map_err(|e| e.to_string())?;
}
let name = container_name(&self.s.label, &m.server);
let _ = Command::new(&self.s.engine).args(["rm", "-f", "-t", "0", &name]).output();
let out = Command::new(&self.s.engine).args(run_args(&self.s, m, &medium, &data)).output().map_err(|e| format!("{}: {e}", self.s.engine))?;
if !out.status.success() {
return Err(format!("[container-run] {}: {}", name, String::from_utf8_lossy(&out.stderr).trim()));
}
self.started.lock().unwrap().insert(m.server.clone(), name);
Ok(())
}
fn power_off(&self, server: &str, hard: bool) {
let name = container_name(&self.s.label, server);
let engine = self.s.engine.clone();
// The estate's lock is held: send the signal and return.
std::thread::spawn(move || {
let _ = if hard { Command::new(&engine).args(["kill", &name]).output() } else { Command::new(&engine).args(["stop", "-t", "20", &name]).output() };
});
}
fn running(&self, server: &str) -> Option<bool> {
let name = self.started.lock().unwrap().get(server).cloned()?;
let out = Command::new(&self.s.engine).args(["inspect", "-f", "{{.State.Running}}", &name]).output().ok()?;
Some(out.status.success() && String::from_utf8_lossy(&out.stdout).trim() == "true")
}
fn copy_disk(&self, from: &str, to: &str) -> Result<(), String> {
let mut media = self.media.lock().unwrap();
if let Some(b) = media.get(from).cloned() {
media.insert(to.to_string(), b);
}
Ok(())
}
fn store_medium(&self, storage: &str, bytes: &[u8]) -> Result<(), String> {
Medium::parse(bytes)?;
self.media.lock().unwrap().insert(storage.to_string(), bytes.to_vec());
Ok(())
}
fn metadata(&self, server: &str, user_data: Option<&str>) {
let mut u = self.user_data.lock().unwrap();
match user_data {
Some(t) => {
u.insert(server.to_string(), t.to_string());
}
None => {
u.remove(server);
}
}
}
fn forget_server(&self, server: &str) {
if let Some(name) = self.started.lock().unwrap().remove(server) {
let engine = self.s.engine.clone();
std::thread::spawn(move || {
let _ = Command::new(&engine).args(["rm", "-f", "-t", "2", &name]).output();
});
}
}
fn forget_storage(&self, storage: &str) {
self.media.lock().unwrap().remove(storage);
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
const P: Duration = Duration::from_secs(15);
const SOON: Duration = Duration::from_secs(1);
const LATE: Duration = Duration::from_secs(16);
/// ★ **A container stuck in `Stopping` is escalated, never waited on forever and never
/// counted too early** (2026-10-25, Gunnar's fast `terminate` under load: "1 container
/// left", the mock's alpine box in `Stopping`). Polled over time: kill (a stop is refused
/// there), wait out the bounded patience, remove, and remove again while the engine is
/// slow, up to [`SWEEP_REMOVES`] times; then done, or GiveUp.
#[test]
fn a_container_stuck_stopping_is_killed_then_removed_and_the_remove_retried() {
assert_eq!(next_sweep(Some("Stopping\n"), None, Duration::ZERO, P, 0), Sweep::Kill, "a stop is refused on a box already stopping: kill it");
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Kill), SOON, P, 0), Sweep::Wait, "the kill still has time");
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Kill), LATE, P, 0), Sweep::Remove, "a kill that did not end it is escalated");
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Remove), SOON, P, 1), Sweep::Wait);
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Remove), LATE, P, 1), Sweep::Remove, "a slow engine (load 1000, 2026-10-25) gets the remove again");
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Remove), LATE, P, SWEEP_REMOVES - 1), Sweep::Remove);
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Remove), LATE, P, SWEEP_REMOVES), Sweep::GiveUp, "bounded: then the run says what is left");
assert_eq!(next_sweep(Some("Exited (137) 1 second ago\n"), Some(Sweep::Kill), SOON, P, 0), Sweep::Remove, "killed: removed at once");
assert_eq!(next_sweep(Some(""), Some(Sweep::Remove), SOON, P, 1), Sweep::Done);
}
/// ★ **The teardown never masks a container that is truly left**: after stop, kill and
/// remove have each had their bounded time, the answer is GiveUp, so the receipt fails.
#[test]
fn a_container_left_after_every_bounded_step_gives_up_never_done() {
assert_eq!(next_sweep(Some("Up 3 minutes\n"), None, Duration::ZERO, P, 0), Sweep::Stop);
assert_eq!(next_sweep(Some("Up 3 minutes\n"), Some(Sweep::Stop), LATE, P, 0), Sweep::Kill);
assert_eq!(next_sweep(Some("Up 3 minutes\n"), Some(Sweep::Kill), LATE, P, 0), Sweep::Remove);
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Remove), SOON, P, 1), Sweep::Wait, "the remove still has time");
assert_eq!(next_sweep(Some("Stopping\n"), Some(Sweep::Remove), LATE, P, SWEEP_REMOVES), Sweep::GiveUp);
}
/// Containers that already exited need no stop: they are removed at once.
#[test]
fn exited_containers_are_removed_without_a_stop() {
assert_eq!(next_sweep(Some("Exited (0) 2 seconds ago\nCreated\n"), None, Duration::ZERO, P, 0), Sweep::Remove);
assert_eq!(next_sweep(Some("Exited (0) 3 seconds ago\n"), Some(Sweep::Stop), SOON, P, 0), Sweep::Remove, "stopped: removed without waiting out the patience");
assert_eq!(next_sweep(Some(""), None, Duration::ZERO, P, 0), Sweep::Done, "nothing to do");
}
/// `podman ps --format '{{.ID}} {{.Status}}'` reads as (id, status), blank lines dropped.
#[test]
fn the_teardown_reads_ids_and_statuses() {
let rows = parse_ps("a1b2 Up 3 minutes\nc3d4 Stopping\n\ne5f6 Exited (137) 2 seconds ago\n");
assert_eq!(rows, [("a1b2".to_string(), "Up 3 minutes".to_string()), ("c3d4".into(), "Stopping".into()), ("e5f6".into(), "Exited (137) 2 seconds ago".into())]);
assert!(run_is_live(&rows.iter().map(|(_, s)| s.as_str()).collect::<Vec<_>>().join("\n")));
}
/// A list that cannot be read is never "none left": it is polled again within the patience,
/// then the run gives up.
#[test]
fn an_unreadable_list_is_never_done() {
assert_eq!(next_sweep(None, None, SOON, P, 0), Sweep::Wait);
assert_eq!(next_sweep(None, Some(Sweep::Stop), SOON, P, 0), Sweep::Wait);
assert_eq!(next_sweep(None, Some(Sweep::Kill), LATE, P, 0), Sweep::GiveUp);
}
/// ★ **A run is live only while one of its containers is UP** (2026-10-25): a dead run's
/// container that exited (or was killed, 137) must not hold its network, or the slot stays
/// blocked; seen on builder-64 (fast602622: its process gone, one container `Exited (137)`).
#[test]
fn a_run_with_only_exited_containers_is_dead() {
assert!(!run_is_live("Exited (137) About a minute ago\n"), "killed and exited: dead");
assert!(!run_is_live(""), "no container: dead");
assert!(!run_is_live("Created\nExited (0) 3 hours ago\n"));
assert!(run_is_live("Exited (0) 1 hour ago\nUp 3 minutes\n"), "one container up: live");
assert!(run_is_live("Stopping\n"), "a container still stopping is not yet ours to remove");
}
/// ★ **Two parallel runs never share a container name** (2026-10-25): a mock's server uuids
/// are deterministic, so the name carries the run's label too.
#[test]
fn a_container_name_carries_its_run() {
assert_eq!(container_name("fast111", "00a1-b2"), "mu-fast111-00a1b2");
assert_ne!(container_name("fast111", "00a1-b2"), container_name("fast222", "00a1-b2"), "two runs, one deterministic uuid: two names");
assert!(container_name("a/b c", "x").starts_with("mu-abc-"), "the label is made a safe name");
}
/// ★ **A failed teardown never blocks its slot** (2026-10-25): a network a DEAD run left on
/// our subnet is found (its mock-upcloud label, no container of that run left) and named for
/// removal; a live run's network and an unlabelled one are never touched.
#[test]
fn a_dead_runs_network_on_our_subnet_is_named_for_removal() {
let ls = r#"[
{"name":"fast-fast3938361","subnets":[{"subnet":"10.213.8.0/21","gateway":"10.213.15.254"}],"labels":{"mock-upcloud-container":"fast3938361"}},
{"name":"fast-fast4000000","subnets":[{"subnet":"10.213.8.0/21"}],"labels":{"mock-upcloud-container":"fast4000000"}},
{"name":"fast-fast5000000","subnets":[{"subnet":"10.214.8.0/21"}],"labels":{"mock-upcloud-container":"fast5000000"}},
{"name":"someone-elses","subnets":[{"subnet":"10.213.8.0/21"}],"labels":{}},
{"name":"podman","subnets":[{"subnet":"10.88.0.0/16"}]}
]"#;
let live = |label: &str| label == "fast4000000";
let (stale, held) = stale_networks(ls, "10.213.8.0/21", "fast-me", live);
assert_eq!(stale, vec!["fast-fast3938361".to_string()], "only the dead run's network on our subnet");
assert_eq!(held, vec!["fast-fast4000000".to_string(), "someone-elses".to_string()], "a live run's and a foreign network are held, never removed");
let (stale, _) = stale_networks(ls, "10.213.8.0/21", "fast-fast3938361", live);
assert!(stale.is_empty(), "our own network is never ours to remove here");
assert_eq!(stale_networks("not json", "10.213.8.0/21", "x", live), (vec![], vec![]), "an unreadable list removes nothing");
}
/// ★ **Parallel fast runs** (2026-10-25): each slot takes its own utility `X.Y`, so up to
/// four runs share the rootless network namespace without one network over another. Slot 0
/// keeps today's choice; a slot never takes another slot's candidate.
#[test]
fn each_parallel_slot_takes_its_own_utility_network() {
let builder64 = " 10.13.10.7 \n";
assert_eq!(free_utility_for_slot(builder64, 0).as_deref(), Some("10.213"), "slot 0 is today's choice");
let all: Vec<String> = (0..FAST_SLOTS).map(|n| free_utility_for_slot(builder64, n).unwrap()).collect();
let distinct: std::collections::BTreeSet<&String> = all.iter().collect();
assert_eq!(distinct.len(), FAST_SLOTS as usize, "four slots, four networks: {all:?}");
let crowded = " 10.13.10.7 \n 10.213.9.1 \n";
let all: Vec<String> = (0..FAST_SLOTS).map(|n| free_utility_for_slot(crowded, n).unwrap()).collect();
assert_eq!(all.iter().collect::<std::collections::BTreeSet<_>>().len(), FAST_SLOTS as usize, "a held candidate never makes two slots share: {all:?}");
assert!(free_utility_for_slot(builder64, FAST_SLOTS).is_none(), "there are {FAST_SLOTS} slots");
}
use super::*;
pub(crate) fn machine(server: &str, hostname: &str) -> Machine {
Machine {
server: server.into(),
mem_mb: 2048,
cores: 1,
disks: Vec::new(),
medium: None,
cdrom_first: false,
disk_first: false,
rtc_skew_seconds: 0,
utility_mac: "de:ad:be:ef:00:01".into(),
utility_offer: crate::net::DhcpOffer::for_address("10.13.8.97"),
dhcp_late_first_boot_ms: 0,
hostname: hostname.into(),
ssh_keys: Vec::new(),
user_data: None,
public_ip: "127.80.0.11".into(),
public_admitted: Vec::new(),
}
}
fn medium() -> Medium {
Medium {
container_medium: 1,
program: "holger-mvp-metered".into(),
args: vec!["start".into(), "--config".into(), "/data/holger.ron".into()],
env: [("HOLGER_APPLIANCE".to_string(), "1".to_string())].into_iter().collect(),
files: [("holger.ron".to_string(), "(repositories: [])\n".to_string())].into_iter().collect(),
}
}
#[test]
fn a_container_medium_round_trips_and_an_iso_is_not_one() {
let m = medium();
assert_eq!(Medium::parse(&m.to_bytes()).unwrap(), Some(m));
assert_eq!(Medium::parse(b"\x00CD001 an iso").unwrap(), None);
assert_eq!(Medium::parse(b"plain text").unwrap(), None);
assert!(Medium::parse(br#"{"container_medium": 1, "program": 3}"#).unwrap_err().contains("[container-medium]"));
}
#[test]
fn a_box_runs_on_its_utility_address_with_its_own_tunnel_and_its_public_ports() {
let s = Settings {
root: "/w".into(),
network: "estate".into(),
image: "alpine:3".into(),
bin_dir: "/b".into(),
by_hostname: Vec::new(),
label: "run1".into(),
engine: "podman".into(),
utility: MOCK_UTILITY.into(),
};
let mut m = machine("00a1-b2", "holger-server-acme");
m.public_admitted = vec![("udp", 51821), ("tcp", 8474)];
let a = run_args(&s, &m, &medium(), Path::new("/w/servers/00a1-b2/data")).join(" ");
assert!(a.starts_with("run -d --init --name mu-run1-00a1b2 --label mock-upcloud-container=run1 --hostname holger-server-acme --network estate --ip 10.13.8.97 --mac-address de:ad:be:ef:00:01"), "{a}");
assert!(a.contains("--cap-add NET_ADMIN --device /dev/net/tun --cpus 1 --memory 2048m"), "the plan's shape: {a}");
assert!(a.contains("-p 127.80.0.11:51821:51821/udp -p 127.80.0.11:8474:8474/tcp"), "{a}");
assert!(a.ends_with("-e HOLGER_APPLIANCE=1 alpine:3 /opt/bin/holger-mvp-metered start --config /data/holger.ron"), "{a}");
m.public_ip = "198.51.100.3".into();
assert!(!run_args(&s, &m, &medium(), Path::new("/d")).join(" ").contains(" -p "), "a TEST-NET address is reached by nobody: nothing published");
}
#[test]
fn the_utility_network_covers_both_prefixes_of_the_pool() {
let mut s = Settings {
root: "/w".into(),
network: "estate".into(),
image: "alpine:3".into(),
bin_dir: "/b".into(),
by_hostname: Vec::new(),
label: "run1".into(),
engine: "podman".into(),
utility: MOCK_UTILITY.into(),
};
assert_eq!(network_args(&s).join(" "), "network create --subnet 10.13.8.0/21 --gateway 10.13.15.254 --label mock-upcloud-container=run1 estate");
s.utility = "10.213".into();
assert_eq!((s.subnet(), s.gateway()), ("10.213.8.0/21".to_string(), "10.213.15.254".to_string()), "another /21 where the host's own utility NIC holds 10.13.8.0/22");
let builder64 = "Main:\n +-- 0.0.0.0/0 3 0 5\n |-- 10.13.8.0\n /22 link UNICAST\n |-- 10.13.10.7\n /32 host LOCAL\n";
assert!(trie_holds(builder64, "10.13") && !trie_holds(builder64, "10.213"));
}
}