installtest/src/pair.rs
//! **A PAIR of appliances, both alive at once** — the rig the replication half
//! of `.nornir/gunnar-quickstart.md` is written against.
//!
//! # Why this exists
//!
//! Every KVM door in this crate boots ONE guest. `firmware::Door` installs a
//! medium and proves the box serves; `mother_kvm::Rig` installs one and drives
//! a bench against it. Neither can express the only question replication has:
//! *does a write that lands on A appear on B, and does `promote` on B make B
//! the box that takes writes?* That needs two appliances up **at the same
//! time**, on a network where each can dial the other.
//!
//! Until this module the answer was "install two by hand", and the quick start's
//! replication section was therefore the one part of the document nobody had
//! typed. A guide written from a `--help` output is a guess with citations.
//!
//! # The network, which is the whole trick
//!
//! Both guests are on QEMU's user-mode network (slirp), so they cannot see each
//! other directly — but each one CAN reach the host through its gateway at
//! `10.0.2.2`, and the host holds a forwarded port for each guest's git-ssh and
//! control plane. So the primary reaches the twin at
//! `ssh://git@10.0.2.2:<twin's forwarded ssh port>`, which is exactly the shape
//! a real deployment has (a name that resolves to somewhere else), reached
//! through the one address a slirp guest is guaranteed to have.
//!
//! Nothing here is bridged and nothing needs root.
//!
//! # What it does, in order
//!
//! 1. installs the declared `gunnar-installer-open` medium onto TWO blank disks —
//! twice, not once-and-copied, so each appliance mints its own SSH host key
//! and the pair is as separate as two machines;
//! 2. boots both off their own disks and waits for each to say it is serving;
//! 3. prints the forwarded ports, and the `10.0.2.2:<port>` address each guest
//! reaches the other by;
//! 4. then either runs `--drive-cmd` with those ports as argv, or — with
//! `--keep` — stands still so a person can type at them, until a `stop` file
//! appears in the rig directory.
//!
//! Nothing in here builds a medium. `cargo xtask iso-installer --variant
//! gunnar-installer-open --build kernel` does that, and a missing one is said by
//! name rather than worked around.
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{Duration, Instant};
use crate::mother_kvm::{Guest, MotherConfig, Rig, CELL};
/// The gateway a slirp guest reaches the host by. Fixed by QEMU's user-mode
/// network, and the reason this rig needs no bridge and no root.
pub const SLIRP_GATEWAY: &str = "10.0.2.2";
/// How long a guest is given to say it serves. An installed appliance on this
/// box reaches it in tens of seconds; the ceiling is for a loaded machine.
pub const BOOT_TIMEOUT: Duration = Duration::from_secs(420);
/// One member of the pair, and the addresses that reach it.
pub struct Member {
/// What the printed rows and the rig directory call it — `main`, `twin`,
/// and whatever else was asked for.
pub name: String,
pub guest: Guest,
}
impl Member {
/// Where a person at a terminal on THIS host reaches its control plane.
///
/// The medium binds the control plane on the second NIC (`--grpc-bind
/// eth1:50051`), so the second forward is the one that answers; the first is
/// tried as a fallback for a medium that binds differently, exactly as
/// [`Rig::bootstrap`] does.
pub fn control_ports(&self) -> Vec<u16> {
let mut p = self.guest.grpc_ports.clone();
p.reverse();
p
}
/// Where the OTHER guest reaches this one's git-ssh: the host, through
/// slirp's gateway.
pub fn peer_ssh_url(&self) -> String {
format!("ssh://git@{SLIRP_GATEWAY}:{}", self.guest.ssh_port)
}
}
/// What the operator asked for.
pub struct Ask {
/// **Who is in the rig**, in order. `main,twin` by default — the pairing.
///
/// A third (`main,twin,spare`) is what the NUKE case needs: the drill is
/// "the main is gone, install a fresh box, make it a standby of the
/// survivor", and a fresh box cannot be installed after the main has died
/// unless one was laid down beforehand. The spare is installed and booted
/// like the others and then simply not spoken to until it is needed, which
/// is exactly what a cold spare is.
pub members: Vec<String>,
/// The rig directory; each member gets a subdirectory of it.
pub kv: PathBuf,
pub iso: PathBuf,
pub mem_mb: u32,
pub vcpus: u32,
/// Host cores to pin every guest task to. Empty pins nothing, which is the
/// default here — this rig is a correctness proof, not a measurement, and a
/// pin that names cores the box does not have is a silent failure.
pub cores: Vec<usize>,
/// Run this once both guests answer, as
/// `PATH <main_ssh> <main_control> <twin_ssh> <twin_control>`.
pub drive_cmd: Option<PathBuf>,
/// Stand still afterwards so a person can type at the pair.
pub keep: bool,
}
impl Ask {
/// Defaults, overridable by environment for the same reason
/// [`MotherConfig::from_env`] is: a knob typed into a shell is a knob that
/// is not in the record.
pub fn from_env(repo: &Path) -> Self {
let env = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty());
Self {
members: env("GUNNAR_PAIR_MEMBERS")
.unwrap_or_else(|| "main,twin".into())
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect(),
kv: env("GUNNAR_PAIR_KV")
.map(PathBuf::from)
.unwrap_or_else(|| crate::expand_home("~/scratch/gunnar-pair")),
iso: env("GUNNAR_PAIR_ISO")
.map(PathBuf::from)
.unwrap_or_else(|| repo.join(".iso-out").join(format!("{CELL}.iso"))),
mem_mb: env("GUNNAR_PAIR_MEM_MB")
.and_then(|v| v.parse().ok())
.unwrap_or(4096),
vcpus: env("GUNNAR_PAIR_VCPUS")
.and_then(|v| v.parse().ok())
.unwrap_or(2),
cores: env("GUNNAR_PAIR_CORES")
.map(|s| crate::mother_kvm::parse_cores(&s).unwrap_or_default())
.unwrap_or_default(),
drive_cmd: None,
keep: false,
}
}
}
/// The configuration for one member.
///
/// Written out rather than taken from [`MotherConfig::from_env`]: that reader
/// belongs to the bench column and carries its shapes, its repetitions and its
/// core pin (`24-27`, which is a statement about one machine). This rig wants
/// none of them, and inheriting knobs it does not use is how a proof quietly
/// starts depending on a bench's environment.
fn member_cfg(ask: &Ask, name: &str) -> MotherConfig {
MotherConfig {
iso: ask.iso.clone(),
// Separate trees, because two appliances sharing a disk image is not two
// appliances.
kv: ask.kv.join(name),
// Unused here: this rig never bootstraps for you. The quick start's
// whole point is that a PERSON types `gunnar control bootstrap`, so the
// rig stops at "both boxes answer".
gunnar_bin: PathBuf::new(),
reps: 0,
vcpus: ask.vcpus,
mem_mb: ask.mem_mb,
cores: ask.cores.clone(),
repos: Vec::new(),
label: name.to_string(),
out_dir: ask.kv.join(name),
}
}
/// Install and boot both appliances, then hand them to the caller.
///
/// Returns them in `[main, twin]` order. The [`Guest`]s kill their qemu on drop,
/// so the pair lives exactly as long as the returned value.
pub fn bring_up(repo: &Path, ask: &Ask) -> Result<Vec<Member>, String> {
if !ask.iso.is_file() {
return Err(format!(
"no medium at {} — build it first: `cargo xtask iso-installer --variant {CELL} \
--opt dhcp=on --build kernel`",
ask.iso.display()
));
}
std::fs::create_dir_all(&ask.kv).map_err(|e| format!("create {}: {e}", ask.kv.display()))?;
let mut out = Vec::new();
for name in &ask.members {
let cfg = member_cfg(ask, name);
let rig = Rig::open(repo, cfg)?;
let t0 = Instant::now();
let disk = rig.install()?;
eprintln!("[pair] {name}: installed in {} s", t0.elapsed().as_secs());
let t1 = Instant::now();
let guest = rig.boot(&disk, BOOT_TIMEOUT)?;
eprintln!(
"[pair] {name}: serving after {} s — ssh 127.0.0.1:{}, control {:?}",
t1.elapsed().as_secs(),
guest.ssh_port,
guest.grpc_ports
);
out.push(Member {
name: name.clone(),
guest,
});
}
Ok(out)
}
/// The rows a person reads off the terminal and types into the next command.
pub fn render(pair: &[Member]) -> String {
let mut s = String::new();
s.push_str("\npair: two appliances, both up\n\n");
s.push_str(
" member git-ssh (from here) control (from here) the OTHER box reaches it at\n",
);
for m in pair {
let control = m
.control_ports()
.iter()
.map(|p| format!("127.0.0.1:{p}"))
.collect::<Vec<_>>()
.join(" ");
s.push_str(&format!(
" {:<6} 127.0.0.1:{:<10} {:<20} {}\n",
m.name,
m.guest.ssh_port,
control,
m.peer_ssh_url()
));
}
s.push_str("\n serial logs:\n");
for m in pair {
s.push_str(&format!(
" {:<6} {}\n",
m.name,
m.guest.serial_log.display()
));
}
s
}
/// The operator verb. Brings the pair up, prints the rows, then either drives a
/// program against them or stands still until told to stop.
pub fn run(repo: &Path, ask: &Ask) -> Result<(), String> {
let pair = bring_up(repo, ask)?;
print!("{}", render(&pair));
if let Some(cmd) = &ask.drive_cmd {
// `PATH <name> <ssh> <control> …`, one triple per member in rig order —
// so a driver reads its argv positionally and a three-member rig does
// not need a second calling convention.
let mut c = Command::new(cmd);
for m in &pair {
c.arg(&m.name)
.arg(m.guest.ssh_port.to_string())
.arg(m.control_ports()[0].to_string());
}
let status = c
.status()
.map_err(|e| format!("run {}: {e}", cmd.display()))?;
if !status.success() {
return Err(format!("--drive-cmd {} exited {status}", cmd.display()));
}
}
if ask.keep {
let stop = ask.kv.join("stop");
let _ = std::fs::remove_file(&stop);
println!(
"\n the pair stays up. To take it down: touch {}\n",
stop.display()
);
while !stop.exists() {
std::thread::sleep(Duration::from_millis(500));
}
println!("pair: stop file seen; powering both down");
}
Ok(())
}