scripts/kvm-scenario.sh
#!/usr/bin/env bash
# The tunnr-etcd main+twin KVM scenario: install, serve, replicate, crash, restart, clean stop.
# Needs the two ISOs in .iso-out/ and PROBE=<the probe example> (cargo build --example probe).
set -uo pipefail
HERE=$(cd "$(dirname "$0")" && pwd); K=$HERE/kvm.sh; P=${PROBE:-probe}; M=http://127.0.0.1:23790; T=http://127.0.0.1:23791; V=${W:-$HERE/../.kvm}
ok() { echo "PASS $*"; }; bad() { echo "FAIL $*"; FAILS=$((${FAILS:-0}+1)); }
now() { date +%s.%N; }
el() { python3 -c "print('%.1f' % ($(now) - $1))"; }
wait_serving() { # role n -> seconds from qemu start to first answered request
local r=$1 n=$2 t0; t0=$(now); $K boot $r $n
for _ in $(seq 1 240); do
if $P $([ $r = main ] && echo $M || echo $T) rev >/dev/null 2>&1; then echo "$(el $t0)"; return 0; fi; sleep 0.25; done
echo timeout; return 1; }
check() { [ "$2" = "$3" ] && ok "$1 ($2)" || bad "$1: got '$2' want '$3'"; }
count() { $P $1 count $2 2>/dev/null | awk '{print $1}'; }
for r in main twin; do t0=$(now); n=$($K install $r | tail -1); check "install $r in $(el $t0) s" "$n" 1; done
echo "boot twin: serving after $(wait_serving twin 1) s"
echo "boot main: serving after $(wait_serving main 1) s"
grep -a -q "signature-over-root=PASS" $V/main-boot1.log && ok "main startup self-test (blake3 + ed25519)" || bad "main self-test"
grep -a -q "signature-over-root=PASS" $V/twin-boot1.log && ok "twin startup self-test (blake3 + ed25519)" || bad "twin self-test"
sleep 2
($P $T watch /t/ 100 30 > $V/watch-twin.log 2>&1 &); sleep 0.5
t0=$(now); $P $M fill /t/ 100 >/dev/null; echo "100 puts main→twin (sync ack) in $(el $t0) s"
check "twin has the 100 keys" "$(count $T /t/)" 100
sleep 1; check "twin watch saw 100 events" "$(tail -1 $V/watch-twin.log | awk '{print $2}')" 100
$P $T put /x y >/dev/null 2>&1 && bad "twin accepted a write" || ok "twin refuses writes"
echo "host RSS per VM (KiB): main $(ps -o rss= -p $(cat $V/main.pid)) twin $(ps -o rss= -p $(cat $V/twin.pid))"
# twin power cut, main keeps writing, twin comes back and catches up
kill -9 $(cat $V/twin.pid); sleep 1
$P $M fill /down/ 50 >/dev/null
echo "twin back: serving after $(wait_serving twin 2) s"
for _ in $(seq 1 20); do [ "$(count $T /down/)" = 50 ] && break; sleep 0.5; done
check "twin caught up after its power cut" "$(count $T /down/)" 50
# main power cut, restart, the twin must take the restarted main
kill -9 $(cat $V/main.pid); sleep 1
echo "main back: serving after $(wait_serving main 2) s"
check "main kept every acked write over the power cut" "$(count $M /t/)" 100
check "main kept the writes made while the twin was down" "$(count $M /down/)" 50
$P $M put /after-crash 1 >/dev/null
for _ in $(seq 1 20); do $P $T get /after-crash 2>/dev/null | grep -q "= 1" && break; sleep 0.5; done
$P $T get /after-crash | grep -q "= 1" && ok "the restarted main re-attached its twin" || bad "the twin never took the restarted main"
# clean stop: the ACPI power button, then boot again
$K powerdown main; for _ in $(seq 1 40); do kill -0 $(cat $V/main.pid) 2>/dev/null || break; sleep 0.25; done
kill -0 $(cat $V/main.pid) 2>/dev/null && bad "main did not power off on the power button" || ok "main powered off on the ACPI power button"
grep -a "stopping (" $V/main-boot2.log | tail -1
echo "main again: serving after $(wait_serving main 3) s"
grep -a -q "Ending clean mount" $V/main-boot3.log && ok "XFS clean mount after the clean stop (no log recovery)" || bad "XFS needed recovery after a clean stop"
check "data intact after the clean stop" "$(count $M /t/)" 100
echo "revisions: main $($P $M rev) / twin $($P $T rev)"
echo "FAILS=${FAILS:-0}"