scripts/flavour-systemd-vm.sh
#!/usr/bin/env bash
# **The systemd flavour, proven in a fresh Ubuntu VM** (lane OX-HOLFLAV, A31 §7.3 "exe").
#
# scripts/flavour-systemd-vm.sh --holger <bin> --toolchain <tar.gz> --base <ubuntu cloud image> [--work <dir>] [--keep]
#
# --holger the binary `cargo install holger` left in ~/.cargo/bin (the default build)
# --toolchain a rust toolchain (bin/cargo, bin/rustc, lib/) as a tar.gz, so the guest runs REAL cargo
# --base an Ubuntu cloud image (qcow2); the guest is a throwaway overlay on it
#
# What it proves, each a PASS line (any FAIL exits 1):
# 1. the operator's steps on a fresh host: the binary in ~/.cargo/bin, then
# `sudo ~/.cargo/bin/holger install add`: the unit is active and enabled, /healthz answers,
# it listens on loopback only;
# 2. a local upload and a pull: `cargo publish` of a crate to the `sparring` registry, then a
# second crate that depends on it resolves, fetches (byte for byte) and checks against it;
# 3. a reboot: the unit comes back by itself and serves the same crate to a fresh cargo home;
# 4. it dials nothing: the guest logs every new outbound connection with its uid (nftables
# `log flags skuid`, kept across the reboot) and every DNS ask to the local resolver; not
# one line carries the holger user's uid, while a deliberate dial by the lab user is logged.
# The guest's only NIC is QEMU user networking with `restrict=on` (no route out at all) plus the
# ssh port forward; the log counts what the service TRIED, not what got through.
set -euo pipefail
HOLGER= TOOLCHAIN= BASE= WORK= KEEP=0
while [ $# -gt 0 ]; do
case "$1" in
--holger) HOLGER=$2; shift 2 ;;
--toolchain) TOOLCHAIN=$2; shift 2 ;;
--base) BASE=$2; shift 2 ;;
--work) WORK=$2; shift 2 ;;
--keep) KEEP=1; shift ;;
*) echo "unknown argument $1" >&2; exit 2 ;;
esac
done
[ -f "$HOLGER" ] && [ -f "$TOOLCHAIN" ] && [ -f "$BASE" ] || { sed -n '4,8p' "$0"; exit 2; }
WORK=${WORK:-$(mktemp -d -t holger-systemd-vm.XXXXXX)}
mkdir -p "$WORK"
SSH_PORT=${SSH_PORT:-22231}
FAILS=0
pass() { echo "PASS $*"; }
fail() { echo "FAIL $*"; FAILS=$((FAILS + 1)); }
step() { echo "-- $*"; }
ssh-keygen -q -t ed25519 -N '' -f "$WORK/key" <<< y >/dev/null 2>&1 || true
cat > "$WORK/user-data" <<EOF
#cloud-config
hostname: holger-systemd
users:
- name: lab
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash
ssh_authorized_keys: ["$(cat "$WORK/key.pub")"]
package_update: false
package_upgrade: false
EOF
echo "instance-id: holger-systemd-$(date +%s)" > "$WORK/meta-data"
cloud-localds "$WORK/seed.iso" "$WORK/user-data" "$WORK/meta-data"
qemu-img create -q -f qcow2 -F qcow2 -b "$(readlink -f "$BASE")" "$WORK/disk.qcow2" 12G
boot() {
qemu-system-x86_64 -name holger-systemd -enable-kvm -cpu host -smp 2 -m 2048 -nographic \
-serial file:"$WORK/serial-$1.log" -monitor none \
-drive file="$WORK/disk.qcow2",if=virtio -drive file="$WORK/seed.iso",media=cdrom \
-netdev user,id=n,restrict=on,hostfwd=tcp:127.0.0.1:$SSH_PORT-:22 -device virtio-net-pci,netdev=n \
> "$WORK/qemu-$1.log" 2>&1 &
echo $! > "$WORK/pid"
}
vm() { ssh -q -i "$WORK/key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -p "$SSH_PORT" lab@127.0.0.1 "$@"; }
put() { scp -q -i "$WORK/key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$SSH_PORT" "$1" "lab@127.0.0.1:$2"; }
wait_ssh() {
for _ in $(seq 1 120); do vm true 2>/dev/null && return 0; sleep 3; done
echo "the guest never answered ssh (serial: $WORK/serial-*.log)"; return 1
}
cleanup() {
[ -f "$WORK/pid" ] && kill "$(cat "$WORK/pid")" 2>/dev/null || true
[ "$KEEP" = 1 ] || rm -f "$WORK/disk.qcow2"
}
trap cleanup EXIT
step "0 a fresh Ubuntu guest"
boot 1
wait_ssh
vm 'cloud-init status --wait >/dev/null 2>&1 || true; . /etc/os-release; echo " guest: $PRETTY_NAME, $(uname -r)"'
step "1 the egress log (every new outbound connection, with its uid, across reboots)"
vm 'sudo tee /etc/nftables.conf >/dev/null' <<'EOF'
#!/usr/sbin/nft -f
flush ruleset
table inet oxegress {
chain out {
type filter hook output priority 0; policy accept;
oifname "lo" meta l4proto { tcp, udp } th dport 53 ct state new log prefix "OXEGRESS dns " flags skuid
oifname "lo" accept
ct state new log prefix "OXEGRESS " flags skuid
}
}
EOF
# A default route into the dead end (`restrict=on` drops what takes it), so a dial to the
# internet leaves the socket and is logged, instead of failing with "no route" before it.
vm 'sudo bash -s' <<'EOF'
dev=$(ip -o -4 route show 10.0.2.0/24 | awk '{print $3}')
printf 'network:\n version: 2\n ethernets:\n %s:\n routes: [{to: default, via: 10.0.2.2}]\n' "$dev" > /etc/netplan/60-oxegress.yaml
chmod 600 /etc/netplan/60-oxegress.yaml
netplan apply 2>/dev/null
EOF
if vm 'sudo systemctl enable --now nftables >/dev/null 2>&1 && sudo nft list table inet oxegress >/dev/null && ip route show default | grep -q 10.0.2.2'; then
pass "nftables logs new outbound connections and loopback DNS asks with their uid; the default route leads nowhere"
else
fail "nftables could not load the egress log, or there is no default route"
fi
step "2 cargo install holger (its binary), then holger install add"
put "$HOLGER" /tmp/holger
vm 'install -D -m 0755 /tmp/holger ~/.cargo/bin/holger && rm /tmp/holger'
put "$TOOLCHAIN" /tmp/toolchain.tar.gz
vm 'mkdir -p ~/rust && tar -C ~/rust -xzf /tmp/toolchain.tar.gz && rm /tmp/toolchain.tar.gz && ~/rust/bin/cargo -V' | sed 's/^/ /'
vm 'sudo ~/.cargo/bin/holger install add' 2>&1 | sed 's/^/ | /' || true
if [ "$(vm 'systemctl is-active holger' 2>/dev/null)" = active ]; then pass "holger.service is active"; else
fail "holger.service is not active"; vm 'sudo journalctl -u holger --no-pager -n 30' | sed 's/^/ | /'; fi
if [ "$(vm 'systemctl is-enabled holger' 2>/dev/null)" = enabled ]; then pass "holger.service is enabled at boot"; else fail "holger.service is not enabled"; fi
health() {
for _ in $(seq 1 60); do
h=$(vm 'curl -fsS -m 2 http://127.0.0.1:18444/healthz' 2>/dev/null) && { echo "$h"; return 0; }
sleep 1
done
return 1
}
if h=$(health); then pass "/healthz answers: $h"; else fail "/healthz never answered"; fi
listen=$(vm "sudo ss -Htlnp | grep holger-server | awk '{print \$4}' | sort -u | tr '\n' ' '")
if [ -n "$listen" ] && ! echo "$listen" | grep -qvE '^(127\.0\.0\.1:[0-9]+ ?)+$'; then
pass "it listens on loopback only: $listen"
else
fail "it listens beyond loopback: $listen"
fi
step "3 a local upload and a pull with real cargo"
vm 'bash -s' <<'EOF'
set -e
export PATH=$HOME/rust/bin:$PATH CARGO_HOME=$HOME/cargo-home CARGO_REGISTRIES_SPARRING_TOKEN=lab-local
mkdir -p $CARGO_HOME ~/w/holflav-demo/src ~/w/app/src
cat > $CARGO_HOME/config.toml <<T
[registries.sparring]
index = "sparse+http://127.0.0.1:18444/sparring/index/"
[net]
retry = 0
T
cat > ~/w/holflav-demo/Cargo.toml <<T
[package]
name = "holflav-demo"
version = "0.1.0"
edition = "2021"
description = "a crate uploaded to a systemd Holger"
license = "MIT"
publish = ["sparring"]
T
echo 'pub fn answer() -> u32 { 42 }' > ~/w/holflav-demo/src/lib.rs
cat > ~/w/app/Cargo.toml <<T
[package]
name = "app"
version = "0.1.0"
edition = "2021"
publish = false
[dependencies]
holflav-demo = { version = "0.1.0", registry = "sparring" }
T
echo 'fn main() { println!("{}", holflav_demo::answer()); }' > ~/w/app/src/main.rs
EOF
if vm 'cd ~/w/holflav-demo && PATH=$HOME/rust/bin:$PATH CARGO_HOME=$HOME/cargo-home CARGO_REGISTRIES_SPARRING_TOKEN=lab-local cargo publish --registry sparring --allow-dirty 2>&1 | tail -3' | sed 's/^/ | /'; then :; fi
pull() { # $1 = cargo home
vm "cd ~/w/app && rm -f Cargo.lock && PATH=\$HOME/rust/bin:\$PATH CARGO_HOME=$1 cargo fetch 2>&1 | tail -2 && PATH=\$HOME/rust/bin:\$PATH CARGO_HOME=$1 cargo check --offline 2>&1 | tail -1"
}
sums() { # $1 = cargo home: the published .crate and the fetched one
vm "sha256sum ~/w/holflav-demo/target/package/holflav-demo-0.1.0.crate $1/registry/cache/*/holflav-demo-0.1.0.crate 2>/dev/null | awk '{print \$1}' | sort -u | wc -l"
}
pull '$HOME/cargo-home' | sed 's/^/ | /' || true
n=$(vm 'ls $HOME/cargo-home/registry/cache/*/holflav-demo-0.1.0.crate 2>/dev/null | wc -l')
if [ "$n" = 1 ] && [ "$(sums '$HOME/cargo-home')" = 1 ]; then
pass "cargo publish, then cargo fetch + check of a crate that depends on it: the same bytes"
else
fail "the published crate did not come back through cargo"
fi
step "4 a reboot"
vm 'sudo systemctl reboot' || true
sleep 10
for _ in $(seq 1 60); do vm true 2>/dev/null || break; sleep 2; done
kill "$(cat "$WORK/pid")" 2>/dev/null || true; sleep 1
boot 2
wait_ssh
if [ "$(vm 'systemctl is-active holger' 2>/dev/null)" = active ]; then pass "after the reboot holger.service is active by itself"; else fail "holger.service did not come back after the reboot"; fi
if h=$(health); then pass "after the reboot /healthz answers: $h"; else fail "after the reboot /healthz never answered"; fi
vm 'mkdir -p ~/cargo-home-2 && cp ~/cargo-home/config.toml ~/cargo-home-2/'
pull '$HOME/cargo-home-2' | sed 's/^/ | /' || true
if [ "$(sums '$HOME/cargo-home-2')" = 1 ] && vm 'ls ~/cargo-home-2/registry/cache/*/holflav-demo-0.1.0.crate >/dev/null 2>&1'; then
pass "after the reboot a fresh cargo home pulls the same crate"
else
fail "after the reboot the crate was not served"
fi
step "5 it dials nothing"
vm 'curl -s -m 3 http://192.0.2.10/ >/dev/null 2>&1; true'
sleep 1
uid=$(vm 'id -u holger' 2>/dev/null || true)
[ -n "$uid" ] || { fail "there is no holger user: the install never ran"; uid=none; }
lab=$(vm 'id -u')
log=$(vm 'sudo journalctl _TRANSPORT=kernel --no-pager -o cat | grep OXEGRESS || true')
boots=$(vm 'sudo journalctl --list-boots --no-pager | tail -n +2 | wc -l')
mine=$(printf '%s\n' "$log" | grep -c " UID=$uid " || true)
control=$(printf '%s\n' "$log" | grep -c " UID=$lab " || true)
total=$(printf '%s\n' "$log" | grep -c OXEGRESS || true)
if [ "$control" -ge 1 ]; then pass "the log works: the lab user's deliberate dial is logged ($control line(s); $total in all, $boots boots)"; else fail "the egress log missed a deliberate dial"; fi
if [ "$mine" = 0 ]; then pass "holger (uid $uid) opened no outbound connection and asked no DNS name, over every boot"; else
fail "holger dialled out $mine time(s)"; printf '%s\n' "$log" | grep " UID=$uid " | head -5 | sed 's/^/ | /'; fi
vm 'sudo journalctl -u holger --no-pager -b 0 | tail -3' | sed 's/^/ | /' || true
echo "== $([ $FAILS = 0 ] && echo GREEN || echo "RED ($FAILS failed)") holger $(sha256sum "$HOLGER" | cut -c1-16) on $(basename "$BASE")"
[ $FAILS = 0 ]