nordisk/holger

scripts/flavour-k8s-cluster.sh

#!/usr/bin/env bash
# **The Kubernetes flavour, proven on a real local cluster** (lane OX-HOLFLAV, A31 §7.3 and §7.7).
#
#   scripts/flavour-k8s-cluster.sh [--node <name>] [--k3s-image <ref>] [--keep]
#
# Run from the repository root after `cargo xtask k8s image` (both images in local podman:
# holger.rs/nordisk/holger:<ver> and holger.rs/nordisk/holger-scan-worker:<ver>).
# The cluster is k3s in one rootful podman container (`sudo podman`), named --node
# (default holger-flavour-k3s); it is made when it is not running and removed at the end
# unless --keep. Nothing is pushed to any registry: the images are imported into the node.
#
# What it proves, each a PASS line (any FAIL exits 1):
#   1. the images: holger is one static binary FROM scratch, the default build without
#      `systemd` (no Skidbladnir crate and no D-Bus in its tree: server/cli/tests/
#      systemd_flavour.rs), and both images import into the node;
#   2. the chart with its DEFAULT values (only the TLS pair given, which it requires): the pod
#      runs both containers (holger + the scan-worker sidecar) and goes Ready on the real
#      /readyz (the sidecar answers);
#   3. upload, scan, pull: a second release whose RON is the default plus static tokens with
#      duties. A wheel is PUT into `incoming` (its ingest inspection runs in the sidecar),
#      walked through the intake chain to AVAILABLE, and pulled from `production` byte for byte;
#   4. it dials nothing: every NEW connection a pod opens is recorded per source address in
#      the node (iptables `recent` in mangle PREROUTING, before the NetworkPolicy can drop it);
#      neither Holger pod address is in the record after the whole run;
#   5. the sidecar from inside: the probe door's run count is 2 (the ingest scan and the probe
#      itself), uid 65533, no new privileges; the probe dials 1.1.1.1 on purpose, is refused,
#      and that dial IS in the record (the record works on a Holger pod).
set -euo pipefail

NODE=holger-flavour-k3s K3S_IMAGE=docker.io/rancher/k3s:v1.33.5-k3s1 KEEP=0
while [ $# -gt 0 ]; do
  case "$1" in
    --node) NODE=$2; shift 2 ;;
    --k3s-image) K3S_IMAGE=$2; shift 2 ;;
    --keep) KEEP=1; shift ;;
    *) echo "unknown argument $1" >&2; exit 2 ;;
  esac
done
ROOT=$(pwd)
CHART=$ROOT/deploy/helm/holger
[ -f "$CHART/Chart.yaml" ] || { echo "run from the holger repository root" >&2; exit 2; }
VER=$(awk -F'"' '/^\[workspace.package\]/{w=1} w && /^version *=/{print $2; exit}' Cargo.toml)
SERVER=holger.rs/nordisk/holger:$VER
WORKER=holger.rs/nordisk/holger-scan-worker:$VER
WORK=$(mktemp -d -t holger-k8s.XXXXXX)
API_PORT=${API_PORT:-16543}
export KUBECONFIG=$WORK/kubeconfig.yaml
FAILS=0
pass() { echo "PASS  $*"; }
fail() { echo "FAIL  $*"; FAILS=$((FAILS + 1)); }
step() { echo "-- $*"; }
node() { sudo podman exec -i "$NODE" "$@"; }
PF=
cleanup() {
  [ -n "$PF" ] && kill "$PF" 2>/dev/null || true
  if [ "$KEEP" = 0 ]; then sudo podman rm -f "$NODE" >/dev/null 2>&1 || true; sudo podman volume rm -f "$NODE-data" >/dev/null 2>&1 || true; fi
  rm -rf "$WORK"
}
trap cleanup EXIT

step "0 the cluster: k3s in podman ($NODE)"
if ! sudo podman container exists "$NODE"; then
  sudo podman run -d --name "$NODE" --hostname "$NODE" --privileged --cgroupns=private \
    -p 127.0.0.1:$API_PORT:6443 -v "$NODE-data":/var/lib/rancher/k3s \
    "$K3S_IMAGE" server --disable=traefik --disable=metrics-server --tls-san 127.0.0.1 >/dev/null
fi
for _ in $(seq 1 60); do node test -f /etc/rancher/k3s/k3s.yaml 2>/dev/null && break; sleep 2; done
node cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$API_PORT#" > "$KUBECONFIG"
for _ in $(seq 1 60); do kubectl get nodes 2>/dev/null | grep -q ' Ready ' && break; sleep 2; done
kubectl get nodes -o wide | sed 's/^/   /'
kubectl delete namespace holger-default holger-e2e --ignore-not-found --wait=true >/dev/null  # a kept cluster's last run
# The record of every new connection a pod opens, per source address (before any policy).
POD_CIDR=$(kubectl get node -o jsonpath='{.items[0].spec.podCIDR}')
node sh -c "iptables -t mangle -C PREROUTING -s $POD_CIDR -m conntrack --ctstate NEW -m recent --name oxegress --set --rsource 2>/dev/null || iptables -t mangle -I PREROUTING -s $POD_CIDR -m conntrack --ctstate NEW -m recent --name oxegress --set --rsource"
node sh -c 'echo / > /proc/net/xt_recent/oxegress'
pass "the egress record is on for $POD_CIDR"

step "1 the images into the node"
for img in "$SERVER" "$WORKER"; do
  if ! podman image exists "$img"; then fail "$img is not in local podman: run cargo xtask k8s image"; continue; fi
  podman save --format docker-archive "$img" | node ctr -n k8s.io images import - >/dev/null
  if node ctr -n k8s.io images ls -q | grep -qx "$img\|docker.io/$img"; then pass "imported $img"; else fail "the node does not list $img"; fi
done
podman image inspect "$SERVER" --format '{{len .RootFS.Layers}} layer(s), {{.Size}} B, user {{.Config.User}}' | sed 's/^/   holger image: /'

step "2 the chart with its default values"
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 2 -subj /CN=holger \
  -addext "subjectAltName=DNS:holger,DNS:localhost,IP:127.0.0.1" -keyout "$WORK/tls.key" -out "$WORK/tls.crt" 2>/dev/null
helm install holger "$CHART" -n holger-default --create-namespace \
  --set-file tls.cert="$WORK/tls.crt" --set-file tls.key="$WORK/tls.key" >/dev/null
if kubectl -n holger-default wait --for=condition=Ready pod -l app.kubernetes.io/name=holger --timeout=180s >/dev/null; then
  pass "the default release is Ready: $(kubectl -n holger-default get pod -l app.kubernetes.io/name=holger -o jsonpath='{.items[0].status.containerStatuses[*].name}' | tr ' ' '+') (the real /readyz)"
else
  fail "the default release never went Ready"; kubectl -n holger-default describe pod | tail -25 | sed 's/^/   | /'
fi
DEFAULT_IP=$(kubectl -n holger-default get pod -l app.kubernetes.io/name=holger -o jsonpath='{.items[0].status.podIP}')

step "3 upload, scan in the sidecar, pull"
values_ron() { awk '/^config:/{c=1;next} c&&/^  ron: \|/{r=1;next} r&&/^(    |$)/{sub(/^    /,"");print;next} r{exit}' "$CHART/values.yaml"; }
tok() { echo "$1-token-of-sufficient-length-for-holger"; }
values_ron | awk -v auth="    auth: Some((
        methods: [
            StaticToken(subject: \"ida\", token: Some(\"$(tok ida)\")),
            StaticToken(subject: \"bob\", token: Some(\"$(tok bob)\")),
            StaticToken(subject: \"adm\", token: Some(\"$(tok adm)\")),
            StaticToken(subject: \"dev\", token: Some(\"$(tok dev)\")),
        ],
        duties: { \"ida\": [Importer], \"bob\": [SecurityApprover], \"adm\": [RepositoryAdmin], \"dev\": [Developer] },
    ))," '{print} /^    audit: /{print auth}' > "$WORK/e2e.ron"
grep -q 'StaticToken(subject: "ida"' "$WORK/e2e.ron" || fail "the test RON did not take its auth block"
helm install holger "$CHART" -n holger-e2e --create-namespace \
  --set-file tls.cert="$WORK/tls.crt" --set-file tls.key="$WORK/tls.key" --set-file config.ron="$WORK/e2e.ron" >/dev/null
kubectl -n holger-e2e wait --for=condition=Ready pod -l app.kubernetes.io/name=holger --timeout=180s >/dev/null || fail "the e2e release never went Ready"
E2E_IP=$(kubectl -n holger-e2e get pod -l app.kubernetes.io/name=holger -o jsonpath='{.items[0].status.podIP}')
PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')
kubectl -n holger-e2e port-forward svc/holger "$PORT:8443" >/dev/null 2>&1 & PF=$!
B=https://127.0.0.1:$PORT
for _ in $(seq 1 30); do curl -sk -m 2 "$B/healthz" >/dev/null && break; sleep 1; done
c() { local who=$1; shift; curl -sk -m 30 -H "Authorization: Bearer $(tok "$who")" "$@"; }
WHEEL=demo_pkg-1.0.0-py3-none-any.whl
python3 - "$WORK/$WHEEL" <<'EOF'
import sys, zipfile
with zipfile.ZipFile(sys.argv[1], "w", zipfile.ZIP_STORED) as z:
    z.writestr("demo_pkg/__init__.py", '__version__ = "1.0.0"\n')
    z.writestr("demo_pkg-1.0.0.dist-info/METADATA", "Metadata-Version: 2.1\nName: demo-pkg\nVersion: 1.0.0\nLicense-Expression: MIT\n")
    z.writestr("demo_pkg-1.0.0.dist-info/WHEEL", "Wheel-Version: 1.0\nGenerator: holger-flavour\nRoot-Is-Purelib: true\nTag: py3-none-any\n")
EOF
runs() { # the probe door: the sidecar's run count and engine, or the refusal's words
  c adm "$B/-/sandbox/probe" | python3 -c '
import json, sys
t = sys.stdin.read()
try:
    v = json.loads(t)
    print(v.get("runs"), "|", v.get("engine"), "| uid", v.get("uid", "").split("\t")[0], "| no_new_privs", v.get("no_new_privs"), "| connect_public", v.get("connect_public"))
except ValueError:
    print("none |", t.strip()[:200])'
}
code=$(c ida -o /dev/null -w '%{http_code}' -X PUT --data-binary "@$WORK/$WHEEL" "$B/incoming/$WHEEL")
[ "$code" = 201 ] && pass "the Importer's PUT into incoming: 201" || fail "the PUT into incoming answered $code"
state=
for who in ida ida bob adm bob adm; do
  state=$(c "$who" -X POST "$B/-/intake/advance?name=$WHEEL&verb=advance" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("state"))' 2>/dev/null || echo "?")
  echo "   $who advances: $state"
done
[ "$state" = AVAILABLE ] && pass "walked through the intake chain to AVAILABLE" || fail "the chain stopped at $state"
c dev -o "$WORK/pulled.whl" "$B/production/$WHEEL"
if cmp -s "$WORK/$WHEEL" "$WORK/pulled.whl"; then pass "the Developer pulls it from production, byte for byte ($(sha256sum < "$WORK/$WHEEL" | cut -c1-16))"; else fail "the pull from production is not the upload"; fi
ingest=$(c adm "$B/-/intake/state?name=$WHEEL" | python3 -c '
import json, sys
v = json.load(sys.stdin)
h = v.get("history", [])
print(" -> ".join(t.get("to", "?") for t in h))
print("; ".join(e.get("source", "") + ": " + e.get("reference", "") for e in (h[0].get("evidence", []) if h else [])))' 2>/dev/null || true)
echo "   history: $(echo "$ingest" | sed -n 1p)"
echo "   ingest evidence: $(echo "$ingest" | sed -n 2p)"

step "4 it dials nothing"
sleep 20
seen=$(node cat /proc/net/xt_recent/oxegress | awk '{print $1}' | sed 's/^src=//' | sort -u)
echo "   pod sources that opened a connection: $(echo $seen)"
for ip in "$DEFAULT_IP" "$E2E_IP"; do
  if echo "$seen" | grep -qx "$ip"; then fail "the Holger pod $ip opened a connection"; else pass "the Holger pod $ip (holger + scan-worker) opened no connection"; fi
done

step "5 the sidecar seen from inside (and the record's control)"
# The probe door runs the worker's probe in the sidecar: it reports the sidecar's run count
# (the ingest scan plus this probe) and DELIBERATELY dials 1.1.1.1 and resolves a name, so
# the e2e pod's address must now be in the record: the record works on a Holger pod.
probe=$(runs); echo "   probe: $probe"
n=${probe%% *}
if [ "$n" = 2 ] && echo "$probe" | grep -q 'scan-worker sidecar' && echo "$probe" | grep -q 'uid 65533 | no_new_privs 1 | connect_public refused'; then
  pass "the ingest scan ran in the sidecar (runs 2: the scan and this probe; uid 65533, no new privileges, its deliberate dial refused)"
else
  fail "the sidecar did not account for the ingest scan: $probe"
fi
sleep 2
if node cat /proc/net/xt_recent/oxegress | grep -q "src=$E2E_IP "; then pass "the record works: the probe's deliberate dial from $E2E_IP is in it"; else fail "the record missed the probe's deliberate dial"; fi

echo "== $([ $FAILS = 0 ] && echo GREEN || echo "RED ($FAILS failed)") chart $(awk '/^version:/{print $2}' "$CHART/Chart.yaml") images $VER on $(kubectl version -o json 2>/dev/null | python3 -c 'import json,sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])')"
[ $FAILS = 0 ]