nordisk/edda

crates/storage-git-gix/parked-gix/.nornir/bench/forge-bakeoff.sh

#!/usr/bin/env bash
# forge-bakeoff — stock `git` against gunnar, gitea and Forgejo, same box.
#
# THE POINT IS FAIRNESS, NOT THE RESULT.
#
#   * One client: the host's stock `git`, which chooses every request. None of
#     the three servers is ever asked by its own client.
#   * One fixture per corpus, pushed to all three from the same bare repo, and
#     the resulting tip oid compared across servers before anything is scored.
#   * One instrument: `cargo xtask wireproxy` sits in front of each server, so
#     the byte counts come from the same read/write loop for all three. Whatever
#     it miscounts, it miscounts identically.
#   * Interleaved: round r runs gunnar, gitea, forgejo back to back, so a load
#     spike lands on all three rather than on whichever went first. `oden`'s
#     loadavg has run 2-218 today; a blocked run of three arms would be noise
#     shaped exactly like a result.
#   * Counted quantities first. Bytes and objects are counted; milliseconds are
#     sampled on a contended 32-core box and are reported, not relied on.
#
# LAW 2. Three independent ways this harness can go green having measured
# nothing, each with a guard that fires before any timing:
#
#   1. A server never came up. `preflight` demands a real `git ls-remote` from
#      every server and aborts the whole run if one fails. "0 bytes" and "no
#      server" are the same number without this.
#   2. A server came up and served the WRONG repository. `verify_seeded`
#      compares the tip oid every server reports against the fixture's, and
#      `score` compares each clone's `rev-list --objects | wc -l` against the
#      fixture's object count. A server that serves an empty repo is very fast.
#   3. The proxy silently stopped counting. Every op asserts its wire bytes are
#      non-zero and its connect-result column is `ok`; a zero row aborts.
#
# GUARD 3 COULD NOT ABORT, AND NEVER HAD (found and fixed 2026-08-04).
#
# `wire_read` is only ever called as `w=$(wire_read …)`, and `die` was
# `echo; exit 1`. `exit` inside `$( )` ends the SUBSHELL. So a tripped guard 3
# printed `FATAL:` to stderr and the script carried on with `w=""`, wrote a row
# with EMPTY `c2s_bytes`/`s2c_bytes` — which every consumer reads as 0 — and
# exited 0. A gunnar scoring 0 wire bytes wins every arm by an infinite ratio,
# which is the exact outcome the twelve lines above say cannot happen. Proven
# red on a four-line reduction before this was changed.
#
# Two independent fixes, because one of them is the one that was missing:
#   * `die` now signals the top-level shell (`kill -s TERM $TOP_PID`), so it
#     aborts from a command substitution as well as from the top level;
#   * `emit` refuses a row whose measurement columns are not non-negative
#     integers, so a blank reaches the TSV over the harness's dead body even if
#     some future call path escapes `die` again. That also catches the other
#     way this column can go blank: `wireproxy` writes its counter row when the
#     connection CLOSES, so a `wire_read` that beats the writer sees no row.
#
# Run:  .nornir/bench/forge-bakeoff.sh [rounds]
set -u -o pipefail

ROUNDS=${1:-5}
B=${BAKEOFF_DIR:-/home/rickard/scratch/bakeoff}
XTASK=${XTASK_BIN:-/home/rickard/scratch/cargo/gunnar-propaganda/release/xtask}
KEY=$B/keys/id_ed25519
OUT=$B/results.tsv
export GIT_SSH_COMMAND="ssh -i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=no"
export GIT_LFS_SKIP_SMUDGE=1
export GIT_TERMINAL_PROMPT=0

# name|direct-port|proxy-port|url-template ($R = repo name)
SERVERS=(
  "gunnar|12224|19001|ssh://git@127.0.0.1:PORT/bench/\$R.git"
  "gitea|12222|19002|ssh://git@127.0.0.1:PORT/bench/\$R.git"
  "forgejo|12223|19003|ssh://git@127.0.0.1:PORT/bench/\$R.git"
)
CORPORA=(small large)

# The pid of the script itself, captured before any subshell can shadow `$$`.
# `$$` is already subshell-proof in bash, but naming it makes the intent of the
# `kill` below unmistakable and survives a future `( )` refactor.
TOP_PID=$$
# Kill the top-level shell, not just whichever subshell noticed. See the GUARD 3
# note in the header: `exit` here reached only the command substitution, and a
# guard that cannot stop the run is not a guard.
trap 'stop_proxies 2>/dev/null; exit 1' TERM
die() { echo "FATAL: $*" >&2; kill -s TERM "$TOP_PID" 2>/dev/null; exit 1; }
field() { echo "$1" | cut -d'|' -f"$2"; }

url_for() { # server corpus -> url through the PROXY port
  local s=$1 r=$2 tpl port
  tpl=$(field "$s" 4); port=$(field "$s" 3)
  echo "${tpl/PORT/$port}" | sed "s/\\\$R/$r/"
}
url_direct() { local s=$1 r=$2; echo "$(field "$s" 4)" | sed "s/PORT/$(field "$s" 2)/;s/\\\$R/$r/"; }

# ── the counting relays ─────────────────────────────────────────────────────
start_proxies() {
  [ -x "$XTASK" ] || die "no xtask binary at $XTASK (cargo build --release -p xtask)"
  for s in "${SERVERS[@]}"; do
    local n lp dp; n=$(field "$s" 1); lp=$(field "$s" 3); dp=$(field "$s" 2)
    : > "$B/wire-$n.tsv"
    "$XTASK" wireproxy --listen "127.0.0.1:$lp" --to "127.0.0.1:$dp" --out "$B/wire-$n.tsv" \
      > "$B/proxy-$n.log" 2>&1 &
    echo $! >> "$B/proxy.pids"
  done
  sleep 1
  for s in "${SERVERS[@]}"; do
    local n; n=$(field "$s" 1)
    grep -q '^ready ' "$B/proxy-$n.log" || die "wireproxy for $n never printed ready: $(cat "$B/proxy-$n.log")"
  done
  echo "proxies up: $(for s in "${SERVERS[@]}"; do printf '%s:%s ' "$(field "$s" 1)" "$(field "$s" 3)"; done)"
}
stop_proxies() { [ -f "$B/proxy.pids" ] && xargs -r kill 2>/dev/null < "$B/proxy.pids"; rm -f "$B/proxy.pids"; }
trap stop_proxies EXIT

# Reset the per-op counter, then read it back. Aborts on a zero or non-ok row,
# which is guard 3.
wire_reset() { : > "$B/wire-$1.tsv"; }
wire_read() { # server label -> "c2s s2c"
  local n=$1 label=$2 f=$B/wire-$n.tsv bad rows c s
  # `grep -c` exits 1 when it matches nothing, so `$(... || echo 0)` prints
  # "0\n0" and the numeric test below blows up. Count with awk, which has one
  # exit status and one answer.
  bad=$(awk -F'\t' '$4!="ok"{n++} END{print n+0}' "$f")
  [ "$bad" -gt 0 ] && die "$label/$n: $bad relayed connection(s) did NOT connect: $(grep -v 'ok$' "$f" | head -2)"
  rows=$(awk 'END{print NR+0}' "$f")
  [ "$rows" -gt 0 ] || die "GUARD 3: $label/$n relayed NO connections at all — the client did not go through the proxy"
  c=$(awk -F'\t' '{t+=$1} END{print t+0}' "$f"); s=$(awk -F'\t' '{t+=$2} END{print t+0}' "$f")
  # A real git operation always moves bytes both ways. Zero in either direction
  # means the proxy stopped counting, and a zero that is charted as a result is
  # the failure this whole harness exists to avoid.
  { [ "$c" -gt 0 ] && [ "$s" -gt 0 ]; } || die "GUARD 3: $label/$n moved c2s=$c s2c=$s over $rows connection(s) — a git operation cannot be free"
  printf '%d %d' "$c" "$s"
}

# ── preflight: guard 1 ──────────────────────────────────────────────────────
preflight() {
  echo "── preflight ──"
  git --version | sed 's/^/  client: /'
  for s in "${SERVERS[@]}"; do
    local n u; n=$(field "$s" 1); u=$(url_direct "$s" fixture-probe)
    # An ls-remote against a repo that may not exist yet still proves the
    # server is listening, authenticating and speaking git: a dead port fails
    # differently (exit 128 with a connect error) from an absent repo.
    local err; err=$(git ls-remote "$u" 2>&1); local rc=$?
    if [ $rc -ne 0 ] && echo "$err" | grep -qiE 'connection refused|connect to host|timed out|no route'; then
      die "$n is NOT LISTENING on $(field "$s" 2): $err"
    fi
    echo "  $n: reachable on $(field "$s" 2)"
  done
}

# ── seeding, and guard 2 ────────────────────────────────────────────────────
seed() {
  echo "── seeding ──"
  for r in "${CORPORA[@]}"; do
    local fx=$B/fixtures/$r.git tip base
    tip=$(git -C "$fx" rev-parse refs/heads/main)
    # `base` is what a fetch client will hold, so the fetch negotiates for real
    # instead of cloning under a different name.
    base=$(git -C "$fx" rev-parse refs/heads/main~50 2>/dev/null || git -C "$fx" rev-list --max-parents=0 refs/heads/main | head -1)
    git -C "$fx" update-ref refs/heads/base "$base"
    for s in "${SERVERS[@]}"; do
      local n u; n=$(field "$s" 1); u=$(url_direct "$s" "$r")
      create_repo "$n" "$r"
      git -C "$fx" push -q --force "$u" refs/heads/main:refs/heads/main refs/heads/base:refs/heads/base \
        || die "seeding $n/$r failed"
      local got; got=$(git ls-remote "$u" refs/heads/main | cut -f1)
      [ "$got" = "$tip" ] || die "GUARD 2: $n/$r serves tip $got, fixture is $tip — a server serving the wrong repo is very fast"
      echo "  $n/$r seeded, tip $(echo "$tip" | cut -c1-8) confirmed"
    done
  done
}

create_repo() { # server corpus
  local n=$1 r=$2
  case $n in
    # gunnar's repositories are created by forge-setup.sh BEFORE `serve` starts:
    # the store lock is exclusive while it serves (S-019), so `repo create`
    # against a live data root correctly refuses.
    gunnar) : ;;
    gitea)  curl -s -o /dev/null -X POST -H "Authorization: token $(cat "$B/gitea.token")" -H 'Content-Type: application/json' \
              -d "{\"name\":\"$r\",\"private\":false,\"auto_init\":false}" http://localhost:13000/api/v1/user/repos ;;
    forgejo) curl -s -o /dev/null -X POST -H "Authorization: token $(cat "$B/forgejo.token")" -H 'Content-Type: application/json' \
              -d "{\"name\":\"$r\",\"private\":false,\"auto_init\":false}" http://localhost:13001/api/v1/user/repos ;;
  esac
}

# ── the operations ──────────────────────────────────────────────────────────
# `date +%s%3N` does NOT truncate on this box's coreutils — it returns full
# nanoseconds, and the first run produced differences like 1607256927688450388
# milliseconds. Divide explicitly.
now_ms() { echo $(( $(date +%s%N) / 1000000 )); }

# The last line of defence, and the one that would have caught GUARD 3's escape
# without knowing anything about `wire_read`: a measurement column that is not a
# non-negative integer never reaches the file.
#
# An empty `c2s_bytes` is read as 0 by `awk '{t+=$6}'`, by a TSV→iceberg ingest
# and by a spreadsheet, and 0 wire bytes is not a fast server, it is a server
# that was not measured. `LAW 2`: check the output, not the exit code.
emit() {
  local rd=$1 srv=$2 corp=$3 op=$4
  shift 4
  local names=(ms c2s_bytes s2c_bytes objects) i=0
  for v in "$@"; do
    case $v in
      ''|*[!0-9]*)
        # `die` first, because aborting the run is the right outcome; `return 1`
        # after it, because this guard must not depend on `die` reaching the
        # top-level shell — that dependency is the bug this whole block exists
        # to make impossible. Whichever of the two works, the row is not written.
        die "GUARD 3: $srv/$corp $op round $rd would have written ${names[$i]}='$v'. \
That column is not a number, and a blank one reads downstream as exactly the \
zero this harness exists to make impossible."
        return 1
        ;;
    esac
    i=$((i + 1))
  done
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$rd" "$srv" "$corp" "$op" "$1" "$2" "$3" "$4" >> "$OUT"
}

op_ls_remote() { # server corpus round
  local s=$1 r=$2 rd=$3 n; n=$(field "$s" 1)
  wire_reset "$n"; local t0 t1; t0=$(now_ms)
  local refs; refs=$(git ls-remote "$(url_for "$s" "$r")" 2>/dev/null); local rc=$?
  t1=$(now_ms)
  [ $rc -eq 0 ] || die "ls-remote failed on $n/$r"
  # GUARD 2 for this op, which was the one op without one.
  #
  # `echo "$refs" | wc -l` reports **1** for an empty advertisement, because
  # `echo ""` emits a newline. So a server answering with no refs at all scored
  # rc=0, a tiny byte count, a beautiful millisecond figure and `objects=1` —
  # a full row, and the fastest one in the table. `grep -c .` counts non-empty
  # lines and `awk` gives it one exit status, and the count is then compared
  # against the two refs `seed` put there.
  local got; got=$(printf '%s' "$refs" | awk 'NF{n++} END{print n+0}')
  [ "$got" -ge 2 ] || die "GUARD 2: ls-remote on $n/$r advertised $got ref(s); \
seed put refs/heads/main and refs/heads/base there, and a server that \
advertises nothing is very fast"
  local w; w=$(wire_read "$n" "ls-remote")
  emit "$rd" "$n" "$r" ls-remote "$((t1-t0))" "${w% *}" "${w#* }" "$got"
}

op_clone() {
  local s=$1 r=$2 rd=$3 n; n=$(field "$s" 1)
  local dst=$B/work/clone-$n-$r-$rd; rm -rf "$dst"
  wire_reset "$n"; local t0 t1; t0=$(now_ms)
  git clone -q --single-branch --branch main "$(url_for "$s" "$r")" "$dst" 2>/dev/null; local rc=$?
  t1=$(now_ms)
  [ $rc -eq 0 ] || die "clone failed on $n/$r"
  # GUARD 2: the clone must contain the fixture's object graph, not an empty one.
  local got want; got=$(git -C "$dst" rev-list --objects --all | wc -l)
  want=$(git -C "$B/fixtures/$r.git" rev-list --objects refs/heads/main | wc -l)
  [ "$got" -eq "$want" ] || die "GUARD 2: clone from $n/$r has $got objects, fixture main has $want"
  local w; w=$(wire_read "$n" "clone")
  emit "$rd" "$n" "$r" clone "$((t1-t0))" "${w% *}" "${w#* }" "$got"
  rm -rf "$dst"
}

op_fetch() { # a REAL negotiation: the client holds `base`, 50 commits behind
  local s=$1 r=$2 rd=$3 n; n=$(field "$s" 1)
  local dst=$B/work/fetch-$n-$r-$rd; rm -rf "$dst"
  git clone -q --single-branch --branch base "$(url_for "$s" "$r")" "$dst" 2>/dev/null || die "fetch setup clone failed on $n/$r"
  local have; have=$(git -C "$dst" rev-list --objects --all | wc -l)
  wire_reset "$n"; local t0 t1; t0=$(now_ms)
  git -C "$dst" fetch -q origin main 2>/dev/null; local rc=$?
  t1=$(now_ms)
  [ $rc -eq 0 ] || die "fetch failed on $n/$r"
  local after; after=$(git -C "$dst" rev-list --objects --all FETCH_HEAD | wc -l)
  [ "$after" -gt "$have" ] || die "GUARD 2: fetch from $n/$r added no objects ($have -> $after) — nothing was negotiated"
  local w; w=$(wire_read "$n" "fetch")
  emit "$rd" "$n" "$r" fetch "$((t1-t0))" "${w% *}" "${w#* }" "$((after-have))"
  rm -rf "$dst"
}

op_push() {
  local s=$1 r=$2 rd=$3 n; n=$(field "$s" 1)
  local dst=$B/work/push-$n-$r-$rd; rm -rf "$dst"
  git clone -q --single-branch --branch main "$(url_for "$s" "$r")" "$dst" 2>/dev/null || die "push setup clone failed on $n/$r"
  # One payload file, generated once, copied to all three: identical bytes, and
  # INCOMPRESSIBLE. The first version pushed 64 KiB of 'a', which deflates to
  # almost nothing and made every push arm measure the pkt-line overhead rather
  # than a transfer.
  cp "$B/push-payload.bin" "$dst/bakeoff.bin"
  git -C "$dst" add bakeoff.bin >/dev/null
  # Fixed dates as well as fixed content, so the commit OID is byte-identical on
  # all three servers. Without this the three pushes differ by a timestamp and
  # the comparison is of three slightly different objects.
  GIT_AUTHOR_DATE='2026-01-01T00:00:00+00:00' GIT_COMMITTER_DATE='2026-01-01T00:00:00+00:00' \
    git -C "$dst" -c user.email=b@b -c user.name=b commit -q -m "bakeoff $rd" >/dev/null
  wire_reset "$n"; local t0 t1; t0=$(now_ms)
  git -C "$dst" push -q --force origin "HEAD:refs/heads/bakeoff-$r-$rd" 2>/dev/null; local rc=$?
  t1=$(now_ms)
  [ $rc -eq 0 ] || die "push failed on $n/$r"
  local got; got=$(git ls-remote "$(url_direct "$s" "$r")" "refs/heads/bakeoff-$r-$rd" | cut -f1)
  local want; want=$(git -C "$dst" rev-parse HEAD)
  [ "$got" = "$want" ] || die "GUARD 2: push to $n/$r did not land ($got != $want)"
  local w; w=$(wire_read "$n" "push")
  emit "$rd" "$n" "$r" push "$((t1-t0))" "${w% *}" "${w#* }" 1
  rm -rf "$dst"
}

# ── run ─────────────────────────────────────────────────────────────────────
mkdir -p "$B/work"; rm -f "$B/proxy.pids"
printf 'round\tserver\tcorpus\top\tms\tc2s_bytes\ts2c_bytes\tobjects\n' > "$OUT"
echo "loadavg at start: $(cut -d' ' -f1-3 /proc/loadavg)  cores: $(nproc)"
preflight
start_proxies
seed
echo "── $ROUNDS interleaved rounds ──"
for rd in $(seq 1 "$ROUNDS"); do
  for r in "${CORPORA[@]}"; do
    for s in "${SERVERS[@]}"; do
      op_ls_remote "$s" "$r" "$rd"; op_clone "$s" "$r" "$rd"
      op_fetch "$s" "$r" "$rd";     op_push  "$s" "$r" "$rd"
    done
  done
  echo "  round $rd done (loadavg $(cut -d' ' -f1 /proc/loadavg))"
done
echo "loadavg at end: $(cut -d' ' -f1-3 /proc/loadavg)"
echo "results: $OUT ($(( $(wc -l < "$OUT") - 1 )) rows)"