ci/runner.sh
#!/bin/bash
# ci-runner, on builder-64 (cron: every 5 minutes with --quick, nightly with --nightly).
# It polls the local mirrors that ci-mirror fills from dwarf, checks every repo out side
# by side under /work/ci/src, and on a new head calls the repo's own verb,
# `cargo xtask ci --quick` (all CI repos with --nightly). The constellation lock check and
# the status page are the runner's own verbs, dwarves' ci/verbs: a crate DETACHED from
# dwarves' workspace, so a sibling's version move that breaks dwarves' graph cannot
# freeze them (it froze STATUS.md at 13:11Z on 2026-10-24, the cron running on).
# The products' fast tests (monetize v2.0 P1) are each product repo's `fast` step
# (`cargo xtask fast`, monetize's monetize-fast library), inside that repo's own plan.
# The runner has no step of its own. It never pushes, never reverts, and uses no
# signing key or secret on this box (it runs with a clean environment).
set -u
W=${CI_WORK:-/work/ci}
# One run at a time: the lock first, so a start that finds a run going exits before it
# touches anything (it once overwrote the running run's script copy). The copy below
# inherits descriptor 9 and with it the lock.
if [ "${CI_RUNNER_COPY:-}" != 1 ]; then
exec 9>"$W/.runner.lock"; flock -n 9 || exit 0
# The runner checks its own repo out anew, so it runs from a copy of itself.
cp "$0" "$W/.runner-copy.sh" && CI_RUNNER_COPY=1 exec bash "$W/.runner-copy.sh" "$@"
exit 1
fi
MODE=${1:---quick}
LIST=$W/src/dwarves/ci/repos.txt; [ -f "$LIST" ] || LIST=$(dirname "$0")/repos.txt
mkdir -p "$W/state" "$W/runs" "$W/target" "$W/src"
stamp=$(date -u +%Y%m%dT%H%M%SZ)
# Check every repo out at its mirrored head; collect the CI repos whose head is new,
# and whether ANY repo moved (a sibling's push can stale another repo's nested lock, so
# the constellation lock check runs on a push to any repo, CI repo or not).
changed=()
moved=()
while read -r name url branch ci fmt; do
[ -n "$name" ] && [ "${name#\#}" = "$name" ] || continue
[ -d "$W/mirror/$name.git" ] || continue
src=$W/src/$name
# LFS objects come from the mirror's store (hard links), never from a forge.
[ -d "$src/.git" ] || GIT_LFS_SKIP_SMUDGE=1 git clone -q --no-checkout "$W/mirror/$name.git" "$src" || continue
if [ -d "$W/mirror/$name.git/lfs/objects" ]; then
mkdir -p "$src/.git/lfs/objects" && cp -alu "$W/mirror/$name.git/lfs/objects/." "$src/.git/lfs/objects/" 2>/dev/null
fi
git -C "$src" fetch -q origin "+refs/heads/$branch:refs/remotes/origin/$branch" || continue
head=$(git -C "$src" rev-parse "origin/$branch")
# Pointers first, then every LFS object the mirror holds; an object the forge lost
# (gunnar's master today) stays a pointer instead of failing the whole checkout.
GIT_LFS_SKIP_SMUDGE=1 git -C "$src" checkout -q -f --detach "$head" && git -C "$src" clean -q -fdx
git -C "$src" lfs checkout >/dev/null 2>&1
if [ "$MODE" != --quick ] || [ "$(cat "$W/state/$name" 2>/dev/null)" != "$head" ]; then
moved+=("$name")
[ "$ci" = yes ] && changed+=("$name")
fi
done < <(grep -v '^#' "$LIST")
[ ${#moved[@]} -gt 0 ] || exit 0
RUN=$W/runs/$stamp
mkdir -p "$RUN"
for d in "$W"/src/*/; do n=$(basename "$d"); echo "$n $(git -C "$d" rev-parse HEAD)"; done > "$RUN/heads.txt"
echo "${MODE#--} moved: ${moved[*]} | ci: ${changed[*]}" > "$RUN/run.txt"
# The verbs run with a clean environment: no secret of this box reaches a step.
XDG=/run/user/$(id -u); [ -w "$XDG" ] || XDG=/tmp/ci-runtime-$(id -u); mkdir -p "$XDG"; chmod 700 "$XDG"
# The CI test image: the pinned recipe plus OpenSSL headers, clippy and rustfmt, and the
# clients a test starts: openssh-client, strace, git-lfs (ci/ci-toolchain.Containerfile; ci3
# had none of the three). Pinned by its manifest DIGEST, never a tag: a tag names another
# image on each box and after each re-mint. A rebuild from the Containerfile is a NEW digest
# (apt's files and the layer compression differ per box), so the pinned image goes from box
# to box as its OCI archive (`podman save --format oci-archive`, `podman load -i`), which
# keeps the digest. A box without it loads $CI_ARCHIVE; failing that it builds $CI_TAG and
# says that the pin is still missing (every toolchain step is then red, by name).
# Every step that starts it runs as the invoking user (`--userns=keep-id`, ci::Container).
CI_TAG=localhost/nordisk-cargo-toolchain:ci4-53fec13a5c1c7f2c
CI_IMAGE=localhost/nordisk-cargo-toolchain@sha256:875a7a325dfe7f6b838535d085a2a502e8ad250076c947f6d023b1e224f03334
CI_ARCHIVE=${CI_IMAGE_ARCHIVE:-/work/ci/images/ci4-53fec13a5c1c7f2c.oci.tar}
XDG_RUNTIME_DIR="$XDG" podman image exists "$CI_IMAGE" \
|| { [ -f "$CI_ARCHIVE" ] && XDG_RUNTIME_DIR="$XDG" podman load -q -i "$CI_ARCHIVE" > "$RUN/ci-image.out" 2>&1; }
XDG_RUNTIME_DIR="$XDG" podman image exists "$CI_IMAGE" \
|| { XDG_RUNTIME_DIR="$XDG" podman build -q -t "$CI_TAG" -f "$W/src/dwarves/ci/ci-toolchain.Containerfile" "$W/src/dwarves/ci" >> "$RUN/ci-image.out" 2>&1
echo "the pinned CI image $CI_IMAGE is missing here: load $CI_ARCHIVE (built $CI_TAG, another digest)" >> "$RUN/ci-image.out"; }
# HOME is the runner's own, so no step can read the owner's files on this box (~/.dwarves,
# ~/*_secrets): cargo, rustup and podman's storage and config are named from the real home.
mkdir -p "$W/home"
ci_env() {
env -i HOME="$W/home" USER="${USER:-$(id -un)}" PATH="$HOME/.cargo/bin:/usr/local/bin:/usr/bin:/bin" LANG=C.UTF-8 \
CARGO_HOME="$HOME/.cargo" RUSTUP_HOME="$HOME/.rustup" \
XDG_DATA_HOME="$HOME/.local/share" XDG_CONFIG_HOME="$HOME/.config" \
XDG_RUNTIME_DIR="$XDG" \
CARGO_BUILD_JOBS=16 MOLD_JOBS=4 NORNIR_CI_REGISTRY=/work/cargo-home NORNIR_CI_FAST=/work/tools \
NORNIR_CI_SCCACHE=/work/sccache NORNIR_CI_IMAGE="$CI_IMAGE" \
CONTAINERS_CONF_OVERRIDE="$W/src/dwarves/ci/containers.conf" \
RUSTC_WRAPPER=/work/tools/sccache SCCACHE_DIR=/work/sccache "$@"
}
# A verb that wrote no receipt (it did not build, or the repo has no ci verb yet) is red.
stub() { # repo dir log
cat > "$2/receipt.json" <<JSON
{
"schema": "nornir-ci/1",
"repo": "$1",
"head": "$(git -C "$W/src/$1" rev-parse HEAD 2>/dev/null)",
"mode": "${MODE#--}",
"finished_unix": $(date +%s),
"green": false,
"failing_step": "xtask",
"failing_detail": "cargo xtask ci wrote no receipt: $(grep -m1 -E '^error' "$3" | tr -d '"\\' | cut -c1-200)",
"failing_log": "$3",
"steps": []
}
JSON
}
# One bounded target per repo: a target above 80 GB is removed before its run.
# ONE shared target for every repo (the constellation shares most of its crates, and
# the jobs run one at a time), bounded: above 40 GB it is removed before the next step.
# /work/ci stays under 60 GB (owner, 2026-10-24); sccache keeps the rebuild cheap.
# Checked before every step, after every step and its filing, and once more after the run.
bound() { sh "$W/src/dwarves/ci/prune-target.sh" "$W/target" 40 >> "$RUN/prune.out" 2>&1; true; }
run_verb() { # label repo-dir out-dir args...
local label=$1 dir=$2 out=$3; shift 3
mkdir -p "$out"
# A repo whose sibling patches are opt-in switches them on, as its developers do.
[ -f "$dir/.cargo/estate-siblings.toml" ] && cp "$dir/.cargo/estate-siblings.toml" "$dir/.cargo/siblings.toml"
bound "$label"
# One CI job at a time in slot 2 (/work/.cargo-job-2.lock), taken per verb, not per run:
# the lane's own heavy work in slot 2 waits for one repo's verb, not for a whole run.
( flock 8; cd "$dir" && ci_env CARGO_TARGET_DIR="$W/target" NORNIR_CI_OUT="$out" timeout 4h cargo xtask ci "$@" ) \
8>/work/.cargo-job-2.lock > "$out/xtask.out" 2>&1
[ -f "$out/receipt.json" ] || stub "$label" "$out" "$out/xtask.out"
}
# The runner's own verbs (ci/verbs, its own lock; not --locked, so an edda version move
# of nornir-xtask-support, a path crate, cannot stop them either).
VERBS_MANIFEST=$W/src/dwarves/ci/verbs/Cargo.toml
# File a run's test and bench steps into the workspace's Mímir (the loki dwarves
# server, workspace holger) with dwarves' own `ci-file`, through the loki lane's
# reverse tunnel and its token, read here and handed to the container by name only.
TOKEN_FILE=/work/dloki/token.env
file_run() { # repo out
[ -r "$TOKEN_FILE" ] && [ -f "$2/receipt.json" ] || return 0
local tok
tok=$(sed -n 's/^NORNIR_SERVER_TOKEN=//p' "$TOKEN_FILE" | tr -d "\"'")
# Never --network host beside env.sh: the container's sccache would talk to another
# lane's server on the host. The loki tunnel is reached as the host's loopback,
# which slirp4netns exposes at 10.0.2.2. As the invoking user, like every toolchain step
# (nornir-xtask-support's ci::Container): keep-id, home /tmp, git trusting the checkouts.
NORNIR_SERVER_TOKEN="$tok" XDG_RUNTIME_DIR="$XDG" flock /work/.cargo-job-2.lock timeout 1h podman run --rm --network slirp4netns:allow_host_loopback=true \
--userns=keep-id --passwd-entry='$USERNAME:*:$UID:$GID:$NAME:/tmp:/bin/sh' \
-e GIT_CONFIG_COUNT=1 -e GIT_CONFIG_KEY_0=safe.directory -e 'GIT_CONFIG_VALUE_0=*' \
-v /work/ci:/work/ci -v /work/cargo-home:/usr/local/cargo/registry \
-v /work/tools:/tools:ro -v /work/sccache:/sccache --env-file /work/tools/env.sh \
-e CARGO_TARGET_DIR=/work/ci/target/toolchain -e CARGO_BUILD_JOBS=16 -e MOLD_JOBS=4 \
-e NORNIR_SERVER=http://10.0.2.2:17879 -e NORNIR_WORKSPACE=holger -e NORNIR_SERVER_TOKEN \
-w /work/ci/src/dwarves localhost/nordisk-cargo-toolchain:mimir-test \
sh -c 'cargo build -q --locked -p dwarves --bin dwarves --features lens-feeds && exec /work/ci/target/toolchain/debug/dwarves ci-file "$0" --checkout "$1"' \
"$2/receipt.json" "$W/src/$1" > "$2/file.out" 2>&1
}
{
mkdir -p "$RUN/constellation"; bound constellation
( flock 8; cd "$W/src/dwarves" && ci_env CARGO_TARGET_DIR="$W/target" NORNIR_CI_OUT="$RUN/constellation" timeout 4h cargo run --quiet --manifest-path "$VERBS_MANIFEST" -- lockcheck --siblings "$W/src" ) \
8>/work/.cargo-job-2.lock > "$RUN/constellation/xtask.out" 2>&1
[ -f "$RUN/constellation/receipt.json" ] || stub dwarves "$RUN/constellation" "$RUN/constellation/xtask.out"
for name in "${moved[@]}"; do git -C "$W/src/$name" rev-parse HEAD > "$W/state/$name"; done
for name in "${changed[@]}"; do
run_verb "$name" "$W/src/$name" "$RUN/$name" "$MODE"
file_run "$name" "$RUN/$name"
bound "$name"
git -C "$W/src/$name" rev-parse HEAD > "$W/state/$name"
GIT_LFS_SKIP_SMUDGE=1 git -C "$W/src/$name" checkout -q -f . 2>/dev/null; git -C "$W/src/$name" clean -q -fdx
git -C "$W/src/$name" lfs checkout >/dev/null 2>&1
done
}
( cd "$W/src/dwarves" && ci_env CARGO_TARGET_DIR="$W/target" cargo run --quiet --manifest-path "$VERBS_MANIFEST" -- status --runs "$W/runs" --out "$W/STATUS.md" ) \
> "$RUN/status.out" 2>&1
# The run leaves no target above the bound behind it.
bound end
# Keep the newest 300 runs.
ls -1d "$W"/runs/*/ 2>/dev/null | sort | head -n -300 | xargs -r rm -rf