ci/forge/setup.sh
#!/bin/sh
# setup.sh — make dwarf's dwarves CI forge (once; safe to run again). Lane SCORECARD.
#
# sh ci/forge/setup.sh <path to a built `dwarves` with --features cli,server,runs-on-container>
#
# Clones every checkout the project names under ~/tgt/ci/src (ONE at a time, with retry:
# git.gunnar.rs caps transfers per user), clones the public status repository into
# ~/tgt/ci/status, installs the env, the containers.conf, the project file and the two
# user units, and starts them. It never kills anything; it stops nothing but its own units.
set -eu
BIN=${1:?usage: setup.sh <dwarves binary>}
HERE=$(cd "$(dirname "$0")" && pwd)
FORGE=ssh://git@git.gunnar.rs:2222
CI=$HOME/tgt/ci
mkdir -p "$CI/src" "$CI/out" "$HOME/.config/dwarves-ci" "$HOME/.local/lib/dwarves-ci" \
"$HOME/.dwarves-ci/warehouse/projects" "$HOME/.dwarves-ci/warehouse/ci/ci/run-now" "$HOME/.config/systemd/user"
# the product suites' TMPDIR (ci.dwarves.toml): a disk with headroom, outside the root array
mkdir -p /backup/ci-tmp/gunnar /backup/ci-tmp/holger /backup/ci-tmp/monetize
# the targets (forge.env DWARVES_CI_TARGETS): on /backup, not the root array the KVM gates need
mkdir -p /backup/ci-target
# the fuzz corpora, kept between nights (pipelines/fuzz.runa.ron, OSSF-NEXT)
mkdir -p /backup/ci-fuzz
# the weekly bench's way to oden (README-BENCH, pipelines/bench-readme-*.runa.ron): a key of
# the forge's own and an ssh config, bound into the step containers (forge.env
# DWARVES_RUNA_BINDS; a step has HOME=/tmp). The public key goes into oden's authorized_keys
# once; the private key never leaves this directory.
B=$HOME/.config/dwarves-ci/bench-ssh
mkdir -p "$B" && chmod 0700 "$B"
[ -f "$B/id_ed25519" ] || ssh-keygen -q -t ed25519 -N "" -C "dwarves-ci-bench@dwarf" -f "$B/id_ed25519"
[ -f "$B/config" ] || cat > "$B/config" <<CONF
Host oden
HostName 100.115.167.95
User rickard
IdentityFile $B/id_ed25519
IdentitiesOnly yes
UserKnownHostsFile $B/known_hosts
StrictHostKeyChecking accept-new
BatchMode yes
CONF
echo "bench key for oden's ~/.ssh/authorized_keys: $(cat "$B/id_ed25519.pub")"
clone() { # owner/name
name=${1#*/}
[ -d "$CI/src/$name/.git" ] && return 0
for i in 1 2 3 4 5; do
GIT_LFS_SKIP_SMUDGE=1 git clone -q "$FORGE/$1.git" "$CI/src/$name" && return 0
rm -rf "$CI/src/$name"; sleep 30
done
echo "setup: $1 did not clone" >&2; return 1
}
for r in nordisk/dwarves nordisk/edda nordisk/Skidbladnir mikronova/draupnir nordisk/facett \
mikronova/funnel-core nordisk/gunnar nordisk/gunnar-ui nordisk/holger nordisk/holger-ui \
mikronova/modgunn nordisk/monetize nordisk/nornir nordisk/skade mikronova/tunnr \
nordisk/korp mikronova/ordning nordisk/knut \
nordisk/znippy nordisk/znippy-zoomies mikronova/openzl-sys-rs nordisk/heimdall; do
clone "$r"
done
# the private checkouts (forge.env DWARVES_CI_PRIVATE): only a verdict-only pipeline sees one,
# read-only beside its repo (monetize-full: ../monetize-impl for the trials; its home is the
# private GitHub repository vetra-automation/monetize-impl, fetched only, never pushed)
PRIV=$CI/private
mkdir -p "$PRIV"
if [ ! -d "$PRIV/monetize-impl/.git" ]; then
for i in 1 2 3 4 5; do
git clone -q git@github.com:vetra-automation/monetize-impl.git "$PRIV/monetize-impl" && break
rm -rf "$PRIV/monetize-impl"; sleep 30
done
fi
# the public status repository: README.md is the page
if [ ! -d "$CI/status/.git" ]; then
git clone -q "$FORGE/nordisk/ci-status.git" "$CI/status"
fi
git -C "$CI/status" config user.name "Rickard Lundin"
git -C "$CI/status" config user.email rickard@ignalina.dk
# the step image (forge.env): ci4 plus Chromium, by digest; built here only when missing
DWARVES_RUNA_IMAGE=$(sed -n 's/^DWARVES_RUNA_IMAGE=//p' "$HERE/forge.env")
DWARVES_CI_IMAGE_TAG=$(sed -n 's/^DWARVES_CI_IMAGE_TAG=//p' "$HERE/forge.env")
podman image exists "$DWARVES_RUNA_IMAGE" \
|| { podman build --network host -q -t "$DWARVES_CI_IMAGE_TAG" -f "$HERE/ci-forge.Containerfile" "$HERE"
echo "setup: built $DWARVES_CI_IMAGE_TAG as $(podman image inspect --format '{{.Digest}}' "$DWARVES_CI_IMAGE_TAG");" \
"set DWARVES_RUNA_IMAGE in forge.env to it when it is not $DWARVES_RUNA_IMAGE" >&2; }
# heimdall-ci's own step image (ci.dwarves.toml `image`): heim-ci from heimdall's own
# Containerfile, built here only when the box lacks the pinned digest
HEIM_IMAGE=$(sed -n '/^name = "heimdall-ci"/,/^\[/s/^image = "\(.*\)"/\1/p' "$HERE/ci.dwarves.toml")
podman image exists "$HEIM_IMAGE" \
|| { podman build --network host -q -t localhost/nordisk-cargo-toolchain:heim-ci \
-f "$CI/src/heimdall/ci/ci.Containerfile" "$CI/src/heimdall/ci"
echo "setup: built heim-ci as $(podman image inspect --format '{{.Digest}}' localhost/nordisk-cargo-toolchain:heim-ci);" \
"set heimdall-ci's image in ci.dwarves.toml to it when it is not $HEIM_IMAGE" >&2; }
# the drift gate's tool (tunnr-drift, `tools = ["heimdall"]`): the row-pinned heimdall, built in
# heim-ci only when the box lacks it; its sha256 must be the pin of tunnr's measure.toml files
TOOLS=$HOME/.dwarves-ci/warehouse/ci/tools
PIN=$(sed -n 's/^sha256 = "\(.*\)"/\1/p' "$CI/src/tunnr/.nornir/profiles/rusternetes-node/measure.toml")
if [ ! -x "$TOOLS/heimdall/heimdall" ]; then
mkdir -p "$TOOLS/heimdall/root"
podman run --rm --network host -v "$TOOLS/heimdall/root:/out" localhost/nordisk-cargo-toolchain:heim-ci \
cargo install nordisk-heimdall-cli --version 0.2.0 --locked --root /out
install -m 0755 "$TOOLS/heimdall/root/bin/heimdall" "$TOOLS/heimdall/heimdall"
fi
HAVE=$(sha256sum "$TOOLS/heimdall/heimdall" | cut -d' ' -f1)
[ "$HAVE" = "$PIN" ] || echo "setup: ci/tools/heimdall/heimdall has sha256 $HAVE, tunnr's rows pin $PIN: tunnr-drift refuses it" >&2
install -m 0755 "$BIN" "$HOME/.local/lib/dwarves-ci/dwarves"
# the gate helper (SC-GATE-STARVE): gates take kvm-gate.lock through it, so the forge's ask goes first
mkdir -p "$HOME/bin" && install -m 0755 "$HERE/kvm-gate.sh" "$HOME/bin/kvm-gate.sh"
install -m 0644 "$HERE/forge.env" "$HOME/.config/dwarves-ci/forge.env"
install -m 0644 "$HERE/containers.conf" "$HOME/.config/dwarves-ci/containers.conf"
install -m 0644 "$HERE/ci.dwarves.toml" "$HOME/.dwarves-ci/warehouse/projects/ci.dwarves.toml"
install -m 0644 "$HERE/dwarves-ci-podman.service" "$HERE/dwarves-ci-step-podman.service" "$HERE/dwarves-ci.service" "$HOME/.config/systemd/user/"
systemctl --user daemon-reload
systemctl --user enable --now dwarves-ci-podman.service dwarves-ci-step-podman.service dwarves-ci.service
systemctl --user --no-pager status dwarves-ci.service | head -5