crates/dwarves-slsa/src/runner.rs
//! ★ A release build is a **Runa pipeline** (owner 2026-10-29, lane RELBUILD-RUNA).
//!
//! One release = one fresh run directory and one Runa run of [`release_pipeline`]:
//!
//! 1. `sources`: a `Forge` step per repository checks out its pin by commit id
//! (`slsa.checkout`); `seal-sources` makes the snapshot read-only, reads the locks,
//! stages what crates.io already has and writes the probe;
//! 2. `fetch`, `probe-lock`, `probe-vendor`: SEALED containers (Runa `Sandbox`) with the
//! network on run cargo only (`cargo vendor --locked`, the probe's
//! `generate-lockfile`): no build script, no proc macro, nothing of the release's code;
//! 3. `vendor-check`: every vendored crate against the locks' checksums, then the
//! vendor and the plan read-only;
//! 4. per crate, a sealed container with `--network=none` runs `cargo package --locked
//! --offline`, and the forge stages the package for the next (`slsa.stage`, in Rust);
//! the binaries are built from the packages (or the sources, a Build Thing variant)
//! the same way, and a workspace example runs from the sources (`cargo run --example`,
//! its declared outputs collected by the forge);
//! 5. the platform (outside every container) hashes `out/` and writes the provenance from
//! the Runa run's record ([`run`]).
//!
//! Every container is fresh, from the pinned image by digest (never pulled), with a
//! read-only root, every capability dropped and `no-new-privileges`, and only the run
//! directory's own mounts ([`check_isolation`]): no `$HOME`, no key, no sccache, no shared
//! registry or target. There is no shell anywhere: each container runs one `cargo`
//! argv. This crate holds the plan and the forge's actions; the dwarves forge runs the
//! pipeline ([`Executor`]) through Runa's runner, with CI's caps and cancel.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::Write as _;
use std::path::{Component, Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};
use dwarves_runa::{Mount, Pipeline, Stage, Step, Tmpfs, Trigger};
use serde_json::{json, Value};
use crate::lock::{dependencies_of, parse_lock, LockedPackage};
use crate::spec::{BinaryStep, CrateStep, ReleaseSpec, Repo, WorkspaceBinary, WorkspaceExample};
use crate::statement::{self, Provenance, ResourceDescriptor, BUILDER_ID, BUILD_TYPE_CARGO_RELEASE};
use crate::subjects::{sha256_file, sha256_hex, subjects_of_dir};
use crate::{bail, Error, Result};
/// The ci4 toolchain image (the recipe plus libssl-dev, clippy, rustfmt,
/// openssh-client, strace and git-lfs), by digest. Loadable from
/// `~/ci-images/ci4-53fec13a5c1c7f2c.oci.tar`.
pub const DEFAULT_IMAGE: &str =
"localhost/nordisk-cargo-toolchain@sha256:875a7a325dfe7f6b838535d085a2a502e8ad250076c947f6d023b1e224f03334";
/// The User-Agent of every request the runner makes.
pub const USER_AGENT: &str = "nordisk-dep-audit";
/// The Runa pipeline name of every release build.
pub const PIPELINE: &str = "release-build";
/// The forge's own actions (Runa `Forge` steps) of a release build.
pub const FORGE_CHECKOUT: &str = "slsa.checkout";
pub const FORGE_SEAL_SOURCES: &str = "slsa.seal-sources";
pub const FORGE_VENDOR_CHECK: &str = "slsa.vendor-check";
pub const FORGE_STAGE: &str = "slsa.stage";
pub const FORGE_PREPARE_BINARY: &str = "slsa.prepare-binary";
pub const FORGE_COLLECT_BINARY: &str = "slsa.collect-binary";
pub const FORGE_COLLECT_EXAMPLE: &str = "slsa.collect-example";
/// Every action [`ReleaseRun::forge`] knows.
pub const ACTIONS: [&str; 7] = [
FORGE_CHECKOUT,
FORGE_SEAL_SOURCES,
FORGE_VENDOR_CHECK,
FORGE_STAGE,
FORGE_PREPARE_BINARY,
FORGE_COLLECT_BINARY,
FORGE_COLLECT_EXAMPLE,
];
/// A run directory.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Layout {
pub root: PathBuf,
}
impl Layout {
pub fn new(root: impl Into<PathBuf>) -> Self {
Self { root: root.into() }
}
pub fn src(&self) -> PathBuf {
self.root.join("src")
}
pub fn vendor(&self) -> PathBuf {
self.root.join("vendor")
}
pub fn fetch_home(&self) -> PathBuf {
self.root.join("fetch-home")
}
pub fn cargo_home(&self) -> PathBuf {
self.root.join("cargo-home")
}
pub fn target(&self) -> PathBuf {
self.root.join("target")
}
pub fn out(&self) -> PathBuf {
self.root.join("out")
}
pub fn plan(&self) -> PathBuf {
self.root.join("plan")
}
pub fn published(&self) -> PathBuf {
self.root.join("published")
}
pub fn probe(&self) -> PathBuf {
self.root.join("probe")
}
/// The directory source cargo resolves from: the vendored crates (links) and each
/// staged step.
pub fn stage(&self) -> PathBuf {
self.root.join("stage")
}
/// A packaged crate copied out to build its binaries in.
pub fn work(&self) -> PathBuf {
self.root.join("work")
}
/// Where each workspace example writes (`example-out/<i>`, its `{out}`), before the
/// forge collects the declared outputs into `out/`.
pub fn example_out(&self) -> PathBuf {
self.root.join("example-out")
}
pub fn logs(&self) -> PathBuf {
self.root.join("logs")
}
/// The unsigned statement the owner signs.
pub fn statement(&self) -> PathBuf {
self.root.join("release.statement.json")
}
/// Where the signed bundle goes.
pub fn bundle(&self) -> PathBuf {
self.root.join("release.intoto.jsonl")
}
/// The pipeline as it ran.
pub fn pipeline(&self) -> PathBuf {
self.logs().join("pipeline.runa.ron")
}
}
/// The plan's step lines: `crate version repo path mode`, mode `package` or
/// `published` (crates.io has that version: its `.crate` is staged, not rebuilt).
pub fn steps_file(spec: &ReleaseSpec, published: &BTreeSet<(String, String)>) -> String {
spec.crates
.iter()
.map(|c| {
let mode = if published.contains(&(c.name.clone(), c.version.clone())) {
"published"
} else {
"package"
};
format!("{} {} {} {} {mode}\n", c.name, c.version, c.repo, c.path)
})
.collect()
}
fn version_of<'a>(spec: &'a ReleaseSpec, crate_name: &str) -> &'a str {
spec.crates
.iter()
.find(|c| c.name == crate_name)
.map(|c| c.version.as_str())
.unwrap_or("?")
}
fn csv_or_dash(f: &[String]) -> String {
if f.is_empty() {
"-".to_string()
} else {
f.join(",")
}
}
/// The plan's binary lines: `crate version bin no-default(0|1) features(csv or -)`.
pub fn binaries_file(spec: &ReleaseSpec) -> String {
spec.binaries
.iter()
.map(|b| {
format!(
"{} {} {} {} {}\n",
b.crate_name,
version_of(spec, &b.crate_name),
b.bin,
u8::from(b.no_default_features),
csv_or_dash(&b.features)
)
})
.collect()
}
/// The plan's workspace-binary lines: `manifest package bin no-default(0|1) features(csv or -)`.
pub fn workspace_binaries_file(spec: &ReleaseSpec) -> String {
spec.workspace_binaries
.iter()
.map(|w| {
format!(
"{} {} {} {} {}\n",
w.manifest,
w.package,
w.bin,
u8::from(w.no_default_features),
csv_or_dash(&w.features)
)
})
.collect()
}
/// `a/b/../c` → `a/c`, lexically (no filesystem access).
fn normalize(p: &Path) -> PathBuf {
let mut out = PathBuf::new();
for c in p.components() {
match c {
Component::ParentDir => {
out.pop();
}
Component::CurDir => {}
other => out.push(other.as_os_str()),
}
}
out
}
fn lock_dir(manifest: &str) -> &str {
manifest.strip_suffix("/Cargo.toml").unwrap_or(manifest)
}
// ───────────────────────────── sealed containers ─────────────────────────────
/// One bind mount of a sealed container.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Bind {
pub host: PathBuf,
pub path: String,
pub read_only: bool,
}
/// ★ A sealed container (a Runa `Sandbox` step): what it runs and the only things it
/// sees. The forge adds the seal itself: a read-only root, every capability dropped,
/// `no-new-privileges`, the image never pulled.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Sealed {
pub image: String,
pub cmd: Vec<String>,
pub env: Vec<(String, String)>,
pub binds: Vec<Bind>,
/// `(path, size in MiB)`.
pub tmpfs: Vec<(String, u32)>,
pub network: bool,
pub workdir: String,
}
impl Sealed {
/// The Runa step.
pub fn to_step(&self) -> Step {
Step::Sandbox {
image: self.image.clone(),
cmd: self.cmd.clone(),
env: self.env.clone(),
mounts: self
.binds
.iter()
.map(|b| Mount {
host: b.host.display().to_string(),
path: b.path.clone(),
read_only: b.read_only,
})
.collect(),
tmpfs: self
.tmpfs
.iter()
.map(|(p, s)| Tmpfs {
path: p.clone(),
size_mb: *s,
})
.collect(),
network: self.network,
workdir: self.workdir.clone(),
}
}
/// The bind mounted at `path`, if any.
pub fn bind(&self, path: &str) -> Option<&Bind> {
self.binds.iter().find(|b| b.path == path)
}
}
fn bind(host: PathBuf, path: &str, read_only: bool) -> Bind {
Bind {
host,
path: path.to_string(),
read_only,
}
}
fn env(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
fn cargo_env() -> Vec<(String, String)> {
env(&[
("CARGO_HOME", "/cargo-home"),
("HOME", "/tmp"),
("CARGO_TERM_COLOR", "never"),
])
}
/// git (cargo package reads the status): any owner, and git-lfs's scratch in a tmpfs.
fn git_env() -> Vec<(String, String)> {
env(&[
("GIT_CONFIG_COUNT", "2"),
("GIT_CONFIG_KEY_0", "safe.directory"),
("GIT_CONFIG_VALUE_0", "*"),
("GIT_CONFIG_KEY_1", "lfs.storage"),
("GIT_CONFIG_VALUE_1", "/lfs"),
])
}
fn argv(a: &[&str]) -> Vec<String> {
a.iter().map(|s| s.to_string()).collect()
}
/// The fetch container: `cargo vendor --locked` of every lock (network on), sources
/// read-only.
pub fn fetch_container(layout: &Layout, spec: &ReleaseSpec) -> Sealed {
let mut cmd = argv(&["cargo", "vendor", "--locked", "--versioned-dirs"]);
for l in &spec.locks[1..] {
cmd.extend(["-s".into(), format!("/src/{l}")]);
}
cmd.push("/vendor/crates".into());
Sealed {
image: spec.image.clone(),
cmd,
env: cargo_env(),
binds: vec![
bind(layout.src(), "/src", true),
bind(layout.vendor(), "/vendor", false),
bind(layout.fetch_home(), "/cargo-home", false),
],
tmpfs: vec![("/tmp".into(), 1024)],
network: true,
workdir: format!("/src/{}", lock_dir(&spec.locks[0])),
}
}
fn probe_container(layout: &Layout, spec: &ReleaseSpec, cmd: Vec<String>) -> Sealed {
Sealed {
image: spec.image.clone(),
cmd,
env: cargo_env(),
binds: vec![
bind(layout.src(), "/src", true),
bind(layout.vendor(), "/vendor", false),
bind(layout.fetch_home(), "/cargo-home", false),
bind(layout.probe(), "/probe", false),
],
tmpfs: vec![("/tmp".into(), 1024)],
network: true,
workdir: "/probe".into(),
}
}
/// The probe's lock: what packaging resolves from crates.io for the patched externals.
pub fn probe_lock_container(layout: &Layout, spec: &ReleaseSpec) -> Sealed {
probe_container(layout, spec, argv(&["cargo", "generate-lockfile"]))
}
/// The probe's crates, into the same vendor directory.
pub fn probe_vendor_container(layout: &Layout, spec: &ReleaseSpec) -> Sealed {
probe_container(
layout,
spec,
argv(&[
"cargo",
"vendor",
"--locked",
"--versioned-dirs",
"--no-delete",
"/vendor/crates",
]),
)
}
fn build_container(layout: &Layout, spec: &ReleaseSpec, jobs: u32, cmd: Vec<String>, workdir: String, extra: Vec<Bind>) -> Sealed {
let mut e = cargo_env();
e.push(("CARGO_BUILD_JOBS".into(), jobs.to_string()));
e.extend(git_env());
let mut binds = vec![
bind(layout.src(), "/src", true),
bind(layout.vendor(), "/vendor", true),
bind(layout.stage(), "/stage", true),
bind(layout.published(), "/published", true),
bind(layout.plan(), "/plan", true),
bind(layout.target(), "/target", false),
bind(layout.cargo_home(), "/cargo-home", false),
];
binds.extend(extra);
Sealed {
image: spec.image.clone(),
cmd,
env: e,
binds,
tmpfs: vec![("/tmp".into(), 2048), ("/lfs".into(), 2048)],
network: false,
workdir,
}
}
/// One crate packaged (network none): `cargo package --locked --offline`.
pub fn package_container(layout: &Layout, spec: &ReleaseSpec, c: &CrateStep, jobs: u32) -> Sealed {
let cmd = vec![
"cargo".into(),
"package".into(),
"--locked".into(),
"--offline".into(),
"--no-verify".into(),
"--registry".into(),
"crates-io".into(),
"-p".into(),
c.name.clone(),
"--target-dir".into(),
format!("/target/{}", c.repo),
];
build_container(layout, spec, jobs, cmd, format!("/src/{}/{}", c.repo, c.path), Vec::new())
}
fn feature_args(cmd: &mut Vec<String>, no_default: bool, features: &[String]) {
if no_default {
cmd.push("--no-default-features".into());
}
if !features.is_empty() {
cmd.extend(["--features".into(), features.join(",")]);
}
}
/// A binary built from the packaged crate (what `cargo install --locked` builds).
pub fn binary_container(layout: &Layout, spec: &ReleaseSpec, b: &BinaryStep, jobs: u32) -> Sealed {
let mut cmd = argv(&["cargo", "build", "--release", "--locked", "--offline", "--bin"]);
cmd.push(b.bin.clone());
cmd.extend(["--target-dir".into(), "/target/bin".into()]);
feature_args(&mut cmd, b.no_default_features, &b.features);
let dir = format!("{}-{}", b.crate_name, version_of(spec, &b.crate_name));
build_container(layout, spec, jobs, cmd, format!("/work/{dir}"), vec![bind(layout.work(), "/work", false)])
}
/// A binary built from the SOURCES (a Build Thing variant: a workspace member), from the
/// workspace's own lock against the vendored crates.
pub fn workspace_binary_container(layout: &Layout, spec: &ReleaseSpec, w: &WorkspaceBinary, jobs: u32) -> Sealed {
let mut cmd = argv(&["cargo", "build", "--release", "--locked", "--offline", "--manifest-path"]);
cmd.push(format!("/src/{}", w.manifest));
cmd.extend(["-p".into(), w.package.clone(), "--bin".into(), w.bin.clone()]);
cmd.extend(["--target-dir".into(), "/target/ws".into()]);
feature_args(&mut cmd, w.no_default_features, &w.features);
let repo = w.manifest.split('/').next().unwrap_or("").to_string();
build_container(layout, spec, jobs, cmd, format!("/src/{repo}"), Vec::new())
}
/// ★ An example of a workspace run from the SOURCES (`cargo run --release --locked
/// --offline --example`), from the workspace's own lock against the vendored crates, in
/// the manifest's directory. `{out}` in its arguments is `/example-out`, its own
/// directory (`example-out/<i>`); the forge collects the declared outputs from there
/// ([`FORGE_COLLECT_EXAMPLE`]), so the build never writes `out/` itself. The example may
/// run cargo itself (Heimdall's `dist` builds the guest for wasm32): that cargo inherits
/// the cargo home (the stage, offline) and `CARGO_NET_OFFLINE`, so it is offline by
/// construction, not only by the missing network.
pub fn workspace_example_container(layout: &Layout, spec: &ReleaseSpec, i: usize, w: &WorkspaceExample, jobs: u32) -> Sealed {
let mut cmd = argv(&["cargo", "run", "--release", "--locked", "--offline", "--manifest-path"]);
cmd.push(format!("/src/{}", w.manifest));
cmd.extend(["--example".into(), w.example.clone()]);
cmd.extend(["--target-dir".into(), format!("/target/example-{i}"), "--".into()]);
cmd.extend(w.args.iter().map(|a| match a.strip_prefix(crate::spec::OUT_PLACEHOLDER) {
Some(rest) => format!("/example-out{rest}"),
None => a.clone(),
}));
let mut c = build_container(
layout,
spec,
jobs,
cmd,
format!("/src/{}", lock_dir(&w.manifest)),
vec![bind(layout.example_out().join(i.to_string()), "/example-out", false)],
);
c.env.push(("CARGO_NET_OFFLINE".into(), "true".into()));
c
}
/// Every file under `dir` (relative, `/`-separated); anything but a regular file or a
/// directory is refused.
fn regular_files(dir: &Path, rel: &str, out: &mut BTreeSet<String>) -> Result<()> {
for e in std::fs::read_dir(dir)? {
let e = e?;
let name = e.file_name().to_string_lossy().into_owned();
let r = if rel.is_empty() { name } else { format!("{rel}/{name}") };
let t = e.file_type()?;
if t.is_dir() {
regular_files(&e.path(), &r, out)?;
} else if t.is_file() {
out.insert(r);
} else {
bail!("{r} is not a regular file");
}
}
Ok(())
}
/// ★ The isolation rules, checked on each sealed container before the pipeline exists:
/// the image by digest; cargo, and nothing else, as the program (no shell); no
/// secret-looking variable; every bind mount from inside the run directory; the sources
/// and the plan read-only; and, for a build, `--network=none` and the vendor, the stage
/// and the published crates read-only.
pub fn check_isolation(c: &Sealed, layout: &Layout, build: bool) -> Result<()> {
if !dwarves_runa::image_by_digest(&c.image) {
bail!("isolation: the image {} is not named by digest", c.image);
}
if c.cmd.first().map(String::as_str) != Some("cargo") {
bail!("isolation: a release container runs cargo only, not {:?} (no shell)", c.cmd.first());
}
if build && c.network {
bail!("isolation: a build container runs with the network none");
}
if let Some((k, _)) = c.env.iter().find(|(k, _)| dwarves_runa::env_looks_secret(k)) {
bail!("isolation: variable {k} looks secret; nothing secret reaches a build");
}
let root = normalize(&layout.root);
for b in &c.binds {
let s = normalize(&b.host);
if !s.is_absolute() || !s.starts_with(&root) || s == root {
bail!(
"isolation: mount {} -> {} is from outside the run directory {}",
b.host.display(),
b.path,
root.display()
);
}
let ro_needed = ["/src", "/plan"].contains(&b.path.as_str())
|| (build && ["/vendor", "/published", "/stage"].contains(&b.path.as_str()));
if ro_needed && !b.read_only {
bail!("isolation: {} must be mounted read-only ({})", b.path, b.host.display());
}
}
Ok(())
}
/// The probe crate's manifest: one exact dependency per patched external (name,
/// version, every feature it has), so `cargo vendor` brings what packaging resolves
/// from crates.io.
pub fn probe_manifest(externals: &[(String, String, Vec<String>)]) -> String {
let mut deps = toml::map::Map::new();
for (name, version, features) in externals {
let mut d = toml::map::Map::new();
d.insert("version".into(), toml::Value::String(format!("={version}")));
d.insert(
"features".into(),
toml::Value::Array(features.iter().cloned().map(toml::Value::String).collect()),
);
deps.insert(name.clone(), toml::Value::Table(d));
}
let mut package = toml::map::Map::new();
for (k, v) in [
("name", "dwarves-slsa-probe"),
("version", "0.0.0"),
("edition", "2021"),
] {
package.insert(k.into(), toml::Value::String(v.into()));
}
package.insert("publish".into(), toml::Value::Boolean(false));
let mut lib = toml::map::Map::new();
lib.insert("path".into(), toml::Value::String("lib.rs".into()));
let mut doc = toml::map::Map::new();
doc.insert("package".into(), toml::Value::Table(package));
doc.insert("lib".into(), toml::Value::Table(lib));
doc.insert(
"workspace".into(),
toml::Value::Table(toml::map::Map::new()),
);
doc.insert("dependencies".into(), toml::Value::Table(deps));
format!(
"# dwarves-slsa: what packaging resolves from crates.io for the patched externals.\n{}",
toml::to_string(&toml::Value::Table(doc)).unwrap_or_default()
)
}
#[derive(serde::Deserialize)]
struct CargoChecksum {
#[serde(default)]
package: Option<String>,
}
/// ★ Every registry package of every lock is vendored (`name-version`) with the
/// lock's checksum in its `.cargo-checksum.json`. Returns how many were checked.
pub fn verify_vendor(vendor_crates: &Path, locks: &[Vec<LockedPackage>]) -> Result<usize> {
let mut want: BTreeMap<(String, String), String> = BTreeMap::new();
for p in locks.iter().flatten().filter(|p| p.is_registry()) {
if let Some(sum) = &p.checksum {
if let Some(prev) = want.insert((p.name.clone(), p.version.clone()), sum.clone()) {
if prev != *sum {
bail!(
"vendor: {} {} has two checksums in the locks",
p.name,
p.version
);
}
}
}
}
for ((name, version), sum) in &want {
let f = vendor_crates
.join(format!("{name}-{version}"))
.join(".cargo-checksum.json");
let text = std::fs::read_to_string(&f).map_err(|_| {
Error(format!(
"vendor: the vendored crate {name} {version} is missing ({})",
f.display()
))
})?;
let got: CargoChecksum = serde_json::from_str(&text)
.map_err(|e| Error(format!("vendor: {}: {e}", f.display())))?;
if got.package.as_deref() != Some(sum.as_str()) {
bail!(
"vendor: {name} {version}: the vendored checksum {:?} is not the lock's {sum}",
got.package
);
}
}
Ok(want.len())
}
// ───────────────────────────── the forge's actions ─────────────────────────────
pub(crate) fn run_cmd(c: &mut Command, what: &str) -> Result<()> {
let st = c.status().map_err(|e| Error(format!("{what}: {e}")))?;
if !st.success() {
bail!("{what}: exit {st}");
}
Ok(())
}
pub(crate) fn capture(c: &mut Command, what: &str) -> Result<String> {
let o = c
.stderr(Stdio::inherit())
.output()
.map_err(|e| Error(format!("{what}: {e}")))?;
if !o.status.success() {
bail!("{what}: exit {}", o.status);
}
Ok(String::from_utf8_lossy(&o.stdout).into_owned())
}
/// git, for the checkouts (git-lfs included: no library does LFS).
fn git(dir: &Path) -> Command {
let mut c = Command::new("git");
c.arg("-C").arg(dir).env("GIT_TERMINAL_PROMPT", "0");
c
}
/// The `git status --porcelain` lines that are NOT a file the commit holds raw although
/// its `.gitattributes` routes it to lfs (`git lfs fsck --pointers` names those; git
/// shows them modified after any checkout). Anything else is real dirt.
pub fn unexplained_dirt(status: &str, fsck_pointers: &str) -> Vec<String> {
let raw: BTreeSet<&str> = fsck_pointers
.lines()
.filter_map(|l| l.strip_prefix("pointer: unexpectedGitObject: \""))
.filter_map(|l| l.split_once("\" (treeish").map(|(p, _)| p))
.collect();
status
.lines()
.filter(|l| !l.trim().is_empty())
.filter(|l| !(l.starts_with(" M ") && raw.contains(l[3..].trim_matches('"'))))
.map(str::to_string)
.collect()
}
/// Up to five tries, 60 s apart (the forge refuses bursts).
fn retry(say: &mut dyn FnMut(&str), what: &str, mut f: impl FnMut() -> Result<()>) -> Result<()> {
let mut last = Error(String::new());
for i in 1..=5 {
match f() {
Ok(()) => return Ok(()),
Err(e) => {
say(&format!(" ({what}: try {i} failed: {e})"));
last = e;
if i < 5 {
std::thread::sleep(std::time::Duration::from_secs(60));
}
}
}
}
Err(last)
}
/// `src` copied into `dst`, recursively (files and directories; links as links).
pub fn copy_tree(src: &Path, dst: &Path) -> Result<()> {
std::fs::create_dir_all(dst)?;
for e in std::fs::read_dir(src)? {
let e = e?;
let t = e.file_type()?;
let to = dst.join(e.file_name());
if t.is_dir() {
copy_tree(&e.path(), &to)?;
} else if t.is_symlink() {
std::os::unix::fs::symlink(std::fs::read_link(e.path())?, &to)?;
} else {
std::fs::copy(e.path(), &to)?;
}
}
Ok(())
}
fn snapshot(spec: &ReleaseSpec, r: &Repo, layout: &Layout, from: Option<&Path>, say: &mut dyn FnMut(&str)) -> Result<()> {
let dst = layout.src().join(&r.name);
let url = spec.clone_url(r);
let local = from
.map(|f| f.join(&r.name))
.filter(|p| p.join(".git").exists());
match &local {
Some(l) => {
say(&format!("source {}: objects from {}", r.name, l.display()));
run_cmd(
Command::new("git")
.args(["clone", "-q", "--no-checkout", "--no-local"])
.arg(l)
.arg(&dst),
"git clone (local)",
)?
}
None => {
say(&format!("source {}: clone {url}", r.name));
retry(say, "git clone", || {
let _ = std::fs::remove_dir_all(&dst);
run_cmd(
Command::new("git")
.env("GIT_TERMINAL_PROMPT", "0")
.args(["clone", "-q", "--no-checkout", &url])
.arg(&dst),
"git clone",
)
})?
}
}
run_cmd(
git(&dst).args(["remote", "set-url", "origin", &url]),
"git remote set-url",
)?;
let has = |c: &str| {
git(&dst)
.args(["cat-file", "-e", &format!("{c}^{{commit}}")])
.stderr(Stdio::null())
.status()
.map(|s| s.success())
.unwrap_or(false)
};
if !has(&r.commit) {
say(&format!("source {}: fetch {} from the forge", r.name, r.commit));
retry(say, "git fetch", || {
run_cmd(
git(&dst).args(["fetch", "-q", "origin", &r.commit]),
"git fetch",
)
})?;
}
run_cmd(
git(&dst).env("GIT_LFS_SKIP_SMUDGE", "1").args([
"-c",
"advice.detachedHead=false",
"checkout",
"-q",
"--detach",
&r.commit,
]),
"git checkout",
)?;
let lfs = capture(
git(&dst).args(["lfs", "ls-files", "-n"]),
"git lfs ls-files",
)
.unwrap_or_default();
if !lfs.trim().is_empty() {
if let Some(objs) = local
.as_ref()
.map(|l| l.join(".git/lfs/objects"))
.filter(|p| p.is_dir())
{
copy_tree(&objs, &dst.join(".git/lfs/objects"))?;
}
say(&format!("source {}: {} lfs file(s)", r.name, lfs.lines().count()));
retry(say, "git lfs pull", || {
run_cmd(git(&dst).args(["lfs", "pull"]), "git lfs pull")
})?;
// Every object in the store against its own oid (--objects: some repos commit a
// few files raw that .gitattributes routes to lfs; that is hygiene, not integrity).
run_cmd(
git(&dst)
.args(["lfs", "fsck", "--objects"])
.stdout(Stdio::null()),
"git lfs fsck",
)?;
}
let head = capture(git(&dst).args(["rev-parse", "HEAD"]), "git rev-parse")?;
if head.trim() != r.commit {
bail!(
"source {}: HEAD is {}, the pin is {}",
r.name,
head.trim(),
r.commit
);
}
// Also refreshes the index, so git inside the read-only container finds it clean.
let dirty = capture(git(&dst).args(["status", "--porcelain"]), "git status")?;
if !dirty.trim().is_empty() {
let o = git(&dst)
.args(["lfs", "fsck", "--pointers"])
.output()
.map_err(|e| Error(format!("git lfs fsck --pointers: {e}")))?;
let fsck = format!(
"{}{}",
String::from_utf8_lossy(&o.stdout),
String::from_utf8_lossy(&o.stderr)
);
let real = unexplained_dirt(&dirty, &fsck);
if !real.is_empty() {
bail!(
"source {}: the checkout is not clean:\n{}",
r.name,
real.join("\n")
);
}
say(&format!("source {}: {} file(s) committed raw against their lfs attribute (shown modified; the commit's own content)", r.name, dirty.lines().count()));
}
Ok(())
}
fn index_path(c: &str) -> String {
match c.len() {
1 => format!("1/{c}"),
2 => format!("2/{c}"),
3 => format!("3/{}/{c}", &c[..1]),
_ => format!("{}/{}/{c}", &c[..2], &c[2..4]),
}
}
/// GET `url`: `Ok(None)` on 404/410.
fn http_get(url: &str, limit: u64) -> Result<Option<Vec<u8>>> {
match ureq::get(url).header("User-Agent", USER_AGENT).call() {
Ok(mut r) => r
.body_mut()
.with_config()
.limit(limit)
.read_to_vec()
.map(Some)
.map_err(|e| Error(format!("{url}: {e}"))),
Err(ureq::Error::StatusCode(404 | 410)) => Ok(None),
Err(e) => Err(Error(format!("{url}: {e}"))),
}
}
/// A file from `url` into `to` (what crates.io serves, as served).
pub(crate) fn http_fetch(url: &str, to: &Path) -> Result<()> {
let bytes = http_get(url, 1 << 30)?.ok_or_else(|| Error(format!("{url}: not found")))?;
std::fs::write(to, bytes)?;
Ok(())
}
/// Every entry crates.io's index has for a crate (one JSON object per version); none
/// if it has no such crate.
fn crates_io_entries(name: &str) -> Result<Vec<Value>> {
let url = format!(
"https://index.crates.io/{}",
index_path(&name.to_ascii_lowercase())
);
let Some(body) = http_get(&url, 64 << 20)? else {
return Ok(Vec::new());
};
String::from_utf8_lossy(&body)
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).map_err(Error::from))
.collect()
}
/// The checksum crates.io's index gives that version, or `None` if it has none.
pub(crate) fn crates_io_cksum(name: &str, version: &str) -> Result<Option<String>> {
Ok(crates_io_entries(name)?
.iter()
.find(|e| e.get("vers").and_then(Value::as_str) == Some(version))
.and_then(|e| e.get("cksum").and_then(Value::as_str).map(str::to_string)))
}
/// A `.crate` from static.crates.io into `to`, checked against `sum`.
pub(crate) fn crates_io_download(name: &str, version: &str, sum: &str, to: &Path) -> Result<()> {
let url = format!("https://static.crates.io/crates/{name}/{name}-{version}.crate");
let bytes = http_get(&url, 1 << 30)?.ok_or_else(|| Error(format!("{url}: not found")))?;
let got = sha256_hex(&bytes);
if got != sum {
bail!("crates.io {name} {version}: the download's sha256 {got} is not the index's {sum}");
}
std::fs::write(to, bytes)?;
Ok(())
}
/// What packaging takes from crates.io for a patched external: the newest
/// semver-compatible version that is not yanked, and every feature it has.
fn crates_io_pick(name: &str, version: &str) -> Result<Option<(String, Vec<String>)>> {
let entries = crates_io_entries(name)?;
let live: Vec<&Value> = entries
.iter()
.filter(|e| !e.get("yanked").and_then(Value::as_bool).unwrap_or(false))
.collect();
let versions: Vec<String> = live
.iter()
.filter_map(|e| e.get("vers").and_then(Value::as_str).map(str::to_string))
.collect();
let Some(picked) = crate::lock::newest_compatible(&versions, version) else {
return Ok(None);
};
let e = live
.iter()
.find(|e| e.get("vers").and_then(Value::as_str) == Some(picked.as_str()))
.copied();
let mut features: BTreeSet<String> = BTreeSet::new();
for key in ["features", "features2"] {
if let Some(m) = e.and_then(|e| e.get(key)).and_then(Value::as_object) {
features.extend(m.keys().cloned());
}
}
Ok(Some((picked, features.into_iter().collect())))
}
/// Every file and directory under `dir` read-only (`a-w`), in Rust.
pub fn set_read_only(dir: &Path) -> Result<()> {
set_writable(dir, false)
}
/// `u+w` (or `a-w`) on everything under `dir`; links are left alone.
pub fn set_writable(dir: &Path, writable: bool) -> Result<()> {
use std::os::unix::fs::PermissionsExt as _;
let m = std::fs::symlink_metadata(dir)?;
if m.file_type().is_symlink() {
return Ok(());
}
let mode = m.permissions().mode();
let set = |p: &Path, mode: u32| std::fs::set_permissions(p, std::fs::Permissions::from_mode(mode));
if writable {
set(dir, mode | 0o200)?;
}
if m.is_dir() {
for e in std::fs::read_dir(dir)? {
set_writable(&e?.path(), writable)?;
}
}
if !writable {
set(dir, mode & !0o222)?;
}
Ok(())
}
/// ★ Stage a `.crate` for the next steps: unpack it into `<stage>/<name>-<version>/`
/// with the `.cargo-checksum.json` a directory source needs, whose `package` is the
/// `.crate`'s sha256 (so a dependent's packed `Cargo.lock` names the checksum crates.io
/// will serve). A vendored crate of the same name and version (a probe pick) gives way
/// to the release's own; a step staged twice is refused. Returns the sha256.
pub fn stage_crate(stage: &Path, name: &str, version: &str, file: &Path) -> Result<String> {
let dir = stage.join(format!("{name}-{version}"));
match std::fs::symlink_metadata(&dir) {
Ok(m) if m.file_type().is_symlink() => std::fs::remove_file(&dir)?,
Ok(_) => bail!("{name} {version} is staged twice (a vendored crate of the same name and version?)"),
Err(_) => {}
}
let bytes = std::fs::read(file).map_err(|e| Error(format!("{}: {e}", file.display())))?;
let sum = sha256_hex(&bytes);
let mut ar = tar::Archive::new(flate2::read::GzDecoder::new(&bytes[..]));
for entry in ar.entries()? {
let mut entry = entry?;
let path = entry.path()?.into_owned();
let mut comps = path.components();
let top = comps.next().map(|c| c.as_os_str().to_string_lossy().into_owned());
if top.as_deref() != Some(&format!("{name}-{version}")[..])
|| path.components().any(|c| !matches!(c, Component::Normal(_)))
{
bail!("{}: entry {} is not under {name}-{version}/", file.display(), path.display());
}
if !entry.unpack_in(stage)? {
bail!("{}: entry {} would leave the stage", file.display(), path.display());
}
}
if !dir.is_dir() {
bail!("{} did not unpack into {name}-{version}/", file.display());
}
let mut files = BTreeMap::new();
let mut stack = vec![dir.clone()];
while let Some(d) = stack.pop() {
for e in std::fs::read_dir(&d)? {
let e = e?;
let p = e.path();
if e.file_type()?.is_dir() {
stack.push(p);
} else if p.file_name().is_some_and(|n| n != ".cargo-checksum.json") {
let rel = p.strip_prefix(&dir).map_err(|e| Error(e.to_string()))?;
files.insert(rel.display().to_string(), sha256_file(&p)?);
}
}
}
std::fs::write(
dir.join(".cargo-checksum.json"),
serde_json::to_vec(&json!({"files": files, "package": sum}))?,
)?;
Ok(sum)
}
/// The cargo home of the build containers: crates.io replaced by the stage, offline.
pub const STAGED_CONFIG: &str = "[source.crates-io]\nreplace-with = \"staged\"\n[source.staged]\ndirectory = \"/stage\"\n[net]\noffline = true\n";
/// ★ The builder details (`runDetails.builder.version`): dwarves-slsa's version, the
/// runner binary's sha256 (`runnerSha256`), the account it ran as (`runnerAccount`,
/// `runnerUid`), then the caller's entries (the dwarves server's version and commit;
/// release mode's mode, binary and request), which win.
pub fn builder_version(
extra: &BTreeMap<String, String>,
exe_sha256: &str,
account: Option<&crate::release_mode::Identity>,
) -> BTreeMap<String, String> {
let mut v = BTreeMap::new();
v.insert(
"dwarves-slsa".to_string(),
env!("CARGO_PKG_VERSION").to_string(),
);
v.insert("runnerSha256".to_string(), exe_sha256.to_string());
if let Some(a) = account {
v.insert("runnerAccount".to_string(), a.name.clone());
v.insert("runnerUid".to_string(), a.uid.to_string());
}
v.extend(extra.clone());
v
}
/// This executable's sha256.
pub fn exe_sha256() -> String {
std::env::current_exe()
.ok()
.and_then(|p| sha256_file(&p).ok())
.unwrap_or_default()
}
fn random_hex() -> String {
let mut b = [0u8; 8];
if let Ok(mut f) = File::open("/dev/urandom") {
let _ = std::io::Read::read_exact(&mut f, &mut b);
}
hex::encode(b)
}
#[derive(Default)]
struct State {
locks: Vec<Vec<LockedPackage>>,
lock_deps: Vec<ResourceDescriptor>,
externals: Vec<(String, String, Vec<String>)>,
vendored: usize,
}
/// ★ One release build in flight: the spec, its run directory, what crates.io already
/// had, and the facts the forge's actions find (the locks, the patched externals, the
/// vendored crates). The executor hands each `Forge` step to [`ReleaseRun::forge`].
pub struct ReleaseRun {
pub spec: ReleaseSpec,
pub layout: Layout,
pub from: Option<PathBuf>,
/// `(name, version)` → crates.io's sha256: staged from crates.io, not rebuilt.
pub published: BTreeMap<(String, String), String>,
state: Mutex<State>,
}
fn arg<'a>(args: &'a [String], i: usize, action: &str) -> Result<&'a str> {
args.get(i)
.map(String::as_str)
.ok_or_else(|| Error(format!("{action}: argument {i} is missing")))
}
impl ReleaseRun {
pub fn new(spec: ReleaseSpec, layout: Layout, from: Option<PathBuf>, published: BTreeMap<(String, String), String>) -> Self {
Self {
spec,
layout,
from,
published,
state: Mutex::new(State::default()),
}
}
fn state(&self) -> std::sync::MutexGuard<'_, State> {
self.state.lock().unwrap_or_else(|e| e.into_inner())
}
/// ★ Run one of the forge's actions ([`ACTIONS`]) of this release; `say` is the
/// step's log. An action this run does not know is refused by name.
pub fn forge(&self, action: &str, args: &[String], say: &mut dyn FnMut(&str)) -> Result<()> {
let l = &self.layout;
match action {
FORGE_CHECKOUT => {
let name = arg(args, 0, action)?;
let r = self
.spec
.repo(name)
.ok_or_else(|| Error(format!("{action}: repository {name} is not pinned")))?;
snapshot(&self.spec, r, l, self.from.as_deref(), say)
}
FORGE_SEAL_SOURCES => self.seal_sources(say),
FORGE_VENDOR_CHECK => self.vendor_check(say),
FORGE_STAGE => {
let (name, version) = (arg(args, 0, action)?, arg(args, 1, action)?);
let key = (name.to_string(), version.to_string());
let file = if self.published.contains_key(&key) {
l.published().join(format!("{name}-{version}.crate"))
} else {
let c = self
.spec
.crates
.iter()
.find(|c| c.name == name && c.version == version)
.ok_or_else(|| Error(format!("{action}: {name} {version} is not a step")))?;
let f = l.target().join(&c.repo).join("package").join(format!("{name}-{version}.crate"));
if !f.is_file() {
bail!("cargo package left no {}", f.display());
}
std::fs::create_dir_all(l.out().join("crates"))?;
std::fs::copy(&f, l.out().join("crates").join(format!("{name}-{version}.crate")))?;
f
};
let sum = stage_crate(&l.stage(), name, version, &file)?;
say(&format!("staged {name} {version} sha256:{sum}"));
Ok(())
}
FORGE_PREPARE_BINARY => {
let (name, version) = (arg(args, 0, action)?, arg(args, 1, action)?);
let d = format!("{name}-{version}");
let to = l.work().join(&d);
let _ = std::fs::remove_dir_all(&to);
copy_tree(&l.stage().join(&d), &to)?;
say(&format!("binary sources: the packaged {name} {version}"));
Ok(())
}
FORGE_COLLECT_BINARY => {
let (bin, from) = (arg(args, 0, action)?, arg(args, 1, action)?);
if !["bin", "ws"].contains(&from) {
bail!("{action}: {from:?} is not bin or ws");
}
let src = l.target().join(from).join("release").join(bin);
let dst = l.out().join("bin").join(bin);
std::fs::create_dir_all(l.out().join("bin"))?;
if dst.exists() {
bail!("two binaries named {bin}");
}
std::fs::copy(&src, &dst).map_err(|e| Error(format!("{}: {e}", src.display())))?;
say(&format!("binary {bin} -> out/bin/{bin}"));
Ok(())
}
FORGE_COLLECT_EXAMPLE => {
let i = arg(args, 0, action)?;
let w = i
.parse::<usize>()
.ok()
.and_then(|n| self.spec.workspace_examples.get(n))
.ok_or_else(|| Error(format!("{action}: {i} is not a workspace example of this release")))?;
let from = l.example_out().join(i);
let mut found = BTreeSet::new();
if from.is_dir() {
regular_files(&from, "", &mut found)?;
}
let declared: BTreeSet<String> = w.outputs.iter().cloned().collect();
if let Some(x) = found.difference(&declared).next() {
bail!("example {}: it wrote {x}, which is not declared as an output", w.example);
}
if let Some(x) = declared.difference(&found).next() {
bail!("example {}: the declared output {x} is missing", w.example);
}
if let Some(x) = declared.iter().find(|o| l.out().join(o).exists()) {
bail!("example {}: out/{x} exists already (an output collected twice)", w.example);
}
for o in &w.outputs {
let to = l.out().join(o);
if let Some(p) = to.parent() {
std::fs::create_dir_all(p)?;
}
std::fs::copy(from.join(o), &to)?;
say(&format!("example {} -> out/{o} sha256:{}", w.example, sha256_file(&to)?));
}
Ok(())
}
other => bail!("{other:?} is not an action of a release build ({})", ACTIONS.join(", ")),
}
}
fn seal_sources(&self, say: &mut dyn FnMut(&str)) -> Result<()> {
let (l, spec) = (&self.layout, &self.spec);
set_read_only(&l.src())?;
// what crates.io already has: staged from crates.io, not rebuilt
for ((name, version), sum) in &self.published {
crates_io_download(name, version, sum, &l.published().join(format!("{name}-{version}.crate")))?;
say(&format!("published {name} {version}: crates.io has it (sha256 {sum}); staged, not rebuilt"));
}
set_read_only(&l.published())?;
let mut st = self.state();
for m in &spec.locks {
let rel = format!("{}/Cargo.lock", lock_dir(m));
let text = std::fs::read_to_string(l.src().join(&rel)).map_err(|_| {
Error(format!(
"{rel}: no committed Cargo.lock (a release resolves from a committed lock)"
))
})?;
let repo_name = rel.split('/').next().unwrap_or("");
let r = spec
.repo(repo_name)
.ok_or_else(|| Error(format!("{rel}: repository not pinned")))?;
let in_repo = rel.split_once('/').map(|(_, p)| p).unwrap_or(&rel);
st.lock_deps.push(ResourceDescriptor::new(
Some(&rel),
&format!("git+{}.git@{}#{in_repo}", spec.web_url(r), r.commit),
"sha256",
&sha256_hex(text.as_bytes()),
));
st.locks.push(parse_lock(&text)?);
}
// the patched externals: path packages the release reaches that packaging names
// from crates.io (gunnar's vendor/gix-features). The probe vendors those.
let step_names: BTreeSet<String> = spec.crates.iter().map(|c| c.name.clone()).collect();
let mut externals = Vec::new();
for (name, version) in crate::lock::path_externals(&st.locks, &step_names) {
match crates_io_pick(&name, &version)? {
Some((picked, features)) => {
say(&format!("patched external {name} {version}: packaging takes {name} {picked} from crates.io"));
externals.push((name, picked, features));
}
None => say(&format!("patched external {name} {version}: crates.io has no compatible version (left to packaging)")),
}
}
std::fs::write(l.probe().join("Cargo.toml"), probe_manifest(&externals))?;
std::fs::write(l.probe().join("lib.rs"), "")?;
st.externals = externals;
Ok(())
}
fn vendor_check(&self, say: &mut dyn FnMut(&str)) -> Result<()> {
let l = &self.layout;
let mut st = self.state();
if !st.externals.is_empty() {
let text = std::fs::read_to_string(l.probe().join("Cargo.lock"))?;
st.lock_deps.push(ResourceDescriptor::new(
Some("probe/Cargo.lock"),
"file:probe/Cargo.lock",
"sha256",
&sha256_hex(text.as_bytes()),
));
st.locks.push(parse_lock(&text)?);
}
set_read_only(&l.probe())?;
let _ = set_writable(&l.fetch_home(), true);
let _ = std::fs::remove_dir_all(l.fetch_home());
let crates = l.vendor().join("crates");
std::fs::create_dir_all(&crates)?;
set_read_only(&l.vendor())?;
st.vendored = verify_vendor(&crates, &st.locks)?;
say(&format!("vendor: {} crates, each with its lock checksum", st.vendored));
// the stage: every vendored crate (as the container sees it), then each step
for e in std::fs::read_dir(&crates)? {
let n = e?.file_name();
std::os::unix::fs::symlink(Path::new("/vendor/crates").join(&n), l.stage().join(&n))?;
}
std::fs::write(l.cargo_home().join("config.toml"), STAGED_CONFIG)?;
set_read_only(&l.plan())?;
Ok(())
}
}
/// ★ **The release as a Runa pipeline** — validated by Runa's own gate, and every
/// sealed container by [`check_isolation`] — over `layout`. `published` are the steps
/// crates.io already has (staged, not packaged).
pub fn release_pipeline(spec: &ReleaseSpec, layout: &Layout, published: &BTreeSet<(String, String)>, jobs: u32) -> Result<Pipeline> {
let mut stages: Vec<Stage> = Vec::new();
let push = |stages: &mut Vec<Stage>, name: String, steps: Vec<Step>| {
let needs = stages.last().map(|s| vec![s.name.clone()]).unwrap_or_default();
stages.push(Stage {
name,
needs,
runs_on: None,
steps,
});
};
let forge = |action: &str, args: &[&str]| Step::Forge {
action: action.to_string(),
args: args.iter().map(|s| s.to_string()).collect(),
};
let sealed = |c: Sealed, build: bool| -> Result<Step> {
check_isolation(&c, layout, build)?;
Ok(c.to_step())
};
push(
&mut stages,
"sources".into(),
spec.repos.iter().map(|r| forge(FORGE_CHECKOUT, &[&r.name])).collect(),
);
push(&mut stages, "seal-sources".into(), vec![forge(FORGE_SEAL_SOURCES, &[])]);
push(&mut stages, "fetch".into(), vec![sealed(fetch_container(layout, spec), false)?]);
push(&mut stages, "probe-lock".into(), vec![sealed(probe_lock_container(layout, spec), false)?]);
push(&mut stages, "probe-vendor".into(), vec![sealed(probe_vendor_container(layout, spec), false)?]);
push(&mut stages, "vendor-check".into(), vec![forge(FORGE_VENDOR_CHECK, &[])]);
for (i, c) in spec.crates.iter().enumerate() {
if !published.contains(&(c.name.clone(), c.version.clone())) {
push(
&mut stages,
format!("package-{i}-{}", c.name),
vec![sealed(package_container(layout, spec, c, jobs), true)?],
);
}
push(
&mut stages,
format!("stage-{i}-{}", c.name),
vec![forge(FORGE_STAGE, &[&c.name, &c.version])],
);
}
for (i, b) in spec.binaries.iter().enumerate() {
let v = version_of(spec, &b.crate_name).to_string();
push(&mut stages, format!("binary-{i}-{}-sources", b.bin), vec![forge(FORGE_PREPARE_BINARY, &[&b.crate_name, &v])]);
push(&mut stages, format!("binary-{i}-{}", b.bin), vec![sealed(binary_container(layout, spec, b, jobs), true)?]);
push(&mut stages, format!("binary-{i}-{}-out", b.bin), vec![forge(FORGE_COLLECT_BINARY, &[&b.bin, "bin"])]);
}
for (i, w) in spec.workspace_binaries.iter().enumerate() {
push(&mut stages, format!("workspace-binary-{i}-{}", w.bin), vec![sealed(workspace_binary_container(layout, spec, w, jobs), true)?]);
push(&mut stages, format!("workspace-binary-{i}-{}-out", w.bin), vec![forge(FORGE_COLLECT_BINARY, &[&w.bin, "ws"])]);
}
for (i, w) in spec.workspace_examples.iter().enumerate() {
push(&mut stages, format!("workspace-example-{i}-{}", w.example), vec![sealed(workspace_example_container(layout, spec, i, w, jobs), true)?]);
push(&mut stages, format!("workspace-example-{i}-{}-out", w.example), vec![forge(FORGE_COLLECT_EXAMPLE, &[&i.to_string()])]);
}
let p = Pipeline {
name: PIPELINE.into(),
triggers: vec![Trigger::Manual],
runs_on: Default::default(),
stages,
};
p.validate().map_err(|e| Error(format!("the release pipeline: {e}")))?;
Ok(p)
}
// ───────────────────────────── the run ─────────────────────────────
/// One step of a finished Runa run.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StepRecord {
pub stage: String,
pub step: String,
/// `passed`, `failed`, `skipped`.
pub status: String,
pub duration_ms: u64,
/// The step's log, when the executor kept one.
pub log: Option<PathBuf>,
}
/// ★ Runa's record of a finished run: what the provenance is written from.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunRecord {
pub run_id: String,
pub pipeline: String,
pub green: bool,
pub steps: Vec<StepRecord>,
/// Why it is red (the first failed step, or the cap that stopped it).
pub reason: Option<String>,
}
/// ★ Runs a release pipeline: the dwarves forge's Runa runner (sealed containers through
/// its one container engine, `Forge` steps handed to [`ReleaseRun::forge`], CI's caps and
/// cancel, the run in the forge's history).
pub trait Executor {
fn execute(&self, pipeline: &Pipeline, run: Arc<ReleaseRun>, run_id: &str) -> Result<RunRecord>;
}
/// How to run one release.
#[derive(Debug, Clone)]
pub struct RunOptions {
/// A fresh (missing or empty) run directory.
pub run_dir: PathBuf,
/// Local clones (`<dir>/<repo>`) to copy objects from; every commit is still
/// checked out by its id, and missing objects come from the forge.
pub from: Option<PathBuf>,
pub jobs: u32,
/// Package steps crates.io already has too (a reproducibility check; such a
/// run's crates are not for publishing).
pub rebuild_published: bool,
/// Keep vendor, stage, target and cargo home after the run.
pub keep: bool,
/// The Runa run id (default `<release>-<utc>-<random>`).
pub run_id: Option<String>,
/// The caller's own identity, added to `runDetails.builder.version` (the dwarves
/// server's version and commit).
pub builder_version: BTreeMap<String, String>,
}
struct Log(File);
impl Log {
fn say(&mut self, s: &str) {
eprintln!("{s}");
let _ = writeln!(self.0, "{s}");
}
}
/// ★ Run one release as a Runa pipeline through `exec`, then write the provenance from
/// the run's record. Returns the path of the unsigned statement.
pub fn run(spec: &ReleaseSpec, opts: &RunOptions, exec: &dyn Executor) -> Result<PathBuf> {
spec.validate()?;
let root = std::path::absolute(&opts.run_dir)?;
if root.exists() && std::fs::read_dir(&root)?.next().is_some() {
bail!(
"run directory {} is not empty: every release gets a fresh one",
root.display()
);
}
let layout = Layout::new(&root);
for d in [
layout.src(),
layout.vendor(),
layout.fetch_home(),
layout.cargo_home(),
layout.target(),
layout.out(),
layout.plan(),
layout.published(),
layout.probe(),
layout.stage(),
layout.work(),
layout.example_out(),
layout.logs(),
] {
std::fs::create_dir_all(d)?;
}
let mut log = Log(File::create(layout.logs().join("runner.log"))?);
let started = crate::time::now_secs();
let invocation_id = format!(
"{}/{}-{}",
spec.name,
crate::time::compact(started),
random_hex()
);
let run_id = opts
.run_id
.clone()
.unwrap_or_else(|| format!("{}-{}", spec.name, crate::time::compact(started)));
log.say(&format!(
"dwarves-slsa {}: release {} invocation {invocation_id}, Runa run {run_id}",
env!("CARGO_PKG_VERSION"),
spec.name
));
log.say(&format!("run directory {}", root.display()));
// what crates.io already has (staged from crates.io by seal-sources, not rebuilt)
let mut published = BTreeMap::new();
if !opts.rebuild_published {
for c in &spec.crates {
if let Some(sum) = crates_io_cksum(&c.name, &c.version)? {
published.insert((c.name.clone(), c.version.clone()), sum);
}
}
if published.len() == spec.crates.len()
&& spec.binaries.is_empty()
&& spec.workspace_binaries.is_empty()
&& spec.workspace_examples.is_empty()
{
bail!("every crate of {} is on crates.io already: nothing to build (bump the versions, or --rebuild-published to check reproducibility)", spec.name);
}
}
let published_set: BTreeSet<(String, String)> = published.keys().cloned().collect();
let steps = steps_file(spec, &published_set);
std::fs::write(layout.plan().join("steps"), &steps)?;
std::fs::write(layout.plan().join("binaries"), binaries_file(spec))?;
std::fs::write(layout.plan().join("workspace_binaries"), workspace_binaries_file(spec))?;
// The spec as run, for the publish step (it reads the image and the steps here).
std::fs::write(layout.plan().join("spec.toml"), spec.to_toml()?)?;
let pipeline = release_pipeline(spec, &layout, &published_set, opts.jobs)?;
let ron = pipeline
.to_ron_string()
.map_err(|e| Error(format!("the release pipeline: {e}")))?;
std::fs::write(layout.pipeline(), &ron)?;
log.say(&format!(
"pipeline {PIPELINE}: {} stages ({})",
pipeline.stages.len(),
layout.pipeline().display()
));
let rr = Arc::new(ReleaseRun::new(spec.clone(), layout.clone(), opts.from.clone(), published.clone()));
let record = exec.execute(&pipeline, rr.clone(), &run_id)?;
for s in &record.steps {
log.say(&format!(" {} · {} · {}", s.stage, s.step, s.status));
}
if !record.green {
bail!(
"Runa run {} is red: {}",
record.run_id,
record.reason.as_deref().unwrap_or("a step failed")
);
}
let finished = crate::time::now_secs();
// the provenance, from outside, from Runa's record
let subjects = subjects_of_dir(&layout.out())?;
let names: BTreeSet<&str> = subjects.iter().map(|s| s.name.as_str()).collect();
for c in spec
.crates
.iter()
.filter(|c| !published_set.contains(&(c.name.clone(), c.version.clone())))
{
let n = format!("crates/{}-{}.crate", c.name, c.version);
if !names.contains(n.as_str()) {
bail!("the build left no {n}");
}
}
for bin in spec
.binaries
.iter()
.map(|b| &b.bin)
.chain(spec.workspace_binaries.iter().map(|w| &w.bin))
{
if !names.contains(format!("bin/{bin}").as_str()) {
bail!("the build left no bin/{bin}");
}
}
for o in spec.workspace_examples.iter().flat_map(|w| &w.outputs) {
if !names.contains(o.as_str()) {
bail!("the build left no {o}");
}
}
let st = rr.state();
let mut resolved: Vec<ResourceDescriptor> = spec
.repos
.iter()
.map(|r| {
ResourceDescriptor::new(
Some(&r.name),
&format!("git+{}.git", spec.web_url(r)),
"gitCommit",
&r.commit,
)
})
.collect();
resolved.push(ResourceDescriptor::new(
Some("build-image"),
&format!("oci://{}", spec.image_name()),
"sha256",
spec.image_digest(),
));
resolved.extend(st.lock_deps.iter().cloned());
for ((n, v), sum) in &published {
resolved.push(ResourceDescriptor::new(
Some(&format!("crates.io:{n}")),
&format!("pkg:cargo/{n}@{v}"),
"sha256",
sum,
));
}
resolved.extend(dependencies_of(&st.locks));
let builder_version = builder_version(
&opts.builder_version,
&exe_sha256(),
crate::release_mode::current_identity().ok().as_ref(),
);
let mut byproducts = vec![ResourceDescriptor::new(
Some("logs/pipeline.runa.ron"),
"file:logs/pipeline.runa.ron",
"sha256",
&sha256_hex(ron.as_bytes()),
)];
for (i, s) in record.steps.iter().enumerate() {
if let Some(p) = s.log.as_ref().filter(|p| p.is_file()) {
byproducts.push(ResourceDescriptor::new(
Some(&format!("runa:{}/{i} {}", record.run_id, s.stage)),
&format!("file:{}", p.display()),
"sha256",
&sha256_file(p)?,
));
}
}
let p = Provenance {
build_type: BUILD_TYPE_CARGO_RELEASE.to_string(),
external_parameters: spec.external_parameters(),
internal_parameters: json!({
"image": spec.image,
"runa": {
"pipeline": record.pipeline,
"runId": record.run_id,
"pipelineSha256": sha256_hex(ron.as_bytes()),
"stages": pipeline.stages.len(),
"steps": record.steps.iter().map(|s| format!("{} {} {}", s.stage, s.step, s.status)).collect::<Vec<_>>(),
},
"sandbox": {"engine": "draupnir (podman REST)", "readOnlyRootfs": true, "capabilities": "none", "noNewPrivileges": true, "pull": "never", "shell": false},
"fetch": {"network": "on", "command": "cargo vendor --locked --versioned-dirs", "runsReleaseCode": false},
"build": {"network": "none", "cargo": ["--locked", "--offline"], "sccache": false, "sharedCaches": false},
"steps": steps.lines().collect::<Vec<_>>(),
"jobs": opts.jobs,
"rebuildPublished": opts.rebuild_published,
"vendoredCrates": st.vendored,
"patchedExternals": st.externals.iter().map(|(n, v, _)| format!("{n} {v}")).collect::<Vec<_>>(),
"clone": {"forge": spec.forge, "forgeByOrg": spec.forge_by_org},
}),
resolved_dependencies: resolved,
builder_id: BUILDER_ID.to_string(),
builder_version,
invocation_id,
started_on: crate::time::rfc3339_secs(started),
finished_on: crate::time::rfc3339_secs(finished),
byproducts,
};
drop(st);
let st = statement::statement(&subjects, &p)?;
std::fs::write(layout.statement(), statement::payload(&st))?;
log.say(&format!(
"provenance: {} subject(s) -> {}",
subjects.len(),
layout.statement().display()
));
if !opts.keep {
for d in [layout.target(), layout.cargo_home(), layout.work(), layout.stage(), layout.vendor(), layout.example_out()] {
let _ = set_writable(&d, true);
let _ = std::fs::remove_dir_all(d);
}
}
Ok(layout.statement())
}