ci/verbs/src/fuzz.rs
//! `ci-verbs fuzz [--checkout <dir>] [--private <sibling>] [--out <dir>] [--seconds <n>] [--corpus <dir>]
//! [--target-dir <dir>] [--only <target>]...` (OSSF-NEXT): the nightly fuzz run of a
//! checkout's cargo-fuzz crate, as a step the dwarves forge runs (a Runa
//! `Xtask(cmd: "fuzz", alias: "ci-verbs")`).
//!
//! * **The crate** is `<checkout>/fuzz/Cargo.toml`, the cargo-fuzz layout OpenSSF
//! Scorecard's Fuzzing check looks for (`libfuzzer-sys` in a `.rs` file). It must say
//! `[package.metadata] cargo-fuzz = true`; its `[[bin]]`s are the targets
//! ([`targets_from_metadata`], from `cargo metadata --no-deps`, so no TOML parser here).
//! * **The build** is plain STABLE cargo (ci4 carries only stable): the flags cargo-fuzz
//! passes with `--sanitizer none` ([`RUSTFLAGS`]: SanitizerCoverage through LLVM, no
//! `-Z`), `--release --locked --bins`, and `--target <host>` so the flags reach only the
//! fuzz binaries, never a build script or a proc-macro. The target directory is
//! `--target-dir`, else `$CARGO_TARGET_DIR/fuzz`, else `<out>/fuzz-target`.
//! * **The run**: every target for `--seconds` (default 300) as
//! `<bin> -max_total_time=<n> -rss_limit_mb=4096 -artifact_prefix=<crashes>/<t>/
//! <corpus>/<t> <checkout>/fuzz/corpus/<t>`. libFuzzer adds what it finds to the FIRST
//! corpus directory, the kept one (`--corpus`, else `$DWARVES_CI_FUZZ_CORPUS/<repo>`,
//! else `<out>/corpus`), so the corpus grows night after night; the committed seeds are
//! read only.
//! * **A crash is a red step**: the input libFuzzer wrote is kept under
//! `<out>/crashes/<target>/`, and `summary.md` names it and shows its first bytes in
//! hex, so the red run on the CI page says what broke without anyone logging in. A
//! target that does not build or start is red too.
use std::path::{Path, PathBuf};
use std::time::Instant;
use serde_json::Value;
/// The flags `cargo fuzz build --sanitizer none` gives rustc: stable, no `-Z`.
pub const RUSTFLAGS: &str = "--cfg fuzzing -Cpasses=sancov-module -Cllvm-args=-sanitizer-coverage-level=4 -Cllvm-args=-sanitizer-coverage-inline-8bit-counters -Cllvm-args=-sanitizer-coverage-pc-table -Cllvm-args=-sanitizer-coverage-trace-compares -Cdebug-assertions -Coverflow-checks";
/// How long each target runs when the pipeline does not say.
pub const DEFAULT_SECONDS: u64 = 300;
/// **The fuzz targets of a cargo-fuzz crate**: the `bin` targets of its package, sorted,
/// from `cargo metadata --no-deps --format-version 1`. A package without
/// `[package.metadata] cargo-fuzz = true`, or without a bin, is refused by name.
pub fn targets_from_metadata(meta: &Value) -> Result<Vec<String>, String> {
let pkgs = meta["packages"].as_array().ok_or("cargo metadata named no package")?;
let pkg = pkgs
.iter()
.find(|p| p["metadata"]["cargo-fuzz"].as_bool() == Some(true))
.ok_or("fuzz/Cargo.toml is not a cargo-fuzz crate: it lacks `[package.metadata] cargo-fuzz = true`")?;
let mut t: Vec<String> = pkg["targets"]
.as_array()
.map(|ts| {
ts.iter()
.filter(|t| t["kind"].as_array().is_some_and(|k| k.iter().any(|k| k == "bin")))
.filter_map(|t| t["name"].as_str().map(String::from))
.collect()
})
.unwrap_or_default();
t.sort();
if t.is_empty() {
return Err(format!("{} has no fuzz target (no [[bin]])", pkg["name"].as_str().unwrap_or("the fuzz crate")));
}
Ok(t)
}
/// `cargo` arguments of the build (RUSTFLAGS = [`RUSTFLAGS`] comes from the environment).
pub fn build_args(manifest: &Path, triple: &str) -> Vec<String> {
let mut a: Vec<String> = ["build", "--release", "--locked", "--bins", "--manifest-path"].iter().map(|s| s.to_string()).collect();
a.push(manifest.display().to_string());
a.push("--target".into());
a.push(triple.into());
a
}
/// Where cargo puts a target's binary.
pub fn bin_path(target_dir: &Path, triple: &str, name: &str) -> PathBuf {
target_dir.join(triple).join("release").join(name)
}
/// What libFuzzer said at the end of a run.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Stats {
/// Inputs executed.
pub runs: Option<u64>,
/// Coverage points (`cov:`).
pub cov: Option<u64>,
/// Inputs in the corpus (`corp: <n>/…`).
pub corpus: Option<u64>,
}
/// **libFuzzer's numbers** from its stderr: the last `#<n> … cov: … corp: <n>/…` line,
/// and `Done <n> runs` when it is there.
pub fn parse_stats(stderr: &str) -> Stats {
let mut s = Stats::default();
for line in stderr.lines() {
let l = line.trim();
if let Some(rest) = l.strip_prefix('#') {
let mut words = rest.split_whitespace();
if let Some(n) = words.next().and_then(|n| n.parse().ok()) {
s.runs = Some(n);
}
let w: Vec<&str> = l.split_whitespace().collect();
for i in 0..w.len().saturating_sub(1) {
match w[i] {
"cov:" => s.cov = w[i + 1].parse().ok().or(s.cov),
"corp:" => s.corpus = w[i + 1].split('/').next().and_then(|n| n.parse().ok()).or(s.corpus),
_ => {}
}
}
} else if let Some(rest) = l.strip_prefix("Done ") {
if let Some(n) = rest.split_whitespace().next().and_then(|n| n.parse().ok()) {
s.runs = Some(n);
}
}
}
s
}
/// One target's night.
#[derive(Clone, Debug, Default)]
pub struct Outcome {
/// The target's name.
pub name: String,
/// libFuzzer's exit code (`None`: it did not start or was killed).
pub code: Option<i32>,
/// Why it could not run, when it could not.
pub error: Option<String>,
/// The inputs it wrote under the crash directory (crash-, oom-, timeout-, leak-…).
pub artifacts: Vec<PathBuf>,
/// Its numbers.
pub stats: Stats,
/// Wall time.
pub seconds: u64,
}
impl Outcome {
/// Ran to its time bound and found nothing.
pub fn is_clean(&self) -> bool {
self.error.is_none() && self.code == Some(0) && self.artifacts.is_empty()
}
}
fn files_in(dir: &Path) -> Vec<PathBuf> {
let mut v: Vec<PathBuf> = std::fs::read_dir(dir)
.map(|rd| rd.filter_map(|e| e.ok()).map(|e| e.path()).filter(|p| p.is_file()).collect())
.unwrap_or_default();
v.sort();
v
}
/// **Run one target** for `seconds`: the kept corpus `<corpus>/<name>` first (it grows),
/// the committed `seeds` after it (read only), crashes into `<crashes>/<name>/`.
pub fn run_target(name: &str, bin: &Path, seconds: u64, corpus: &Path, seeds: &Path, crashes: &Path) -> Outcome {
let keep = corpus.join(name);
let art = crashes.join(name);
let mut o = Outcome { name: name.to_string(), ..Default::default() };
for d in [&keep, &art] {
if let Err(e) = std::fs::create_dir_all(d) {
o.error = Some(format!("{}: {e}", d.display()));
return o;
}
}
let before = files_in(&art);
let t0 = Instant::now();
let mut c = std::process::Command::new(bin);
c.arg(format!("-max_total_time={seconds}"))
.arg("-rss_limit_mb=4096")
.arg("-print_final_stats=1")
.arg(format!("-artifact_prefix={}/", art.display()))
.arg(&keep);
if seeds.is_dir() {
c.arg(seeds);
}
c.stdin(std::process::Stdio::null());
match c.output() {
Ok(out) => {
o.code = out.status.code();
o.stats = parse_stats(&String::from_utf8_lossy(&out.stderr));
if o.code != Some(0) && o.code.is_none() {
o.error = Some(format!("{} was killed ({})", bin.display(), out.status));
}
}
Err(e) => o.error = Some(format!("could not start {}: {e}", bin.display())),
}
o.seconds = t0.elapsed().as_secs();
o.artifacts = files_in(&art).into_iter().filter(|p| !before.contains(p)).collect();
o
}
fn hex_head(p: &Path, n: usize) -> String {
std::fs::read(p).map(|b| b.iter().take(n).map(|x| format!("{x:02x}")).collect::<String>()).unwrap_or_default()
}
/// **The page of one night**: a row per target, then each kept input with its first
/// bytes in hex.
pub fn summary(repo: &str, sha: &str, outcomes: &[Outcome]) -> String {
let red = outcomes.iter().filter(|o| !o.is_clean()).count();
let mut s = format!(
"Fuzzing (libFuzzer through cargo-fuzz's layout, stable cargo) on `{repo}` at `{}`: **{} of {} targets clean**.\n\n| target | result | runs | coverage | corpus | seconds |\n|---|---|---|---|---|---|\n",
sha.get(..8).unwrap_or(sha),
outcomes.len() - red,
outcomes.len()
);
let n = |v: Option<u64>| v.map(|x| x.to_string()).unwrap_or_else(|| "?".into());
for o in outcomes {
let result = if o.is_clean() {
"clean".to_string()
} else if !o.artifacts.is_empty() {
"CRASH".to_string()
} else if let Some(e) = &o.error {
format!("NOT RUN: {}", e.replace('|', "\\|"))
} else {
format!("FAILED (exit {})", n(o.code.map(|c| c as u64)))
};
s.push_str(&format!("| {} | {result} | {} | {} | {} | {} |\n", o.name, n(o.stats.runs), n(o.stats.cov), n(o.stats.corpus), o.seconds));
}
for o in outcomes {
for a in &o.artifacts {
s.push_str(&format!(
"\n`{}` kept `{}` ({} bytes), first bytes: `{}`\n",
o.name,
a.file_name().map(|f| f.to_string_lossy().into_owned()).unwrap_or_default(),
std::fs::metadata(a).map(|m| m.len()).unwrap_or(0),
hex_head(a, 64)
));
}
}
s
}
fn host_triple() -> String {
std::process::Command::new("rustc")
.arg("-vV")
.output()
.ok()
.and_then(|o| String::from_utf8_lossy(&o.stdout).lines().find_map(|l| l.strip_prefix("host: ").map(String::from)))
.unwrap_or_else(|| "x86_64-unknown-linux-gnu".into())
}
/// **The checkout to fuzz**: `--checkout`, else the run's checkout (`$DWARVES_CI_CHECKOUT`);
/// with `--private <name>`, the private sibling the forge mounts read-only beside it
/// (`<checkout>/../<name>`: a verdict-only pipeline fuzzes the private vendors without their code
/// or logs leaving dwarf). The name is one plain directory name, refused otherwise.
pub fn checkout_of(checkout: Option<String>, private: Option<String>, env_checkout: Option<PathBuf>) -> Result<PathBuf, String> {
let base = checkout.map(PathBuf::from).or(env_checkout).ok_or("no checkout: pass --checkout <dir> (or run under the forge, which sets DWARVES_CI_CHECKOUT)")?;
let Some(name) = private else { return Ok(base) };
let plain = !name.is_empty() && !name.starts_with('.') && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_');
if !plain {
return Err(format!("--private {name:?}: a private sibling is one plain directory name (letters, digits, `-`, `_`)"));
}
Ok(base.join("..").join(name))
}
fn env_dir(k: &str) -> Option<PathBuf> {
std::env::var(k).ok().filter(|v| !v.is_empty()).map(PathBuf::from)
}
/// The verb.
pub fn verb(args: &[String]) -> i32 {
let v = |f: &str| crate::value(args, f);
let checkout = match checkout_of(v("--checkout"), v("--private"), env_dir("DWARVES_CI_CHECKOUT")) {
Ok(c) => c,
Err(e) => {
eprintln!("ci-verbs fuzz: {e}\nusage: ci-verbs fuzz --checkout <dir> [--private <sibling>] [--out <dir>] [--seconds <n>] [--corpus <dir>] [--target-dir <dir>] [--only <target>]...");
return 2;
}
};
let checkout = checkout.canonicalize().unwrap_or(checkout);
let repo = checkout.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
let out = v("--out").map(PathBuf::from).or_else(|| env_dir("NORNIR_CI_OUT")).unwrap_or_else(|| PathBuf::from("."));
if let Err(e) = std::fs::create_dir_all(&out) {
eprintln!("ci-verbs fuzz: {}: {e}", out.display());
return 1;
}
let seconds = v("--seconds").and_then(|s| s.parse().ok()).unwrap_or(DEFAULT_SECONDS);
let corpus = v("--corpus")
.map(PathBuf::from)
.or_else(|| env_dir("DWARVES_CI_FUZZ_CORPUS").map(|d| d.join(&repo)))
.unwrap_or_else(|| out.join("corpus"));
let target_dir = v("--target-dir")
.map(PathBuf::from)
.or_else(|| env_dir("CARGO_TARGET_DIR").map(|d| d.join("fuzz")))
.unwrap_or_else(|| out.join("fuzz-target"));
let only = crate::values(args, "--only");
let sha = std::process::Command::new("git")
.arg("-C")
.arg(&checkout)
.args(["rev-parse", "HEAD"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_default();
let fail = |why: String| {
let _ = std::fs::write(out.join("summary.md"), format!("Fuzzing on `{repo}` at `{}`: **not run**: {why}\n", sha.get(..8).unwrap_or(&sha)));
eprintln!("ci-verbs fuzz: {why}");
1
};
let manifest = checkout.join("fuzz/Cargo.toml");
if !manifest.is_file() {
return fail(format!("{} has no fuzz/Cargo.toml (the cargo-fuzz layout)", checkout.display()));
}
let meta = match std::process::Command::new("cargo")
.args(["metadata", "--no-deps", "--format-version", "1", "--manifest-path"])
.arg(&manifest)
.output()
{
Ok(o) if o.status.success() => serde_json::from_slice::<Value>(&o.stdout).unwrap_or(Value::Null),
Ok(o) => return fail(format!("cargo metadata of fuzz/Cargo.toml: {}", String::from_utf8_lossy(&o.stderr).lines().last().unwrap_or(""))),
Err(e) => return fail(format!("cargo metadata: {e}")),
};
let mut targets = match targets_from_metadata(&meta) {
Ok(t) => t,
Err(e) => return fail(e),
};
if !only.is_empty() {
targets.retain(|t| only.contains(t));
}
let triple = host_triple();
println!("ci-verbs fuzz: {repo}: {} target(s), {seconds} s each, build in {}", targets.len(), target_dir.display());
let build = std::process::Command::new("cargo")
.args(build_args(&manifest, &triple))
.env("RUSTFLAGS", RUSTFLAGS)
.env("CARGO_TARGET_DIR", &target_dir)
.status();
match build {
Ok(s) if s.success() => {}
Ok(s) => return fail(format!("the fuzz build failed ({s}); its errors are in the step log")),
Err(e) => return fail(format!("cargo build: {e}")),
}
let crashes = out.join("crashes");
let mut outcomes = Vec::new();
for t in &targets {
let o = run_target(t, &bin_path(&target_dir, &triple, t), seconds, &corpus, &checkout.join("fuzz/corpus").join(t), &crashes);
println!(
"ci-verbs fuzz: {t}: {} (runs {:?}, cov {:?}, {} s)",
if o.is_clean() { "clean" } else { "RED" },
o.stats.runs,
o.stats.cov,
o.seconds
);
outcomes.push(o);
}
let md = summary(&repo, &sha, &outcomes);
let _ = std::fs::write(out.join("summary.md"), &md);
let rows: Vec<Value> = outcomes
.iter()
.map(|o| {
serde_json::json!({
"target": o.name, "clean": o.is_clean(), "code": o.code, "error": o.error,
"runs": o.stats.runs, "cov": o.stats.cov, "corpus": o.stats.corpus, "seconds": o.seconds,
"artifacts": o.artifacts.iter().map(|a| a.display().to_string()).collect::<Vec<_>>(),
})
})
.collect();
let _ = std::fs::write(
out.join("fuzz.json"),
serde_json::to_vec_pretty(&serde_json::json!({ "repo": repo, "sha": sha, "seconds_each": seconds, "targets": rows })).unwrap_or_default(),
);
let red: Vec<&str> = outcomes.iter().filter(|o| !o.is_clean()).map(|o| o.name.as_str()).collect();
if red.is_empty() {
0
} else {
eprintln!("ci-verbs fuzz: red: {}", red.join(", "));
1
}
}