ci/verbs/src/coverage.rs
//! `ci-verbs coverage [--checkout <dir>] [--out <dir>] [--package <p>]... [--features <f>]...
//! [--min <percent>] [--cargo <cargo>]` (OSSF-NEXT): the nightly test coverage of a
//! checkout, as a step the dwarves forge runs (a Runa `Xtask(cmd: "coverage", alias:
//! "ci-verbs")`).
//!
//! It runs `cargo llvm-cov --json --summary-only` over the named packages (`--package`,
//! else `$DWARVES_CI_COVERAGE_PACKAGES`, space-separated, else the workspace) in the checkout,
//! skipping the tests `$DWARVES_CI_COVERAGE_SKIP` names (space-separated), and writes:
//!
//! * `llvm-cov-export.data`: llvm-cov's own export (not published: it is large);
//! * `coverage.json`: the totals (lines, functions, regions) and one row per crate of the
//! checkout (files outside it, such as registry sources, are not a crate of it);
//! * `summary.md`: the same as a table, which the forge shows on the CI status page.
//!
//! `--min` is a floor on the line coverage: under it the step is red by name. Without
//! it a number is never red; only a run that could not measure is.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use serde_json::Value;
/// covered of count.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Ratio {
/// Covered.
pub covered: u64,
/// Of.
pub count: u64,
}
impl Ratio {
fn of(v: &Value) -> Ratio {
Ratio { covered: v["covered"].as_u64().unwrap_or(0), count: v["count"].as_u64().unwrap_or(0) }
}
/// Percent (100 when there is nothing to cover).
pub fn percent(&self) -> f64 {
if self.count == 0 {
100.0
} else {
100.0 * self.covered as f64 / self.count as f64
}
}
fn add(&mut self, o: Ratio) {
self.covered += o.covered;
self.count += o.count;
}
}
/// The three totals.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Totals {
/// Lines.
pub lines: Ratio,
/// Functions.
pub functions: Ratio,
/// Regions.
pub regions: Ratio,
}
/// One crate of the checkout.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Row {
/// Its directory, relative to the checkout (`crates/gunnar-wire`, `xtask`).
pub name: String,
/// Its lines.
pub lines: Ratio,
}
/// **The totals of an llvm-cov export.**
pub fn totals(export: &Value) -> Result<Totals, String> {
let t = &export["data"][0]["totals"];
if !t.is_object() {
return Err("the llvm-cov export has no totals (no data)".into());
}
Ok(Totals { lines: Ratio::of(&t["lines"]), functions: Ratio::of(&t["functions"]), regions: Ratio::of(&t["regions"]) })
}
/// The crate directory of a file of the checkout: `crates/<name>` under `crates/`, else
/// the first directory.
fn crate_of(rel: &Path) -> Option<String> {
let parts: Vec<String> = rel.components().map(|c| c.as_os_str().to_string_lossy().into_owned()).collect();
match parts.as_slice() {
[c, name, _, ..] if c == "crates" => Some(format!("crates/{name}")),
[first, _, ..] => Some(first.clone()),
_ => None,
}
}
/// The nearest directory above a file of the checkout that holds a `Cargo.toml` (the
/// checkout's root itself does not count: a root file is not a crate's).
fn manifest_dir(root: &Path, rel: &Path) -> Option<String> {
let mut d = rel.parent()?;
while !d.as_os_str().is_empty() {
if root.join(d).join("Cargo.toml").is_file() {
return Some(d.display().to_string());
}
d = d.parent()?;
}
None
}
/// **One row per crate** of the checkout (the nearest directory with a `Cargo.toml`, else
/// `crates/<name>` or the first directory), sorted by name.
pub fn per_crate(export: &Value, root: &Path) -> Vec<Row> {
let mut m: BTreeMap<String, Ratio> = BTreeMap::new();
for f in export["data"][0]["files"].as_array().into_iter().flatten() {
let Some(name) = f["filename"].as_str() else { continue };
let Ok(rel) = Path::new(name).strip_prefix(root) else { continue };
if let Some(c) = manifest_dir(root, rel).or_else(|| crate_of(rel)) {
m.entry(c).or_default().add(Ratio::of(&f["summary"]["lines"]));
}
}
m.into_iter().map(|(name, lines)| Row { name, lines }).collect()
}
/// `cargo` arguments: `llvm-cov --json --summary-only --output-path <out>` and the
/// packages (else `--workspace`) and features.
pub fn args(packages: &[String], features: &[String], out: &Path) -> Vec<String> {
args_skipping(packages, features, &[], out)
}
/// [`args`], and `-- --skip <name>` for each test the step container cannot run
/// (`$DWARVES_CI_COVERAGE_SKIP`, space-separated): libtest skips by name in every binary.
pub fn args_skipping(packages: &[String], features: &[String], skip: &[String], out: &Path) -> Vec<String> {
let mut a: Vec<String> = ["llvm-cov", "--json", "--summary-only", "--output-path"].iter().map(|s| s.to_string()).collect();
a.push(out.display().to_string());
if packages.is_empty() {
a.push("--workspace".into());
}
for p in packages {
a.push("-p".into());
a.push(p.clone());
}
if !features.is_empty() {
a.push("--features".into());
a.push(features.join(","));
}
if !skip.is_empty() {
a.push("--".into());
for s in skip {
a.push("--skip".into());
a.push(s.clone());
}
}
a
}
/// Why the totals are under the floor, if they are.
pub fn below_floor(t: &Totals, min: Option<f64>) -> Option<String> {
let min = min?;
let p = t.lines.percent();
(p < min).then(|| format!("line coverage {p:.1} % is under the floor of {min} %"))
}
fn pct(r: &Ratio) -> String {
format!("{:.1} % ({} / {})", r.percent(), r.covered, r.count)
}
/// **The page**: the totals, the floor, one row per crate.
pub fn summary(repo: &str, sha: &str, t: &Totals, rows: &[Row], min: Option<f64>) -> String {
let mut s = format!(
"Test coverage (cargo llvm-cov) on `{repo}` at `{}`: **{:.1} %** of lines, {:.1} % of functions, {:.1} % of regions.\n",
sha.get(..8).unwrap_or(sha),
t.lines.percent(),
t.functions.percent(),
t.regions.percent()
);
if let Some(m) = min {
s.push_str(&format!("Floor: {m} % of lines ({}).\n", if below_floor(t, min).is_some() { "UNDER IT" } else { "met" }));
}
s.push_str("\n| crate | lines |\n|---|---|\n");
for r in rows {
s.push_str(&format!("| {} | {} |\n", r.name, pct(&r.lines)));
}
s
}
fn env_dir(k: &str) -> Option<PathBuf> {
std::env::var(k).ok().filter(|v| !v.is_empty()).map(PathBuf::from)
}
/// The verb.
pub fn verb(args_in: &[String]) -> i32 {
let v = |f: &str| crate::value(args_in, f);
let Some(checkout) = v("--checkout").map(PathBuf::from).or_else(|| env_dir("DWARVES_CI_CHECKOUT")) else {
eprintln!("usage: ci-verbs coverage --checkout <dir> [--out <dir>] [--package <p>]... [--features <f>]... [--min <percent>] [--cargo <cargo>]");
return 2;
};
let checkout = checkout.canonicalize().unwrap_or(checkout);
let repo = checkout.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
let out = v("--out").map(PathBuf::from).or_else(|| env_dir("NORNIR_CI_OUT")).unwrap_or_else(|| PathBuf::from("."));
if let Err(e) = std::fs::create_dir_all(&out) {
eprintln!("ci-verbs coverage: {}: {e}", out.display());
return 1;
}
let out = out.canonicalize().unwrap_or(out);
let mut packages = crate::values(args_in, "--package");
// the pipeline file is shared: a project names each repository's packages in its env
if packages.is_empty() {
packages = std::env::var("DWARVES_CI_COVERAGE_PACKAGES").unwrap_or_default().split_whitespace().map(String::from).collect();
}
let skip: Vec<String> = std::env::var("DWARVES_CI_COVERAGE_SKIP").unwrap_or_default().split_whitespace().map(String::from).collect();
let features = crate::values(args_in, "--features");
let min = v("--min").and_then(|m| m.parse::<f64>().ok());
let cargo = v("--cargo").unwrap_or_else(|| "cargo".into());
let sha = std::process::Command::new("git")
.arg("-C")
.arg(&checkout)
.args(["rev-parse", "HEAD"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_default();
let fail = |why: String| {
let _ = std::fs::write(out.join("summary.md"), format!("Test coverage on `{repo}` at `{}`: **not measured**: {why}\n", sha.get(..8).unwrap_or(&sha)));
eprintln!("ci-verbs coverage: {why}");
1
};
let raw = out.join("llvm-cov-export.data");
let a = args_skipping(&packages, &features, &skip, &raw);
println!("ci-verbs coverage: {cargo} {} (in {})", a.join(" "), checkout.display());
match std::process::Command::new(&cargo).args(&a).current_dir(&checkout).stdin(std::process::Stdio::null()).status() {
Ok(s) if s.success() => {}
Ok(s) => return fail(format!("cargo llvm-cov exited {s}: a test failed or it could not build; the step log says which")),
Err(e) => return fail(format!("could not run cargo llvm-cov (`{cargo}`): {e}")),
}
let export: Value = match std::fs::read(&raw).map_err(|e| e.to_string()).and_then(|b| serde_json::from_slice(&b).map_err(|e| e.to_string())) {
Ok(v) => v,
Err(e) => return fail(format!("cargo llvm-cov wrote no readable export: {e}")),
};
let t = match totals(&export) {
Ok(t) => t,
Err(e) => return fail(e),
};
let rows = per_crate(&export, &checkout);
let _ = std::fs::write(out.join("summary.md"), summary(&repo, &sha, &t, &rows, min));
let j = serde_json::json!({
"repo": repo, "sha": sha, "packages": packages, "min": min,
"lines": { "covered": t.lines.covered, "count": t.lines.count, "percent": t.lines.percent() },
"functions": { "covered": t.functions.covered, "count": t.functions.count, "percent": t.functions.percent() },
"regions": { "covered": t.regions.covered, "count": t.regions.count, "percent": t.regions.percent() },
"crates": rows.iter().map(|r| serde_json::json!({ "crate": r.name, "covered": r.lines.covered, "count": r.lines.count, "percent": r.lines.percent() })).collect::<Vec<_>>(),
});
let _ = std::fs::write(out.join("coverage.json"), serde_json::to_vec_pretty(&j).unwrap_or_default());
println!("ci-verbs coverage: {repo} {:.1} % of lines", t.lines.percent());
match below_floor(&t, min) {
Some(why) => {
eprintln!("ci-verbs coverage: {why}");
1
}
None => 0,
}
}