crates/mimir-ingest/src/product.rs
//! BUILDER 2 — what a Build Thing run built, in the log.
//!
//! A Build Thing run (a recipe's variant: dwarves' ⚒ pane, `dwarves
//! buildthing`, a runa `Build` step) produces artifacts. Each one lands here as
//!
//! * an **artifact node** (`kind = artifact`, id [`moniker::artifact`] of its
//! sha256): `name` = the file, `crate` = the recipe's repo, `version` = the
//! variant, `ty` = `<triple>:<profile>`, `cfg` = the features, `doc` = one
//! line of provenance;
//! * **BUILT_FROM** edges from it to every crate root of its dependency closure
//! ([`moniker::crate_moniker`]), to the built package's enabled feature nodes
//! ([`moniker::feature`]) and to every commit it read ([`moniker::commit`]);
//! * its **provenance** (recipe, variant, features, toolchain, commits, builder,
//! time, the whole build record) as the release's `bin_ingest_runs.ledger`.
//!
//! They ride the BINARY stream's own tables (`bin_node_events`,
//! `bin_edge_events`, `bin_ingest_runs`, `bin_offsets`) — no table of their own
//! — under the log's pseudo repo [`PRODUCTS_REPO`], with the artifact as the
//! OWNER of its keys ([`ProductFacts::owner`]). A pseudo repo because a product
//! is not one member's: it has no consumer offset, so no member's binary
//! release ever closes its keys and no room scope (members by offset) folds it
//! by accident. The same bytes built again are the same owner: a ledger row,
//! no events.
//!
//! The read half is [`artifacts`]: "which artifacts were built from crate X /
//! commit Y".
use std::collections::{BTreeMap, BTreeSet};
use anyhow::{Context, Result};
use mimir_schema::{EdgeKind, EdgeRow, Level, NodeRow, Origin, kind, moniker};
use serde::{Deserialize, Serialize};
use skade::arrow_array::Array;
use crate::delta::Kind;
use crate::dump::BinaryDump;
use crate::log::{EventLog, FoldScope, str_col};
/// The log's repo for every built product (like `@universe` for the ring).
pub const PRODUCTS_REPO: &str = "@build-thing";
/// One file a Build Thing run produced.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProductArtifact {
/// The file name (`gunnar-server`, `holger.iso`).
pub file: String,
pub bytes: u64,
/// Content hash (hex sha256) — the artifact's identity.
pub sha256: String,
}
/// Everything builder 2 knows about one finished Build Thing run.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProductFacts {
/// The build record's id.
pub build_id: String,
/// The repo whose recipe declares the variant.
pub repo: String,
pub variant: String,
/// The cargo package built, when the variant names one.
#[serde(default)]
pub package: Option<String>,
#[serde(default)]
pub features: Vec<String>,
pub profile: String,
/// The target triple the artifacts are for.
pub triple: String,
/// The toolchain (`rustc -V` line, or the build image).
pub toolchain: String,
/// `user@host` of the builder box.
pub builder: String,
/// RFC 3339.
pub built_at: String,
/// The recipe code's coordinate (`krate:version:hash`), when known.
#[serde(default)]
pub recipe: Option<String>,
/// repo → commit sha, for the recipe's repo and every sibling repo a path
/// dependency of the build lives in.
#[serde(default)]
pub commits: BTreeMap<String, String>,
/// `(name, version)` of every crate of the build's dependency closure.
#[serde(default)]
pub crates: Vec<(String, String)>,
pub artifacts: Vec<ProductArtifact>,
/// The whole build record as filed (JSON), kept in the ledger.
#[serde(default)]
pub record: Option<serde_json::Value>,
}
impl ProductFacts {
/// The owner of `a`'s keys: `<file>@<triple>:<profile>#<sha256[..16]>` —
/// the binary stream's `name@triple:profile`, made per content.
pub fn owner(&self, a: &ProductArtifact) -> String {
let short = &a.sha256[..a.sha256.len().min(16)];
format!("{}#{short}", BinaryDump::artifact_id(&a.file, &self.triple, &self.profile))
}
/// The commit the release is stamped with: the recipe repo's.
pub fn sha(&self) -> &str {
self.commits.get(&self.repo).map(String::as_str).unwrap_or("")
}
/// The version of the built package, when it is in [`Self::crates`].
fn package_version(&self) -> Option<&str> {
let p = self.package.as_deref()?;
self.crates.iter().find(|(n, _)| n == p).map(|(_, v)| v.as_str())
}
/// The binary-stream rows of every artifact: one [`BinaryDump`] each (its
/// node, its BUILT_FROM edges, no symbols; the ledger = provenance).
pub fn dumps(&self) -> Result<Vec<BinaryDump>> {
let mut edges_to: BTreeSet<String> = BTreeSet::new();
for (name, version) in &self.crates {
edges_to.insert(moniker::crate_moniker(name, version));
}
if let (Some(p), Some(v)) = (self.package.as_deref(), self.package_version()) {
for f in &self.features {
edges_to.insert(moniker::feature(p, v, f));
}
}
for (repo, sha) in &self.commits {
edges_to.insert(moniker::commit(repo, sha));
}
let mut out = Vec::with_capacity(self.artifacts.len());
for a in &self.artifacts {
let id = moniker::artifact(&a.sha256);
let mut node = NodeRow::new(&id, kind::ARTIFACT, Level::Crate, &a.file, &self.repo, &self.variant, Origin::Local);
node.ty = Some(format!("{}:{}", self.triple, self.profile));
node.cfg = (!self.features.is_empty()).then(|| self.features.join(","));
node.doc = Some(format!(
"{} B · built {} by {} · {} · build {}",
a.bytes, self.built_at, self.builder, self.toolchain, self.build_id
));
let edges: Vec<EdgeRow> = edges_to.iter().map(|to| EdgeRow::new(&id, to, EdgeKind::BuiltFrom)).collect();
let ledger = serde_json::json!({
"builder2": 1,
"build_id": self.build_id,
"repo": self.repo,
"variant": self.variant,
"package": self.package,
"features": self.features,
"profile": self.profile,
"triple": self.triple,
"toolchain": self.toolchain,
"builder": self.builder,
"built_at": self.built_at,
"recipe": self.recipe,
"commits": self.commits,
"crates": self.crates.len(),
"artifact": a,
"record": self.record,
});
out.push(BinaryDump {
artifact: self.owner(a),
nodes: vec![mimir_schema::nodes_batch(&[node]).context("artifact node batch")?],
edges: vec![mimir_schema::edges_batch(&edges).context("BUILT_FROM edge batch")?],
symbols: Vec::new(),
ledger: Some(ledger.to_string()),
ms: 0,
});
}
Ok(out)
}
}
/// What [`artifacts`] is asked: every filter given must hold.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactQuery {
/// A crate the artifact was built from: `name` or `name@version`.
#[serde(default)]
pub krate: Option<String>,
/// A commit it was built from: a sha (or a prefix of one, ≥ 7), or
/// `repo@sha`.
#[serde(default)]
pub commit: Option<String>,
/// The variant that built it.
#[serde(default)]
pub variant: Option<String>,
}
/// One built artifact as the log holds it now.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactHit {
/// The artifact node id.
pub id: String,
/// The owner of its keys ([`ProductFacts::owner`]).
pub owner: String,
pub file: String,
pub repo: String,
pub variant: String,
/// `<triple>:<profile>`.
pub target: String,
pub features: Vec<String>,
/// The provenance line.
pub provenance: String,
/// `crate@version` it was built from.
pub crates: Vec<String>,
/// `repo@sha` it was built from.
pub commits: Vec<String>,
}
/// **Which artifacts** match `q` — the open artifact nodes of
/// [`PRODUCTS_REPO`] and their BUILT_FROM edges, from ONE fold of each table.
pub async fn artifacts(log: &EventLog, q: &ArtifactQuery) -> Result<Vec<ArtifactHit>> {
let scope = FoldScope { repos: [(PRODUCTS_REPO.to_string(), i64::MAX)].into(), universe: false };
let nodes = log.fold_rows(Kind::BinNodes, &scope, &["id", "kind", "name", "crate", "version", "ty", "cfg", "doc"]).await?;
let edges = log.fold_rows(Kind::BinEdges, &scope, &["from", "to", "kind"]).await?;
let mut built_from: BTreeMap<String, (Vec<String>, Vec<String>)> = BTreeMap::new();
for &(b, r) in &edges.open {
let batch = &edges.batches[b as usize];
let (from, to, k) = (str_col(batch, "from")?, str_col(batch, "to")?, str_col(batch, "kind")?);
let r = r as usize;
if k.value(r) != EdgeKind::BuiltFrom.as_str() {
continue;
}
let Some((package, version, d)) = moniker::parse(to.value(r)) else { continue };
let slot = built_from.entry(from.value(r).to_string()).or_default();
if package == moniker::COMMIT_PACKAGE {
if let Some(c) = meta_value(d, "commit=") {
slot.1.push(c);
}
} else if d == moniker::CRATE_ROOT_DESCRIPTOR {
slot.0.push(format!("{package}@{version}"));
}
}
let mut out = Vec::new();
for &(b, r) in &nodes.open {
let batch = &nodes.batches[b as usize];
let r = r as usize;
if str_col(batch, "kind")?.value(r) != kind::ARTIFACT {
continue;
}
let opt = |c: &str| -> Result<String> {
let a = str_col(batch, c)?;
Ok(if a.is_null(r) { String::new() } else { a.value(r).to_string() })
};
let id = opt("id")?;
let (mut crates, mut commits) = built_from.remove(&id).unwrap_or_default();
crates.sort();
commits.sort();
let hit = ArtifactHit {
owner: opt(crate::schema::COL_KRATE)?,
file: opt("name")?,
repo: opt("crate")?,
variant: opt("version")?,
target: opt("ty")?,
features: opt("cfg")?.split(',').filter(|s| !s.is_empty()).map(str::to_string).collect(),
provenance: opt("doc")?,
crates,
commits,
id,
};
if matches(&hit, q) {
out.push(hit);
}
}
out.sort_by(|a, b| (&a.repo, &a.variant, &a.file, &a.owner).cmp(&(&b.repo, &b.variant, &b.file, &b.owner)));
Ok(out)
}
/// `name=value` out of a single `Meta` descriptor (`` `name=value`: ``).
fn meta_value(d: &str, prefix: &str) -> Option<String> {
let inner = d.strip_suffix(':')?.strip_prefix('`')?.strip_suffix('`')?.replace("``", "`");
inner.strip_prefix(prefix).map(str::to_string)
}
fn matches(h: &ArtifactHit, q: &ArtifactQuery) -> bool {
if let Some(v) = &q.variant {
if &h.variant != v {
return false;
}
}
if let Some(k) = &q.krate {
let hit = if k.contains('@') { h.crates.iter().any(|c| c == k) } else { h.crates.iter().any(|c| c.split('@').next() == Some(k.as_str())) };
if !hit {
return false;
}
}
if let Some(c) = &q.commit {
let hit = h.commits.iter().any(|rc| {
if c.contains('@') {
rc == c || rc.split_once('@').zip(c.split_once('@')).is_some_and(|((r1, s1), (r2, s2))| r1 == r2 && s2.len() >= 7 && s1.starts_with(s2))
} else {
c.len() >= 7 && rc.split_once('@').is_some_and(|(_, s)| s.starts_with(c.as_str()))
}
});
if !hit {
return false;
}
}
true
}