crates/mimir-ingest/src/artifact.rs
//! Artifacts that ALREADY EXIST for a consumed commit — the binary layer
//! without a build.
//!
//! [`crate::driver::BinaryLayer`] links its own artifact: `cargo mimir-dump
//! --build`, one nightly compile per commit. This is the other road: somebody
//! else linked the commit (the product build, CI, a person) and left cargo's
//! output in a directory named by the commit's sha —
//!
//! ```text
//! <dir>/<sha>/<profile>/<bin> e.g. …/9c1e22aa…/release/lgz
//! <dir>/<sha>/<profile>/deps/*.rlib optional: names the crate ids exactly
//! ```
//!
//! — which is what `cargo build --release --target-dir <dir>/<sha>` writes. The
//! sha in the path is the whole claim "this artifact is of that commit": nothing
//! is inferred from file times.
//!
//! The SOURCE half of the join is not a dump directory here: it is the log
//! itself ([`source_half`]) — the repo's open node rows at its offset plus the
//! universe ring (dependencies, and std once the toolchain's dump is in). An
//! artifact linked by another compiler than the driver's has other
//! StableCrateIds and other symbol hashes, so it joins by its v0 PATHS
//! (`mimir-binary` step 2); its crates are named by its DWARF, by the rlibs of
//! its `deps` dir, or by crate name.
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
pub use mimir_binary::crates::{MetaPackage, cargo_metadata};
pub use mimir_binary::source::SourceDump;
use mimir_schema::{Level, NodeRow, Origin, kind};
use skade::arrow_array::{Array, BooleanArray};
use crate::delta::Kind;
use crate::dump::BinaryDump;
use crate::log::{EventLog, FoldScope, str_col};
use crate::schema::{COL_REPO, UNIVERSE_REPO};
/// One linked artifact found on disk.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FoundArtifact {
/// The bin target's name (the executable's file name).
pub name: String,
/// cargo's profile (`release`, `dev`, a custom one) — from the directory
/// it was found under (`debug` is the `dev` profile's).
pub profile: String,
pub path: PathBuf,
/// `<profile dir>/deps`, when it is there.
pub deps_dir: Option<PathBuf>,
}
/// The local bin targets of a workspace, by CRATE name (`-` → `_`).
pub fn declared_bins(packages: &[MetaPackage]) -> BTreeSet<String> {
packages.iter().filter(|p| p.local).flat_map(|p| p.targets.iter().filter(|t| t.2 == "bin").map(|t| t.0.clone())).collect()
}
/// The executables under `dir/<profile>/` whose name is one of `bins` (crate
/// names, see [`declared_bins`]): an uplifted cargo bin. Hashed files (`deps/`),
/// test harnesses and anything that is not an ELF file are not artifacts of
/// the workspace. A `dir` that does not exist finds nothing.
pub fn find(dir: &Path, bins: &BTreeSet<String>) -> Vec<FoundArtifact> {
let mut out = Vec::new();
let Ok(profiles) = std::fs::read_dir(dir) else { return out };
for profile in profiles.flatten().filter(|e| e.path().is_dir()) {
let pdir = profile.path();
let pname = profile.file_name().to_string_lossy().into_owned();
let deps = pdir.join("deps");
let Ok(files) = std::fs::read_dir(&pdir) else { continue };
for f in files.flatten() {
let path = f.path();
let name = f.file_name().to_string_lossy().into_owned();
if !path.is_file() || !bins.contains(&name.replace('-', "_")) || !is_elf(&path) {
continue;
}
out.push(FoundArtifact {
name,
profile: if pname == "debug" { "dev".to_string() } else { pname.clone() },
path,
deps_dir: deps.is_dir().then(|| deps.clone()),
});
}
}
out.sort_by(|a, b| (&a.profile, &a.name).cmp(&(&b.profile, &b.name)));
out
}
fn is_elf(path: &Path) -> bool {
use std::io::Read;
let mut magic = [0u8; 4];
std::fs::File::open(path).and_then(|mut f| f.read_exact(&mut magic)).is_ok() && magic == [0x7f, b'E', b'L', b'F']
}
/// Node kinds the join never looks at (no symbol, no generic scope, never on
/// a parent chain).
const NOT_JOINED: [&str; 5] = [kind::PARAM, kind::FIELD, kind::VARIANT, kind::FEATURE, kind::CFG_GATED_OUT];
/// The source half of `repo` as the log holds it at ordinal `seq`: every
/// DEFINED node of the repo's own crates and of the universe ring. Read
/// through `log` as it is — pin it first for one committed state.
pub async fn source_half(log: &EventLog, repo: &str, seq: i64) -> Result<SourceDump> {
let spec = log.spec(Kind::Nodes);
let want = ["kind", "level", "name", "parent", "crate", "version", "origin", "defined_here"];
for c in want {
if spec.schema.field_with_name(c).is_err() {
bail!("the log's node schema has no `{c}` column: an artifact cannot be joined against it");
}
}
// the FFI border's link names (schema 4), when this log has the column
let has_link = spec.schema.field_with_name("link_name").is_ok();
let mut read: Vec<&str> = want.to_vec();
if has_link {
read.push("link_name");
}
let folded = log.fold_rows(Kind::Nodes, &FoldScope { repos: [(repo.to_string(), seq)].into(), universe: true }, &read).await?;
let mut cols = Vec::with_capacity(folded.batches.len());
let mut links = Vec::with_capacity(folded.batches.len());
for b in &folded.batches {
links.push(if has_link { Some(str_col(b, "link_name")?) } else { None });
let defined = b.column_by_name("defined_here").and_then(|c| c.as_any().downcast_ref::<BooleanArray>()).context("code_node_events.defined_here")?.clone();
cols.push((
str_col(b, COL_REPO)?,
str_col(b, "id")?,
str_col(b, "kind")?,
str_col(b, "level")?,
str_col(b, "name")?,
str_col(b, "parent")?,
str_col(b, "crate")?,
str_col(b, "version")?,
str_col(b, "origin")?,
defined,
));
}
let rows = folded.open.iter().filter_map(|&(b, r)| {
let (repo_c, id, k, level, name, parent, krate, version, origin, defined) = &cols[b as usize];
let r = r as usize;
if defined.is_null(r) || !defined.value(r) || NOT_JOINED.contains(&k.value(r)) {
return None;
}
let mut row = NodeRow::new(
id.value(r),
k.value(r),
Level::parse(level.value(r)).unwrap_or(Level::Item),
name.value(r),
krate.value(r),
version.value(r),
Origin::parse(origin.value(r)).unwrap_or(Origin::Library),
);
row.parent = (!parent.is_null(r)).then(|| parent.value(r).to_string());
row.repo = (repo_c.value(r) != UNIVERSE_REPO).then(|| repo_c.value(r).to_string());
row.link_name = links[b as usize].as_ref().filter(|l| !l.is_null(r)).map(|l| l.value(r).to_string());
Some(row)
});
Ok(SourceDump::from_rows(rows))
}
/// What every artifact of one commit is joined with.
#[derive(Debug, Clone)]
pub struct JoinContext {
/// `cargo metadata` of the workspace at the commit.
pub packages: Vec<MetaPackage>,
/// The sysroot of the toolchain that linked the artifacts, when known
/// (its rlibs name std's crate ids); without it the toolchain's crates
/// are recognised by name.
pub sysroot: Option<PathBuf>,
/// The artifacts' target triple.
pub triple: String,
}
/// Join one found artifact onto `src`: the binary stream's rows of it
/// (owner `name@triple:profile`).
pub fn join(src: &SourceDump, a: &FoundArtifact, cx: &JoinContext, sha: &str) -> Result<BinaryDump> {
let t0 = std::time::Instant::now();
let opts = mimir_binary::Options {
binary: a.path.clone(),
dump: PathBuf::new(),
extra_dumps: Vec::new(),
deps_dir: a.deps_dir.clone(),
manifest: None,
packages: Some(cx.packages.clone()),
sysroot: cx.sysroot.clone(),
// no driver symbol table in a log: every symbol goes by its v0 path
ignore_driver_symbols: true,
sha: Some(sha.to_string()),
};
let o = mimir_binary::run_with(&opts, src).map_err(|m| anyhow::anyhow!("mimir-binary {}: {m}", a.path.display()))?;
crate::driver::binary_dump(BinaryDump::artifact_id(&a.name, &cx.triple, &a.profile), &o, t0.elapsed().as_millis() as i64)
}
#[cfg(test)]
mod tests {
use super::*;
fn elf(path: &Path) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, [0x7f, b'E', b'L', b'F', 2, 1, 1, 0]).unwrap();
}
#[test]
fn only_the_declared_bins_of_a_profile_dir_are_artifacts() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().join("abc123");
elf(&dir.join("release/my-tool"));
elf(&dir.join("release/deps/my_tool-0123456789abcdef"));
elf(&dir.join("release/stray"));
std::fs::write(dir.join("release/my-tool.d"), "x").unwrap();
std::fs::write(dir.join("release/other"), "not elf").unwrap();
elf(&dir.join("debug/other"));
let bins: BTreeSet<String> = ["my_tool".to_string(), "other".to_string()].into();
let found = find(&dir, &bins);
let got: Vec<(&str, &str, bool)> = found.iter().map(|a| (a.profile.as_str(), a.name.as_str(), a.deps_dir.is_some())).collect();
assert_eq!(got, vec![("dev", "other", false), ("release", "my-tool", true)]);
assert!(find(&tmp.path().join("no-such-sha"), &bins).is_empty());
}
#[test]
fn declared_bins_are_the_local_bin_targets() {
let p = |name: &str, local: bool, targets: &[(&str, &str)]| MetaPackage {
name: name.into(),
version: "0.1.0".into(),
local,
targets: targets.iter().map(|(n, k)| (n.to_string(), PathBuf::from("/x"), k.to_string())).collect(),
};
let pk = [p("a", true, &[("a", "lib"), ("a_cli", "bin"), ("it", "test")]), p("dep", false, &[("depbin", "bin")])];
assert_eq!(declared_bins(&pk), ["a_cli".to_string()].into());
}
}