crates/mimir-ingest/src/boundary/mod.rs
//! **The system boundary** in the dependency universe: system call nodes,
//! the C functions and raw doors that make them, and the kernel's types
//! read from BTF. Written ONCE per toolchain / target (the table and each
//! dependency owner's map) and once per kernel (release + BTF hash) — never
//! per commit, never by compiling anything.
//!
//! ```text
//! @syscall@<arch>.r1:table crate @syscall · module <arch>/ (x86_64, aarch64) · one `syscall` node per call
//! (kernel table: number, entry point; ty = class) + `<indirect>`
//! @syscall@<owner>.<key>.r1:map one per dependency-universe owner and target (key = the libc on
//! x86_64, `<arch>-<libc>` elsewhere) with an answer: its crate
//! DEPENDS_ON the map crate → @syscall; SYSCALLS edges
//! ffi `open64` → openat (glibc), a fn handing the raw door a
//! number → that call (name; UNCERTAIN), the raw door → <indirect>
//! @kernel@<release>+<hash>:btf crate @kernel · struct/ union/ enum/ typedef/ func/ · USES_TYPE;
//! @syscall crate DEPENDS_ON it; CALLS syscall → its entry fn
//! ```
//!
//! Who reaches the boundary is then a plain walk of the code room with the
//! dependency universe folded in: CALLS to a function with SYSCALLS edges.
//! The DEPENDS_ON edges are what bring these owners into a room — the room
//! folds the owners its own crates depend on, and a crate that makes system
//! calls now depends on its map.
pub mod btf;
pub mod class;
pub mod libc_map;
mod aarch64;
mod x86_64;
#[cfg(test)]
mod tests;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::path::{Path, PathBuf};
use std::time::Instant;
use anyhow::{Context, Result};
use mimir_schema::{EdgeKind, EdgeRow, Level, NodeRow, Origin, kind as nk, moniker};
use skade::ScanFilter;
use skade::arrow_array::{Array, BooleanArray, RecordBatch, StringArray};
pub use class::{Class, class_of};
pub use libc_map::{Libc, Mapped};
use crate::consumer::sort_by;
use crate::delta::{self, Kind};
use crate::log::{CommitWrite, EDGE_SORT_COLUMN, EventLog, FoldScope, LogWriter, UniverseRow};
use crate::schema::{COL_KRATE, UNIVERSE_REPO};
/// The name of the ONE system call node per architecture that stands for a
/// call whose number is chosen at run time. `<` can be no kernel name.
pub const INDIRECT: &str = "<indirect>";
/// Revision of the syscall tables + classes (a new revision is a new owner).
pub const TABLE_REV: u32 = 1;
/// Revision of the C-library map and the raw-door rules.
pub const MAP_REV: u32 = 1;
/// One system call of one architecture's kernel table.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Syscall {
pub nr: u32,
pub name: &'static str,
/// The kernel's entry point as the table names it (`sys_read`,
/// `sys_newstat`); empty = no entry (removed / never implemented).
pub entry: &'static str,
/// From the UAPI header, past the kernel table this was generated from.
pub from_uapi: bool,
}
/// The architectures with a syscall table.
pub const ARCHES: &[&str] = &["x86_64", "aarch64"];
/// The kernel syscall table of `arch` (`x86_64`, `aarch64`).
pub fn table(arch: &str) -> Option<&'static [Syscall]> {
match arch {
"x86_64" => Some(x86_64::X86_64),
"aarch64" => Some(aarch64::AARCH64),
_ => None,
}
}
/// The calls of the generic table (aarch64) that stand in for a legacy
/// call it does not have: what a C library or `std` makes there instead
/// (`open` → `openat`, `fork` → `clone`, `stat` → `newfstatat` …). More
/// than one name = the C libraries differ (glibc `rename` → `renameat2`,
/// musl → `renameat`); every one is a possible call. A legacy call with no
/// stand-in (`iopl`, `modify_ldt`, the x86 thread-area calls) is absent.
pub const GENERIC_STAND_INS: &[(&str, &[&str])] = &[
("open", &["openat"]),
("creat", &["openat"]),
("stat", &["newfstatat"]),
("lstat", &["newfstatat"]),
("access", &["faccessat", "faccessat2"]),
("chmod", &["fchmodat"]),
("chown", &["fchownat"]),
("lchown", &["fchownat"]),
("dup2", &["dup3"]),
("epoll_create", &["epoll_create1"]),
("epoll_wait", &["epoll_pwait"]),
("eventfd", &["eventfd2"]),
("fork", &["clone"]),
("vfork", &["clone"]),
("futimesat", &["utimensat"]),
("getdents", &["getdents64"]),
("getpgrp", &["getpgid"]),
("inotify_init", &["inotify_init1"]),
("link", &["linkat"]),
("mkdir", &["mkdirat"]),
("mknod", &["mknodat"]),
("pause", &["ppoll", "rt_sigsuspend"]),
("pipe", &["pipe2"]),
("poll", &["ppoll"]),
("readlink", &["readlinkat"]),
("rename", &["renameat", "renameat2"]),
("rmdir", &["unlinkat"]),
("select", &["pselect6"]),
("signalfd", &["signalfd4"]),
("symlink", &["symlinkat"]),
("time", &["clock_gettime"]),
("unlink", &["unlinkat"]),
("utime", &["utimensat"]),
("utimes", &["utimensat"]),
("alarm", &["setitimer"]),
];
/// The calls `name` is on `arch`: itself when the table has it, else its
/// stand-ins on a generic-table architecture ([`GENERIC_STAND_INS`]), else
/// none. [`INDIRECT`] is itself on every architecture.
pub fn calls_on(arch: &str, name: &str) -> Vec<&'static str> {
if name == INDIRECT {
return vec![INDIRECT];
}
if let Some(s) = syscall(arch, name) {
return vec![s.name];
}
if arch == "aarch64"
&& let Some((_, v)) = GENERIC_STAND_INS.iter().find(|(n, _)| *n == name)
{
return v.iter().filter_map(|n| syscall(arch, n).map(|s| s.name)).collect();
}
Vec::new()
}
/// One system call of `arch` by name.
pub fn syscall(arch: &str, name: &str) -> Option<&'static Syscall> {
table(arch)?.iter().find(|s| s.name == name)
}
/// The `'static` spelling of a system call name (any table), or of [`INDIRECT`].
pub(crate) fn static_name(name: &str) -> Option<&'static str> {
if name == INDIRECT {
return Some(INDIRECT);
}
ARCHES.iter().filter_map(|a| table(a)).flat_map(|t| t.iter()).find(|s| s.name == name).map(|s| s.name)
}
/// The symbol prefix of `arch`'s syscall entry stubs in the kernel.
pub fn entry_prefix(arch: &str) -> Option<&'static str> {
match arch {
"x86_64" => Some("__x64_"),
"aarch64" => Some("__arm64_"),
_ => None,
}
}
/// A compilation target the boundary is written for.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Target {
pub triple: String,
pub arch: &'static str,
pub libc: Libc,
}
impl Target {
/// A Linux target with a known syscall table and C library.
pub fn parse(triple: &str) -> Option<Target> {
let arch = triple.split('-').next()?;
let arch: &'static str = match arch {
"x86_64" => "x86_64",
"aarch64" => "aarch64",
_ => return None,
};
Some(Target { triple: triple.to_string(), arch, libc: Libc::of_target(triple)? })
}
/// The key of this target's boundary maps: the libc (`gnu`, `musl`) on
/// x86_64, the first architecture (its owners keep their names), and
/// `<arch>-<libc>` on every other one, so two architectures' maps of the
/// same dependency are two owners.
pub fn map_key(&self) -> String {
match self.arch {
"x86_64" => self.libc.as_str().to_string(),
a => format!("{a}-{}", self.libc.as_str()),
}
}
/// The target this process was built for (the std dump's `--target` is the host's).
pub fn host() -> Option<Target> {
let env = if cfg!(target_env = "musl") { "musl" } else { "gnu" };
Target::parse(&format!("{}-unknown-linux-{env}", std::env::consts::ARCH))
}
}
/// Which kernel's BTF the boundary reads.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct KernelSource {
pub path: PathBuf,
pub release: String,
/// `box` (the running kernel: `/sys/kernel/btf/vmlinux`) or
/// `configured` (`$NORNIR_KERNEL_BTF`).
pub chosen: &'static str,
}
/// `/sys/kernel/btf/vmlinux` of the running kernel.
pub const BOX_BTF: &str = "/sys/kernel/btf/vmlinux";
impl KernelSource {
/// `$NORNIR_KERNEL_BTF` (a vmlinux BTF file, `none` = read no kernel; release from
/// `$NORNIR_KERNEL_RELEASE`, else the file's name) when set, else the
/// box's own kernel (its BTF and `uname -r`). `None` when neither is there.
pub fn configured_or_box() -> Option<KernelSource> {
if let Some(p) = std::env::var_os("NORNIR_KERNEL_BTF").filter(|p| !p.is_empty()) {
if p == "none" {
return None;
}
let path = PathBuf::from(p);
let release = std::env::var("NORNIR_KERNEL_RELEASE")
.ok()
.filter(|r| !r.trim().is_empty())
.unwrap_or_else(|| path.file_name().map_or("file".into(), |n| n.to_string_lossy().into_owned()));
return Some(KernelSource { path, release: release.trim().to_string(), chosen: "configured" });
}
let path = PathBuf::from(BOX_BTF);
if !path.is_file() {
return None;
}
let release = std::fs::read_to_string("/proc/sys/kernel/osrelease").ok()?.trim().to_string();
Some(KernelSource { path, release, chosen: "box" })
}
}
/// The owner of `arch`'s syscall table in the universe.
pub fn table_owner(arch: &str) -> String {
format!("@syscall@{arch}.r{TABLE_REV}:table")
}
/// The owner of the boundary map of dependency owner `owner`.
pub fn map_owner(owner: &str, target: &Target) -> String {
let clean: String = owner.chars().map(|c| if c.is_ascii_alphanumeric() || c == '.' || c == '_' { c } else { '-' }).collect();
format!("@syscall@{clean}.{}.r{MAP_REV}:map", target.map_key())
}
/// The owner of one kernel's BTF read.
pub fn kernel_owner(release: &str, hash: &str) -> String {
format!("@kernel@{release}+{}:btf", &hash[..hash.len().min(12)])
}
/// Is `owner` one of the boundary's own (never a dependency to map)?
pub fn is_boundary_owner(owner: &str) -> bool {
owner.starts_with('@')
}
// ------------------------------------------------------------------ rows
/// The rows of one owner, ready for the log.
#[derive(Debug, Default)]
pub struct OwnerRows {
pub owner: String,
pub nodes: Vec<NodeRow>,
pub edges: Vec<EdgeRow>,
}
fn boundary_node(id: String, kind: &str, level: Level, name: &str, krate: &str, version: &str) -> NodeRow {
let mut n = NodeRow::new(id, kind, level, name, krate, version, Origin::Library);
n.repo = Some(UNIVERSE_REPO.to_string());
n
}
/// The table owner's rows: the `@syscall` crate, `arch`'s module, one node
/// per system call (+ [`INDIRECT`]).
pub fn table_rows(arch: &str) -> Option<OwnerRows> {
let t = table(arch)?;
let (krate, ver) = (moniker::SYSCALL_PACKAGE, moniker::TABLE_VERSION);
let (root, module) = (moniker::syscall_crate(), moniker::syscall_arch(arch));
let mut nodes = vec![boundary_node(root.clone(), nk::CRATE, Level::Crate, krate, krate, ver)];
let mut m = boundary_node(module.clone(), nk::MOD, Level::Module, arch, krate, ver);
m.parent = Some(root.clone());
m.doc = Some(format!("the {arch} system calls ({} in the table)", t.len()));
nodes.push(m);
let mut edges = vec![EdgeRow::new(&root, &module, EdgeKind::Contains)];
let mut push = |name: &str, doc: String| {
let id = moniker::syscall(arch, name);
let mut n = boundary_node(id.clone(), nk::SYSCALL, Level::Item, name, krate, ver);
n.parent = Some(module.clone());
n.ty = Some(class_of(name).as_str().to_string());
n.doc = Some(doc);
n.has_doc = Some(true);
nodes.push(n);
edges.push(EdgeRow::new(&module, &id, EdgeKind::Contains));
};
for s in t {
let entry = if s.entry.is_empty() { "no entry point".to_string() } else { format!("→ {}", s.entry) };
push(s.name, format!("{arch} #{} {entry}{}", s.nr, if s.from_uapi { " (UAPI)" } else { "" }));
}
push(INDIRECT, format!("{arch}: a system call whose number is chosen at run time (libc `syscall`, a raw `syscall` instruction)"));
Some(OwnerRows { owner: table_owner(arch), nodes, edges })
}
/// The kernel owner's rows: every named type / function of the BTF, their
/// type edges, `@syscall` DEPENDS_ON the kernel, and CALLS from each system
/// call of the BTF's architecture to the kernel function it enters.
pub struct KernelRead {
pub rows: OwnerRows,
pub arch: Option<&'static str>,
pub types: usize,
/// Syscalls linked to an entry function / with an entry the BTF lacks.
pub entries_linked: usize,
pub entries_missing: Vec<&'static str>,
}
pub fn kernel_rows(bytes: &[u8], release: &str, owner: &str) -> Result<KernelRead> {
let b = btf::Btf::parse(bytes)?;
let names = btf::Names::of(&b);
let krate = moniker::KERNEL_PACKAGE;
let root = moniker::kernel_crate(release);
let mut nodes = vec![boundary_node(root.clone(), nk::CRATE, Level::Crate, krate, krate, release)];
let mut edges: Vec<EdgeRow> = Vec::new();
nodes[0].doc = Some(format!("Linux {release}, read from BTF ({} types)", b.types.len() - 1));
for ns in ["struct", "union", "enum", "typedef", "func"] {
let id = moniker::kernel_ns(release, ns);
let mut n = boundary_node(id.clone(), nk::MOD, Level::Module, ns, krate, release);
n.parent = Some(root.clone());
nodes.push(n);
edges.push(EdgeRow::new(&root, &id, EdgeKind::Contains));
}
let id_of = |tid: u32| -> Option<String> { names.node_of.get(&tid).map(|(ns, name, dup)| moniker::kernel_item(release, ns, name, *dup)) };
let mut funcs: HashMap<String, String> = HashMap::new();
let mut written: BTreeSet<String> = BTreeSet::new();
let mut ids: Vec<u32> = names.node_of.keys().copied().collect();
ids.sort_unstable();
for tid in ids {
let (ns, name, _) = &names.node_of[&tid];
let id = id_of(tid).expect("named");
let t = b.types[tid as usize];
if t.kind == btf::k::FWD {
// a forward declaration a definition answers is that definition
let defined = names.node_of.iter().any(|(o, (ons, oname, odup))| *o != tid && ons == ns && oname == name && odup.is_none() && b.types[*o as usize].kind != btf::k::FWD);
if defined || !written.insert(id.clone()) {
continue;
}
} else if !written.insert(id.clone()) {
continue;
}
let kind = match t.kind {
btf::k::STRUCT | btf::k::FWD if *ns == "struct" => nk::STRUCT,
btf::k::UNION | btf::k::FWD => nk::UNION,
btf::k::ENUM | btf::k::ENUM64 => nk::ENUM,
btf::k::TYPEDEF => nk::TYPE,
btf::k::FUNC => nk::FN,
_ => nk::STRUCT,
};
let mut n = boundary_node(id.clone(), kind, Level::Item, name, krate, release);
n.parent = Some(moniker::kernel_ns(release, ns));
n.visibility = Some(mimir_schema::visibility::PUB.into());
n.ty = Some(match t.kind {
btf::k::STRUCT | btf::k::UNION => format!("{} bytes, {} members", t.size_or_type, t.vlen),
btf::k::ENUM | btf::k::ENUM64 => format!("{} values", t.vlen),
btf::k::TYPEDEF => b.c_type(t.size_or_type),
btf::k::FUNC => b.func_signature(tid),
_ => "forward declaration (no definition in this BTF)".into(),
});
if t.kind == btf::k::FUNC {
n.link_name = Some(name.clone());
n.doc = Some(match t.vlen {
0 => "static".into(),
1 => "global".into(),
_ => "extern".into(),
});
funcs.insert(name.clone(), id.clone());
}
nodes.push(n);
edges.push(EdgeRow::new(moniker::kernel_ns(release, ns), &id, EdgeKind::Contains));
// what it uses, by name
let mut uses = BTreeSet::new();
let mut seen = BTreeSet::new();
match t.kind {
btf::k::STRUCT | btf::k::UNION => {
for &(_, ty) in b.refs_of(tid) {
btf::named_uses(&b, &names, ty, &mut uses, &mut seen);
}
}
btf::k::TYPEDEF => {
// a typedef of an anonymous aggregate names what it holds
let target = t.size_or_type;
if names.node_of.contains_key(&target) {
uses.insert(target);
} else if matches!(b.types.get(target as usize).map(|x| x.kind), Some(btf::k::STRUCT | btf::k::UNION)) {
for &(_, ty) in b.refs_of(target) {
btf::named_uses(&b, &names, ty, &mut uses, &mut seen);
}
} else {
btf::named_uses(&b, &names, target, &mut uses, &mut seen);
}
}
btf::k::FUNC => btf::named_uses(&b, &names, t.size_or_type, &mut uses, &mut seen),
_ => {}
}
uses.remove(&tid);
let mut targets: BTreeSet<String> = uses.into_iter().filter_map(id_of).collect();
targets.remove(&id);
for to in targets {
edges.push(EdgeRow::new(&id, to, EdgeKind::UsesType));
}
}
// which architecture: whose entry stubs the kernel has
let arch = ["x86_64", "aarch64"].into_iter().find(|a| entry_prefix(a).is_some_and(|p| funcs.contains_key(&format!("{p}sys_read"))));
let (mut linked, mut missing) = (0usize, Vec::new());
if let Some(arch) = arch {
edges.push(EdgeRow::new(moniker::syscall_crate(), &root, EdgeKind::DependsOn));
let prefix = entry_prefix(arch).expect("known arch");
for s in table(arch).unwrap_or(&[]) {
if s.entry.is_empty() {
continue;
}
// SYSCALL_DEFINE0 calls keep only `__do_sys_*` in BTF
let hit = funcs.get(&format!("{prefix}{}", s.entry)).or_else(|| funcs.get(&format!("__do_{}", s.entry)));
match hit {
Some(f) => {
linked += 1;
edges.push(EdgeRow::new(moniker::syscall(arch, s.name), f, EdgeKind::Calls));
}
None => missing.push(s.name),
}
}
}
Ok(KernelRead { rows: OwnerRows { owner: owner.to_string(), nodes, edges }, arch, types: b.types.len() - 1, entries_linked: linked, entries_missing: missing })
}
// ------------------------------------------------------------------ the map
/// One function of a dependency owner, as the map reads it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OwnerFn {
pub id: String,
pub kind: String,
pub name: String,
pub link_name: Option<String>,
}
/// What the map found for the owners it read.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MapStats {
pub owners_read: usize,
pub owners_with_edges: usize,
pub edges: usize,
pub uncertain: usize,
pub ffi_mapped: usize,
pub ffi_pure: usize,
/// C symbols the map has no answer for, with how many `ffi` nodes link them.
pub unmapped: BTreeMap<String, usize>,
/// Owners with edges but no crate node (their map cannot be folded in).
pub without_crate: usize,
}
fn is_raw_door_rust(owner_crate: &str, name: &str) -> bool {
// rustix's linux_raw backend: `syscall0` … `syscall6`, `_readonly`, `_noreturn`
if owner_crate != "rustix" {
return false;
}
let Some(rest) = name.strip_prefix("syscall") else { return false };
let digits = rest.trim_end_matches("_readonly").trim_end_matches("_noreturn");
digits.len() == 1 && digits.as_bytes()[0].is_ascii_digit()
}
/// Is a CALLS target a raw door: libc's `syscall` or a rustix raw stub?
pub fn is_raw_door_target(to: &str) -> bool {
let Some((package, _, d)) = moniker::parse(to) else { return false };
let last = d.trim_end_matches("().");
let leaf = last.rsplit(['/', '#', ']']).next().unwrap_or(last);
(package == "libc" && leaf == "syscall" && d.ends_with("syscall().")) || is_raw_door_rust(package, leaf)
}
/// The SYSCALLS edges of ONE dependency owner: its `ffi` nodes through the
/// libc map, its raw doors to [`INDIRECT`], and its fns named like a system
/// call that hand the raw door a number (`calls_raw` = those fns' ids).
pub fn map_edges(owner: &str, fns: &[OwnerFn], calls_raw: &BTreeSet<String>, target: &Target, stats: &mut MapStats) -> Vec<EdgeRow> {
let arch = target.arch;
let is_syscall = |n: &str| !calls_on(arch, n).is_empty();
let owner_crate = owner.split('@').next().unwrap_or(owner);
let mut out: BTreeMap<(String, &'static str), bool> = BTreeMap::new();
let mut add = |from: &str, call: &'static str, sure: bool| {
let e = out.entry((from.to_string(), call)).or_insert(sure);
*e = *e || sure;
};
for f in fns {
if f.kind == nk::FFI {
let symbol = f.link_name.as_deref().unwrap_or(&f.name);
match libc_map::lookup(symbol, target.libc, &is_syscall) {
Some(Mapped::Calls(calls)) => {
stats.ffi_mapped += 1;
for (c, sure) in calls {
for c in calls_on(arch, c) {
add(&f.id, c, sure);
}
}
}
Some(Mapped::Pure) => stats.ffi_pure += 1,
None => *stats.unmapped.entry(symbol.to_string()).or_default() += 1,
}
} else if f.kind == nk::FN || f.kind == nk::METHOD {
if is_raw_door_rust(owner_crate, &f.name) {
add(&f.id, INDIRECT, false);
} else if calls_raw.contains(&f.id) {
// std's `syscall!` fns, rustix's backend: the name IS the call
for n in calls_on(arch, &f.name) {
add(&f.id, n, false);
}
}
}
}
out.into_iter()
.map(|((from, call), sure)| {
let mut e = EdgeRow::new(from, moniker::syscall(arch, call), EdgeKind::Syscalls);
e.partial = Some(!sure);
e
})
.collect()
}
// ------------------------------------------------------------------ the writer
/// What [`ensure`] did.
#[derive(Debug, Clone, Default)]
pub struct Outcome {
pub target: String,
/// The table was written now (its node count), else it was there.
pub table_written: Option<usize>,
pub kernel: Option<KernelOutcome>,
/// Why the kernel was not read (no BTF, a parse error) — said, not fatal.
pub kernel_gap: Option<String>,
pub map: MapStats,
/// Map owners written now (with edges or empty).
pub maps_written: usize,
pub ms: u128,
}
#[derive(Debug, Clone, Default)]
pub struct KernelOutcome {
pub owner: String,
pub release: String,
pub hash: String,
pub source: String,
pub chosen: &'static str,
pub written: bool,
pub types: usize,
pub nodes: usize,
pub edges: usize,
pub arch: Option<&'static str>,
pub entries_linked: usize,
pub entries_missing: usize,
pub read_ms: u128,
}
impl Outcome {
/// One line for the consumer's log.
pub fn line(&self) -> String {
let table = match self.table_written {
Some(n) => format!("table written ({n} nodes)"),
None => "table there".into(),
};
let kernel = match (&self.kernel, &self.kernel_gap) {
(Some(k), _) if k.written => format!("kernel {} read ({}, {} types → {} nodes, {} edges, {} ms; {} syscalls linked to their entry)", k.release, k.chosen, k.types, k.nodes, k.edges, k.read_ms, k.entries_linked),
(Some(k), _) => format!("kernel {} there", k.release),
(None, Some(g)) => format!("no kernel: {g}"),
(None, None) => "no kernel asked".into(),
};
format!(
"boundary {}: {table}; {kernel}; {} owner map(s) written ({} with edges, {} SYSCALLS edges, {} uncertain, {} C symbol(s) unmapped) in {} ms",
self.target,
self.maps_written,
self.map.owners_with_edges,
self.map.edges,
self.map.uncertain,
self.map.unmapped.len(),
self.ms
)
}
}
fn str_col<'a>(b: &'a RecordBatch, name: &str) -> Option<&'a StringArray> {
b.column_by_name(name)?.as_any().downcast_ref::<StringArray>()
}
fn opt(a: Option<&StringArray>, r: usize) -> Option<String> {
a.filter(|a| !a.is_null(r)).map(|a| a.value(r).to_string())
}
/// The defined functions (`fn`, `method`, `ffi`) and crate nodes of the
/// dependency owners `owners`, folded.
async fn owner_fns(log: &EventLog, owners: &[String]) -> Result<(BTreeMap<String, Vec<OwnerFn>>, BTreeMap<String, String>)> {
let mut fns: BTreeMap<String, Vec<OwnerFn>> = BTreeMap::new();
let mut crates: BTreeMap<String, String> = BTreeMap::new();
let has_link = log.spec(Kind::Nodes).schema.field_with_name("link_name").is_ok();
let mut cols = vec!["id", "kind", "name", "defined_here"];
if has_link {
cols.push("link_name");
}
for chunk in owners.chunks(4096) {
let mut rows = log.scan_rows_where(Kind::Nodes, &ScanFilter::is_in(COL_KRATE, chunk.iter().map(String::as_str)), &cols).await?;
log.fold_scanned(Kind::Nodes, &FoldScope { repos: BTreeMap::new(), universe: true }, &mut rows)?;
for &(bi, r) in &rows.open {
let (b, r) = (&rows.batches[bi as usize], r as usize);
let (Some(id), Some(kind), Some(owner)) = (opt(str_col(b, "id"), r), opt(str_col(b, "kind"), r), opt(str_col(b, COL_KRATE), r)) else { continue };
let defined = b.column_by_name("defined_here").and_then(|c| c.as_any().downcast_ref::<BooleanArray>()).is_some_and(|c| !c.is_null(r) && c.value(r));
if !defined {
continue;
}
if kind == nk::CRATE {
crates.entry(owner).or_insert(id);
continue;
}
if kind != nk::FFI && kind != nk::FN && kind != nk::METHOD {
continue;
}
let name = opt(str_col(b, "name"), r).unwrap_or_default();
fns.entry(owner).or_default().push(OwnerFn { id, kind, name, link_name: opt(str_col(b, "link_name"), r) });
}
}
Ok((fns, crates))
}
/// Of `ids`, those with a CALLS edge to a raw door.
async fn calling_raw(log: &EventLog, ids: &[&str]) -> Result<BTreeSet<String>> {
let mut out = BTreeSet::new();
for chunk in ids.chunks(4096) {
let mut rows = log.scan_rows_where(Kind::Edges, &ScanFilter::is_in("from", chunk.iter().copied()), &["from", "to", "kind"]).await?;
log.fold_scanned(Kind::Edges, &FoldScope { repos: BTreeMap::new(), universe: true }, &mut rows)?;
for &(bi, r) in &rows.open {
let (b, r) = (&rows.batches[bi as usize], r as usize);
let (Some(from), Some(to), Some(kind)) = (opt(str_col(b, "from"), r), opt(str_col(b, "to"), r), opt(str_col(b, "kind"), r)) else { continue };
if kind == EdgeKind::Calls.as_str() && is_raw_door_target(&to) {
out.insert(from);
}
}
}
Ok(out)
}
/// **The boundary, once.** Writes what is missing for `target` — the
/// syscall table, a map per dependency owner not mapped yet, the kernel of
/// `kernel` (when given and not read yet) — in ONE release through `writer`.
/// Nothing missing = no write (a manifest read and a set difference).
pub async fn ensure(writer: &LogWriter, target: &Target, toolchain: &str, kernel: Option<&KernelSource>) -> Result<Outcome> {
let t0 = Instant::now();
let log = writer.log();
let mut out = Outcome { target: target.triple.clone(), ..Default::default() };
let have: BTreeSet<String> = log.universe_rows().await?.into_iter().map(|r| r.0).collect();
let mut owners: Vec<OwnerRows> = Vec::new();
let mut manifest: Vec<UniverseRow> = Vec::new();
let row = |o: &OwnerRows, tc: &str, release: Option<String>| UniverseRow { krate: o.owner.clone(), toolchain: tc.to_string(), rows: (o.nodes.len() + o.edges.len()) as i64, rustc_release: release };
// 1. the table
let towner = table_owner(target.arch);
if !have.contains(&towner) {
let t = table_rows(target.arch).context("no syscall table for this architecture")?;
out.table_written = Some(t.nodes.len());
manifest.push(row(&t, toolchain, None));
owners.push(t);
}
// 2. the kernel
if let Some(src) = kernel {
let t = Instant::now();
match std::fs::read(&src.path) {
Err(e) => out.kernel_gap = Some(format!("{} ({}): {e}", src.path.display(), src.chosen)),
Ok(bytes) => {
let hash = blake3::hash(&bytes).to_hex().to_string();
let owner = kernel_owner(&src.release, &hash);
let mut k = KernelOutcome { owner: owner.clone(), release: src.release.clone(), hash: hash.clone(), source: src.path.display().to_string(), chosen: src.chosen, ..Default::default() };
if !have.contains(&owner) {
match kernel_rows(&bytes, &src.release, &owner) {
Ok(kr) => {
(k.written, k.types, k.nodes, k.edges, k.arch) = (true, kr.types, kr.rows.nodes.len(), kr.rows.edges.len(), kr.arch);
(k.entries_linked, k.entries_missing) = (kr.entries_linked, kr.entries_missing.len());
manifest.push(row(&kr.rows, &format!("btf:{}", &hash[..16]), Some(src.release.clone())));
owners.push(kr.rows);
}
Err(e) => out.kernel_gap = Some(format!("{} ({}): {e:#}", src.path.display(), src.chosen)),
}
}
k.read_ms = t.elapsed().as_millis();
if out.kernel_gap.is_none() {
out.kernel = Some(k);
}
}
}
}
// 3. a map per dependency owner not mapped yet
let pending: Vec<String> = have.iter().filter(|o| !is_boundary_owner(o) && !have.contains(&map_owner(o, target))).cloned().collect();
if !pending.is_empty() {
let (fns, crates) = owner_fns(log, &pending).await?;
let candidates: Vec<&str> = fns.values().flatten().filter(|f| f.kind == nk::FN && !calls_on(target.arch, &f.name).is_empty()).map(|f| f.id.as_str()).collect();
let raw = if candidates.is_empty() { BTreeSet::new() } else { calling_raw(log, &candidates).await? };
out.map.owners_read = pending.len();
for owner in &pending {
let mowner = map_owner(owner, target);
let edges = map_edges(owner, fns.get(owner).map_or(&[][..], Vec::as_slice), &raw, target, &mut out.map);
let mut rows = OwnerRows { owner: mowner.clone(), ..Default::default() };
if !edges.is_empty() {
out.map.owners_with_edges += 1;
out.map.edges += edges.len();
out.map.uncertain += edges.iter().filter(|e| e.partial == Some(true)).count();
let mcrate = moniker::syscall_map_crate(owner, &target.map_key());
rows.nodes.push(boundary_node(mcrate.clone(), nk::CRATE, Level::Crate, moniker::SYSCALL_PACKAGE, moniker::SYSCALL_PACKAGE, moniker::TABLE_VERSION));
rows.nodes[0].doc = Some(format!("the system calls of {owner} ({})", target.libc.as_str()));
match crates.get(owner) {
Some(c) => rows.edges.push(EdgeRow::new(c, &mcrate, EdgeKind::DependsOn)),
None => out.map.without_crate += 1,
}
rows.edges.push(EdgeRow::new(&mcrate, moniker::syscall_crate(), EdgeKind::DependsOn));
rows.edges.extend(edges);
}
manifest.push(row(&rows, toolchain, None));
owners.push(rows);
out.maps_written += 1;
}
}
if !manifest.is_empty() {
commit(writer, toolchain, owners, manifest).await?;
}
out.ms = t0.elapsed().as_millis();
Ok(out)
}
/// ONE atomic release of the boundary owners' rows + their manifest rows.
async fn commit(writer: &LogWriter, toolchain: &str, owners: Vec<OwnerRows>, universe: Vec<UniverseRow>) -> Result<()> {
let log = writer.log();
let sha = format!("boundary:{toolchain}");
let mut per: Vec<(String, Vec<RecordBatch>, Vec<RecordBatch>)> = Vec::new();
for o in &owners {
let nodes = if o.nodes.is_empty() { vec![] } else { vec![mimir_schema::nodes_batch(&o.nodes)?] };
let edges = if o.edges.is_empty() { vec![] } else { vec![mimir_schema::edges_batch(&o.edges)?] };
per.push((o.owner.clone(), nodes, edges));
}
let mut events = Vec::new();
for kind in [Kind::Nodes, Kind::Edges] {
let spec = log.spec(kind);
let keyed = delta::key_rows(spec, per.iter().map(|(o, n, e)| (o.as_str(), if kind == Kind::Nodes { n.as_slice() } else { e.as_slice() })))?;
let d = delta::universe_delta(keyed);
if let Some(b) = delta::event_batch(spec, UNIVERSE_REPO, 0, None, &sha, &d)? {
let b = if kind == Kind::Edges { sort_by(&b, EDGE_SORT_COLUMN)? } else { b };
events.push((kind, vec![b]));
}
}
let (events, ring_events) = if log.ring_split() { (vec![], events) } else { (events, vec![]) };
writer.commit(CommitWrite { repo: UNIVERSE_REPO.into(), sha, offset: None, run: None, bin_runs: vec![], events, ring_events, universe }).await
}
/// Read a BTF file's rows without a log (tools, tests): what [`ensure`]
/// would write for this kernel.
pub fn read_kernel_file(path: &Path, release: &str) -> Result<KernelRead> {
let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?;
let hash = blake3::hash(&bytes).to_hex().to_string();
kernel_rows(&bytes, release, &kernel_owner(release, &hash))
}