crates/mimir-room/tests/build_peak.rs
//! **A room build never holds two tables' history at once.**
//!
//! The build reads every event of a table (each `+` and `-` of the history,
//! and ring on, the whole ring), folds it to the open rows, and builds the
//! index over those. It read the three big tables first and folded them side
//! by side: three scans and three folds at once — the 8.7 GB peak of dwarf's
//! server (2026-10-16), 5.6 GB for the ring view alone. Now a table is cut to
//! its open rows before the next is read.
//!
//! Counted at the allocator (its own test binary: nothing else allocates
//! meanwhile), against what the node and the edge scans of the history cost
//! together (`scans`): three tables scanned and folded side by side peaked at
//! 2.3 × that; one at a time, cut to its open rows before the next, 1.4 ×.
//! The bound is 1.8 ×.
#[allow(dead_code)]
#[path = "room/common.rs"]
mod common;
use std::alloc::{GlobalAlloc, Layout, System};
use std::sync::atomic::{AtomicIsize, Ordering::Relaxed};
use common::*;
use mimir_ingest::{FoldScope, Kind};
use mimir_room::{Room, Scope};
use mimir_schema::{EdgeKind, Origin};
use skade::arrow_array::Array;
static NOW: AtomicIsize = AtomicIsize::new(0);
static PEAK: AtomicIsize = AtomicIsize::new(0);
struct Counting;
fn add(n: isize) {
let now = NOW.fetch_add(n, Relaxed) + n;
PEAK.fetch_max(now, Relaxed);
}
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc(l) };
if !p.is_null() {
add(l.size() as isize);
}
p
}
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc_zeroed(l) };
if !p.is_null() {
add(l.size() as isize);
}
p
}
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
unsafe { System.dealloc(p, l) };
add(-(l.size() as isize));
}
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
let q = unsafe { System.realloc(p, l, new) };
if !q.is_null() {
add(new as isize - l.size() as isize);
}
q
}
}
#[global_allocator]
static COUNTING: Counting = Counting;
/// A moniker the size of a real one (dwarf's are ~95 bytes): `krate`'s
/// item `i`.
fn item(krate: &str, i: usize) -> String {
format!("rust-analyzer cargo {krate} 0.1.0 module_{}/impl#[Type{}][Trait{}]item_{i}().", i % 40, i % 97, i % 13)
}
const COMMITS: usize = 8;
const ROWS: usize = 3000;
/// Every commit changes every row: a node's payload flips, an edge's kind
/// flips (its identity: the old closes, the new opens). The history is
/// COMMITS times the open rows.
fn churn() -> Vec<Step> {
(0..COMMITS)
.map(|c| Step {
owner: "k@0.1.0:lib",
nodes: (0..ROWS).map(|i| node(&item("k", i), "fn", "k", Origin::Local, c % 2 == 0)).collect(),
edges: (0..ROWS)
.map(|i| {
let kind = if c % 2 == 0 { EdgeKind::Calls } else { EdgeKind::UsesType };
edge(&item("k", i), &item("k", (i + 1) % ROWS), kind)
})
.collect(),
binary: None,
ring: vec![],
})
.collect()
}
/// One build measured at a time: the counter is the process's.
static ONE: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
#[tokio::test(flavor = "current_thread")]
async fn a_room_build_never_holds_two_tables_history_at_once() {
let _one = ONE.lock().await;
let tmp = tempfile::tempdir().unwrap();
let log = consume(open_log(tmp.path()).await, tmp.path(), ROOM, churn()).await;
// what the scans of the history cost: nodes, edges (every event, the
// log's own columns — the build reads these and more)
let fs = FoldScope { repos: [(ROOM.to_string(), COMMITS as i64)].into(), universe: false };
let mut scans = 0usize;
for kind in [Kind::Nodes, Kind::Edges] {
let rows = log.scan_rows(kind, &fs, &[]).await.unwrap();
scans += rows.batches.iter().map(|b| b.columns().iter().map(|c| c.get_array_memory_size()).sum::<usize>()).sum::<usize>();
}
Room::clear_cache();
let before = NOW.load(Relaxed);
PEAK.store(before, Relaxed);
let room = Room::build_uncached(&log, &Scope::current()).await.unwrap();
let peak = (PEAK.load(Relaxed) - before) as usize;
assert!(room.len() >= ROWS, "the room holds the open nodes");
eprintln!("room build: peak {peak} B over what it started with; the node + edge scans of the history: {scans} B ({:.2} ×)", peak as f64 / scans as f64);
let ratio = peak as f64 / scans as f64;
assert!(ratio < 1.8, "the build peaked at {peak} B, {ratio:.2} × the node and edge scans of the history ({scans} B): the tables' history held side by side again");
}
const BIG_CRATES: usize = 20;
const BIG_ROWS: usize = 2000;
/// **The ring view reads only the ring rows of the crates its scope depends
/// on.** The ring holds every dumped dependency of every repo; a scope's
/// ring view is the DEPENDS_ON closure of its local crates. dwarf
/// (2026-10-16): the whole ring read and trimmed after — 15.7 M nodes,
/// 37.7 M edges, a 22 GB peak — for 6.3 M nodes kept. Here the ring holds
/// `dep` (which `k` depends on) and twenty big crates nothing depends on: a
/// ring-on build must peak under what reading the whole ring costs (it was
/// 2.11 × that, read whole and trimmed after; 0.60 × trimmed at read time —
/// the scan's own decode of a data file is the rest).
#[tokio::test(flavor = "current_thread")]
async fn a_ring_view_reads_only_the_ring_its_scope_depends_on() {
let _one = ONE.lock().await;
let tmp = tempfile::tempdir().unwrap();
let big: Vec<(String, Vec<mimir_schema::NodeRow>, Vec<mimir_schema::EdgeRow>)> = (0..BIG_CRATES)
.map(|c| {
let krate = format!("big{c}");
let mut nodes = vec![node(&krate, "crate", &krate, Origin::Library, true)];
nodes.extend((0..BIG_ROWS).map(|i| node(&item(&krate, i), "fn", &krate, Origin::Library, true)));
let edges = (0..BIG_ROWS)
.map(|i| edge(&item(&krate, i), &item(&krate, (i + 1) % BIG_ROWS), EdgeKind::Calls))
.collect();
(krate, nodes, edges)
})
.collect();
let owners: Vec<&'static str> = big.iter().map(|(k, _, _)| &*Box::leak(format!("{k}@1.0.0:lib").into_boxed_str())).collect();
let step = Step {
owner: "k@0.1.0:lib",
nodes: vec![node("k", "crate", "k", Origin::Local, true), node("k a()", "fn", "k", Origin::Local, true)],
edges: vec![edge("k", "dep", EdgeKind::DependsOn), edge("k a()", "dep h", EdgeKind::Calls)],
binary: None,
ring: owners.iter().zip(big).map(|(o, (_, n, e))| (*o, n, e)).collect(),
};
let log = consume(open_log(tmp.path()).await, tmp.path(), ROOM, vec![step]).await;
// what reading the whole ring costs: its node and edge rows
let ring = FoldScope { repos: Default::default(), universe: true };
let mut whole = 0usize;
for kind in [Kind::Nodes, Kind::Edges] {
let rows = log.scan_rows(kind, &ring, &[]).await.unwrap();
whole += rows.batches.iter().map(|b| b.columns().iter().map(|c| c.get_array_memory_size()).sum::<usize>()).sum::<usize>();
}
Room::clear_cache();
let before = NOW.load(Relaxed);
PEAK.store(before, Relaxed);
let room = Room::build_uncached(&log, &Scope::current().with_universe(true)).await.unwrap();
let peak = (PEAK.load(Relaxed) - before) as usize;
assert!(room.ordinal("dep h").is_some(), "the ring the scope depends on is in the view");
assert!(room.ordinal("big0").is_none() && room.ordinal(&item("big19", 0)).is_none(), "a ring crate nothing depends on is not");
let ratio = peak as f64 / whole as f64;
eprintln!("ring view build: peak {peak} B; reading the whole ring: {whole} B ({ratio:.2} ×)");
assert!(ratio < 1.0, "the ring view peaked at {peak} B, {ratio:.2} × reading the whole ring ({whole} B): the ring is read whole and trimmed after");
}
const OWNERS: usize = 6;
/// **A ring crate dumped under several owners is held once.** dwarf's ring
/// carries one crate@version under several owners (lib, test harness, each
/// feature set a repo builds it with) dumping the same items: 14.6 M ring
/// node rows for 6.3 M monikers. Here `dep` is dumped by six owners with the
/// same 2000 items and edges: the ring view must hold the nodes once while it
/// reads the edges, and still show every item and edge. Against the event
/// columns of the whole ring (what `scan_rows` reads bare): every copy held
/// peaked at 2.61 ×, one copy held at 1.93 ×; the bound is 2.25 ×.
#[tokio::test(flavor = "current_thread")]
async fn a_ring_crate_under_several_owners_is_held_once() {
let _one = ONE.lock().await;
let tmp = tempfile::tempdir().unwrap();
let items = || {
let mut nodes = vec![node("dep", "crate", "dep", Origin::Library, true)];
nodes.extend((0..BIG_ROWS).map(|i| node(&item("dep", i), "fn", "dep", Origin::Library, true)));
let edges: Vec<mimir_schema::EdgeRow> = (0..BIG_ROWS)
.map(|i| edge(&item("dep", i), &item("dep", (i + 1) % BIG_ROWS), EdgeKind::Calls))
.collect();
(nodes, edges)
};
let owners: Vec<&'static str> = (0..OWNERS).map(|o| &*Box::leak(format!("dep@1.0.0:lib#f{o}").into_boxed_str())).collect();
let step = Step {
owner: "k@0.1.0:lib",
nodes: vec![node("k", "crate", "k", Origin::Local, true), node("k a()", "fn", "k", Origin::Local, true)],
edges: vec![edge("k", "dep", EdgeKind::DependsOn), edge("k a()", &item("dep", 0), EdgeKind::Calls)],
binary: None,
ring: owners.iter().map(|o| {
let (n, e) = items();
(*o, n, e)
}).collect(),
};
let log = consume(open_log(tmp.path()).await, tmp.path(), ROOM, vec![step]).await;
let ring = FoldScope { repos: Default::default(), universe: true };
let mut whole = 0usize;
for kind in [Kind::Nodes, Kind::Edges] {
let rows = log.scan_rows(kind, &ring, &[]).await.unwrap();
whole += rows.batches.iter().map(|b| b.columns().iter().map(|c| c.get_array_memory_size()).sum::<usize>()).sum::<usize>();
}
Room::clear_cache();
let before = NOW.load(Relaxed);
PEAK.store(before, Relaxed);
let room = Room::build_uncached(&log, &Scope::current().with_universe(true)).await.unwrap();
let peak = (PEAK.load(Relaxed) - before) as usize;
for i in [0, BIG_ROWS / 2, BIG_ROWS - 1] {
assert!(room.ordinal(&item("dep", i)).is_some(), "item {i} of the ring crate is in the view");
}
// item_0 CALLS item_1: item_1's callers reach item_0
let callers = room.dependents_by(&[item("dep", 1).as_str()], &[EdgeKind::Calls], Some(1), None).unwrap();
assert!(room.monikers(&callers.value.reached).contains(&item("dep", 0).as_str()), "the ring crate's edges are in the view");
let ratio = peak as f64 / whole as f64;
eprintln!("ring view, one crate under {OWNERS} owners: peak {peak} B; reading the whole ring: {whole} B ({ratio:.2} ×)");
assert!(ratio < 2.25, "the ring view peaked at {peak} B, {ratio:.2} × reading the whole ring ({whole} B): every owner's copy held");
}
/// **The index keeps no scanned row.** The room's index used to borrow its
/// strings from the scanned batches, so every open row's columns (~490 bytes
/// a row on dwarf) were held until the index was done. Built as the rows
/// stream by, a row leaves a fixed-size record and its strings once each.
/// A log of many releases (four files a core, so the read comes in waves),
/// against what its node and edge scans cost: the index over kept rows
/// peaked at 2.92 ×, built while streaming at 1.96 ×; the bound is 2.4 ×
/// (the finished room itself is in the peak).
#[tokio::test(flavor = "current_thread")]
async fn the_index_keeps_no_scanned_row() {
let _one = ONE.lock().await;
let tmp = tempfile::tempdir().unwrap();
let commits = 4 * std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
// every commit adds a module of its own: nothing closes, the log grows
let mut nodes: Vec<mimir_schema::NodeRow> = Vec::new();
let mut edges: Vec<mimir_schema::EdgeRow> = Vec::new();
let mut steps = Vec::new();
for c in 0..commits {
for i in 0..400 {
let id = item("k", c * 1000 + i);
nodes.push(node(&id, "fn", "k", Origin::Local, true));
edges.push(edge(&id, &item("k", c * 1000 + (i + 1) % 400), EdgeKind::Calls));
}
steps.push(Step { owner: "k@0.1.0:lib", nodes: nodes.clone(), edges: edges.clone(), binary: None, ring: vec![] });
}
let log = consume(open_log(tmp.path()).await, tmp.path(), ROOM, steps).await;
let fs = FoldScope { repos: [(ROOM.to_string(), commits as i64)].into(), universe: false };
let mut scans = 0usize;
for kind in [Kind::Nodes, Kind::Edges] {
let rows = log.scan_rows(kind, &fs, &[]).await.unwrap();
scans += rows.batches.iter().map(|b| b.columns().iter().map(|c| c.get_array_memory_size()).sum::<usize>()).sum::<usize>();
}
Room::clear_cache();
let before = NOW.load(Relaxed);
PEAK.store(before, Relaxed);
let room = Room::build_uncached(&log, &Scope::current()).await.unwrap();
let peak = (PEAK.load(Relaxed) - before) as usize;
assert!(room.ordinal(&item("k", (commits - 1) * 1000 + 399)).is_some());
let ratio = peak as f64 / scans as f64;
eprintln!("index over {commits} releases: peak {peak} B; the node + edge scans: {scans} B ({ratio:.2} ×)");
assert!(ratio < 2.4, "the build peaked at {peak} B, {ratio:.2} × the node and edge scans ({scans} B): the index kept the scanned rows");
}