src/warehouse/agent_model_runs.rs
//! Iceberg writer + reader for the **H5 local-LLM bake-off** (`agent_model_runs`).
//!
//! A bake-off pits one prompt against a *set* of local models (ollama tags,
//! or the embedding registry's model ids) and records, per model, the answer
//! plus its latency + token economics. The viz **Leaderboard** pane reads a run
//! back and ranks the models by `tokens_per_s` + `score`.
//!
//! Write path: [`append_agent_model_runs`] appends one Iceberg snapshot per
//! bake-off run. Every row in a run shares one `run_id` + `ts_micros` so the
//! reader can group a run back together (exactly how `test_results` does).
//!
//! Read path: [`query_agent_model_runs`] scans the table, scopes by `run_id`
//! or `model`, and returns rows sorted by `(ts_micros, run_id, model, prompt_id)`
//! — Iceberg gives no scan order, so the reader imposes a stable one.
//!
//! Model-call layer: [`ModelCaller`] is a trait so the bake-off can be driven
//! by a real ollama backend ([`OllamaCaller`]) **or** by a canned-output mock
//! ([`MockCaller`]) — the tests inject the mock so they never require a running
//! LLM. [`run_bakeoff`] is the registry→call→row pipeline both the CLI and the
//! tests share.
//!
//! Schema: [`super::iceberg_schema::agent_model_runs`].
use std::collections::BTreeMap;
use std::sync::Arc;
use anyhow::{Result, anyhow};
use arrow::array::{
Array, BooleanArray, Float64Array, Int64Array, RecordBatch, StringArray,
TimestampMicrosecondArray,
};
use chrono::{TimeZone, Utc};
use iceberg::Catalog;
use iceberg::arrow::schema_to_arrow_schema;
use uuid::Uuid;
use super::iceberg::{IcebergWarehouse, TABLE_AGENT_MODEL_RUNS, append_batch, ensure_table_schema};
// Column indices match `iceberg_schema::agent_model_runs` field order.
const COL_RUN_ID: usize = 0;
const COL_TS_MICROS: usize = 1;
const COL_MODEL: usize = 2;
const COL_PROMPT_ID: usize = 3;
const COL_PROMPT: usize = 4;
const COL_OUTPUT: usize = 5;
const COL_LATENCY_MS: usize = 6;
const COL_TOKENS_IN: usize = 7;
const COL_TOKENS_OUT: usize = 8;
const COL_TOKENS_PER_S: usize = 9;
const COL_SCORE: usize = 10;
const COL_OK: usize = 11;
const COL_ERROR: usize = 12;
const COL_AGENT: usize = 13;
const COL_COST_USD: usize = 14;
const COL_MCP_TOOL_CALLS: usize = 15;
/// The fallback `agent` for rows written before the matrix axis existed (the
/// schema field is OPTIONAL, so a legacy null reads back as this).
pub const AGENT_UNSET: &str = "-";
/// One row of the `agent_model_runs` table — a single model's answer to one
/// prompt in a bake-off run.
///
/// `serde` derives let the server ship the exact rows to a thin viz client over
/// the `Viz.BakeoffResults` RPC (serialize → JSON → deserialize), so the remote
/// Leaderboard renders the identical leaderboard the embedded path computes.
#[derive(Debug, Clone, PartialEq, Default, serde::Serialize, serde::Deserialize)]
pub struct AgentModelRunRow {
/// The bake-off-run identity (groups a run's rows). One per `nornir bakeoff`.
pub run_id: String,
/// Run time, microseconds since the Unix epoch (UTC). Shared across a run's rows.
pub ts_micros: i64,
/// The **agent** (harness/caller identity) that drove this answer: `claude`,
/// `kilo`, a local-LLM driver, … The matrix's first axis. Defaults to
/// [`AGENT_UNSET`] for legacy single-axis rows. `serde` defaults it so a thin
/// client decoding a pre-matrix server's JSON still parses.
#[serde(default = "default_agent")]
pub agent: String,
/// The model that produced this answer (an ollama tag or registry id). The
/// matrix's second axis.
pub model: String,
/// Stable handle for the prompt (so two runs of the same prompt compare).
pub prompt_id: String,
/// The prompt text fed to the model.
pub prompt: String,
/// The model's completion (`""` = none, never null).
pub output: String,
/// Wall-clock duration of the model call, milliseconds.
pub latency_ms: f64,
/// Prompt token count.
pub tokens_in: i64,
/// Completion token count.
pub tokens_out: i64,
/// Decode throughput: completion tokens per second.
pub tokens_per_s: f64,
/// Quality score in `0.0..=1.0` (a judge / heuristic; `0.0` if unscored).
pub score: f64,
/// Did the model answer without error?
pub ok: bool,
/// Error detail when `ok == false` (`None` when the call succeeded).
pub error: Option<String>,
/// USD cost of this cell (`0.0` = free/local model). Defaults to `0.0` for
/// legacy rows / a thin client decoding a pre-matrix server.
#[serde(default)]
pub cost_usd: f64,
/// How many MCP tool calls the agent made answering (`0` if none / unknown).
#[serde(default)]
pub mcp_tool_calls: i64,
}
/// `serde` default for [`AgentModelRunRow::agent`] — keeps a pre-matrix server's
/// JSON (no `agent` field) decoding to [`AGENT_UNSET`].
fn default_agent() -> String {
AGENT_UNSET.to_string()
}
impl AgentModelRunRow {
/// The `(ts_micros, run_id, agent, model, prompt_id)` stable sort key.
fn key(&self) -> (i64, String, String, String, String) {
(
self.ts_micros,
self.run_id.clone(),
self.agent.clone(),
self.model.clone(),
self.prompt_id.clone(),
)
}
/// The `(agent, model)` matrix cell this row belongs to.
pub fn cell(&self) -> (String, String) {
(self.agent.clone(), self.model.clone())
}
}
/// Append a batch of `agent_model_runs` rows (one Iceberg snapshot).
pub async fn append_agent_model_runs(
wh: &IcebergWarehouse,
rows: &[AgentModelRunRow],
) -> Result<()> {
if rows.is_empty() {
return Ok(());
}
let ident = wh.table_ident(TABLE_AGENT_MODEL_RUNS);
let table = wh.catalog().load_table(&ident).await?;
// Stale-server tables (created before the matrix axis + economics columns
// existed) carry only 13 of the 16 canonical columns; evolve them forward
// with an Iceberg add-column migration so the full row appends cleanly.
let table = ensure_table_schema(
wh.catalog(),
&ident,
table,
&super::iceberg_schema::agent_model_runs()?,
)
.await?;
let arrow_schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?);
let cols: Vec<Arc<dyn Array>> = vec![
Arc::new(StringArray::from(
rows.iter().map(|r| r.run_id.clone()).collect::<Vec<_>>(),
)),
Arc::new(
TimestampMicrosecondArray::from(rows.iter().map(|r| r.ts_micros).collect::<Vec<_>>())
.with_timezone("+00:00"),
),
Arc::new(StringArray::from(
rows.iter().map(|r| r.model.clone()).collect::<Vec<_>>(),
)),
Arc::new(StringArray::from(
rows.iter().map(|r| r.prompt_id.clone()).collect::<Vec<_>>(),
)),
Arc::new(StringArray::from(
rows.iter().map(|r| r.prompt.clone()).collect::<Vec<_>>(),
)),
Arc::new(StringArray::from(
rows.iter().map(|r| r.output.clone()).collect::<Vec<_>>(),
)),
Arc::new(Float64Array::from(
rows.iter().map(|r| r.latency_ms).collect::<Vec<_>>(),
)),
Arc::new(Int64Array::from(
rows.iter().map(|r| r.tokens_in).collect::<Vec<_>>(),
)),
Arc::new(Int64Array::from(
rows.iter().map(|r| r.tokens_out).collect::<Vec<_>>(),
)),
Arc::new(Float64Array::from(
rows.iter().map(|r| r.tokens_per_s).collect::<Vec<_>>(),
)),
Arc::new(Float64Array::from(
rows.iter().map(|r| r.score).collect::<Vec<_>>(),
)),
Arc::new(BooleanArray::from(
rows.iter().map(|r| r.ok).collect::<Vec<_>>(),
)),
Arc::new(StringArray::from(
rows.iter()
.map(|r| r.error.clone())
.collect::<Vec<Option<String>>>(),
)),
// Matrix axis + economics (optional cols; always written going forward).
Arc::new(StringArray::from(
rows.iter()
.map(|r| Some(r.agent.clone()))
.collect::<Vec<Option<String>>>(),
)),
Arc::new(Float64Array::from(
rows.iter()
.map(|r| Some(r.cost_usd))
.collect::<Vec<Option<f64>>>(),
)),
Arc::new(Int64Array::from(
rows.iter()
.map(|r| Some(r.mcp_tool_calls))
.collect::<Vec<Option<i64>>>(),
)),
];
let batch = RecordBatch::try_new(arrow_schema, cols)?;
append_batch(wh.catalog(), table, batch).await?;
Ok(())
}
/// Which bake-off rows to read.
#[derive(Debug, Clone)]
pub enum BakeoffSelector {
/// Exactly one bake-off run (its `run_id`).
Run(String),
/// Every row whose `model` matches — across all runs (a per-model history).
Model(String),
/// Every row whose `agent` matches — across all runs (a per-agent history).
Agent(String),
/// Everything in the table.
All,
}
/// Read bake-off rows, scoped by `sel`, returned sorted by
/// `(ts_micros, run_id, model, prompt_id)`.
pub async fn query_agent_model_runs(
wh: &IcebergWarehouse,
sel: &BakeoffSelector,
) -> Result<Vec<AgentModelRunRow>> {
// Blank/torn Iceberg metadata JSON → 0 rows, never a JSON-EOF crash (load+read).
let batches: Vec<RecordBatch> =
super::iceberg::load_and_read_all(wh, TABLE_AGENT_MODEL_RUNS).await?;
let mut out: Vec<AgentModelRunRow> = Vec::new();
for b in &batches {
let run_id = col_str(b, COL_RUN_ID)?;
let ts = col_ts(b, COL_TS_MICROS)?;
let model = col_str(b, COL_MODEL)?;
let prompt_id = col_str(b, COL_PROMPT_ID)?;
let prompt = col_str(b, COL_PROMPT)?;
let output = col_str(b, COL_OUTPUT)?;
let latency = col_f64(b, COL_LATENCY_MS)?;
let tin = col_i64(b, COL_TOKENS_IN)?;
let tout = col_i64(b, COL_TOKENS_OUT)?;
let tps = col_f64(b, COL_TOKENS_PER_S)?;
let score = col_f64(b, COL_SCORE)?;
let ok = col_bool(b, COL_OK)?;
let error = col_str(b, COL_ERROR)?;
// Matrix columns are OPTIONAL + may be absent from a pre-matrix batch:
// probe by column count, default when missing/null.
let agent = if b.num_columns() > COL_AGENT {
Some(col_str(b, COL_AGENT)?)
} else {
None
};
let cost = if b.num_columns() > COL_COST_USD {
Some(col_f64(b, COL_COST_USD)?)
} else {
None
};
let mcp = if b.num_columns() > COL_MCP_TOOL_CALLS {
Some(col_i64(b, COL_MCP_TOOL_CALLS)?)
} else {
None
};
for i in 0..b.num_rows() {
let row = AgentModelRunRow {
run_id: run_id.value(i).to_string(),
ts_micros: ts.value(i),
agent: match agent {
Some(a) if !a.is_null(i) => a.value(i).to_string(),
_ => AGENT_UNSET.to_string(),
},
model: model.value(i).to_string(),
prompt_id: prompt_id.value(i).to_string(),
prompt: prompt.value(i).to_string(),
output: output.value(i).to_string(),
latency_ms: latency.value(i),
tokens_in: tin.value(i),
tokens_out: tout.value(i),
tokens_per_s: tps.value(i),
score: score.value(i),
ok: ok.value(i),
error: if error.is_null(i) {
None
} else {
Some(error.value(i).to_string())
},
cost_usd: match cost {
Some(c) if !c.is_null(i) => c.value(i),
_ => 0.0,
},
mcp_tool_calls: match mcp {
Some(m) if !m.is_null(i) => m.value(i),
_ => 0,
},
};
let keep = match sel {
BakeoffSelector::Run(id) => &row.run_id == id,
BakeoffSelector::Model(m) => &row.model == m,
BakeoffSelector::Agent(a) => &row.agent == a,
BakeoffSelector::All => true,
};
if keep {
out.push(row);
}
}
}
out.sort_by_key(|r| r.key());
Ok(out)
}
// ─── leaderboard ranking ─────────────────────────────────────────────────
/// One ranked model in a bake-off's leaderboard — the unit the viz pane and the
/// CLI table both render.
#[derive(Debug, Clone, PartialEq)]
pub struct LeaderboardEntry {
/// 1-based rank (1 = best).
pub rank: usize,
/// The agent (matrix axis 1) that produced this cell.
pub agent: String,
/// The model (matrix axis 2).
pub model: String,
pub tokens_per_s: f64,
pub score: f64,
pub latency_ms: f64,
pub tokens_out: i64,
pub cost_usd: f64,
pub mcp_tool_calls: i64,
pub ok: bool,
/// The (truncated for display) answer this cell gave.
pub output: String,
}
impl LeaderboardEntry {
/// `agent/model` cell label (the matrix coordinate).
pub fn cell_label(&self) -> String {
format!("{}/{}", self.agent, self.model)
}
}
/// Rank a run's rows into a leaderboard over `(agent, model)` cells. **Best
/// first**: green (`ok`) cells outrank failed ones; within those, higher `score`
/// wins, then higher `tokens_per_s`, then lower `latency_ms`, then lower
/// `cost_usd` — a stable, deterministic order so the viz pane and the CLI agree
/// and a test can assert the exact ranking. Final tie-break is the
/// `(agent, model)` cell name so equal cells stay in a fixed order.
pub fn leaderboard(rows: &[AgentModelRunRow]) -> Vec<LeaderboardEntry> {
let mut ranked: Vec<&AgentModelRunRow> = rows.iter().collect();
ranked.sort_by(|a, b| {
// ok first
b.ok.cmp(&a.ok)
// higher score first
.then(
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal),
)
// higher throughput first
.then(
b.tokens_per_s
.partial_cmp(&a.tokens_per_s)
.unwrap_or(std::cmp::Ordering::Equal),
)
// lower latency first
.then(
a.latency_ms
.partial_cmp(&b.latency_ms)
.unwrap_or(std::cmp::Ordering::Equal),
)
// cheaper first
.then(
a.cost_usd
.partial_cmp(&b.cost_usd)
.unwrap_or(std::cmp::Ordering::Equal),
)
// stable tie-break by (agent, model) cell name
.then(a.agent.cmp(&b.agent))
.then(a.model.cmp(&b.model))
});
ranked
.into_iter()
.enumerate()
.map(|(i, r)| LeaderboardEntry {
rank: i + 1,
agent: r.agent.clone(),
model: r.model.clone(),
tokens_per_s: r.tokens_per_s,
score: r.score,
latency_ms: r.latency_ms,
tokens_out: r.tokens_out,
cost_usd: r.cost_usd,
mcp_tool_calls: r.mcp_tool_calls,
ok: r.ok,
output: r.output.clone(),
})
.collect()
}
// ─── matrix grid ─────────────────────────────────────────────────────────
/// One `(agent, model)` cell of a bake-off matrix — the unit the viz grid and
/// the CLI matrix table render. Aggregates the rows at that cell (a single task
/// gives one row; multiple tasks average) into the comparable metrics, plus the
/// cell's rank within the run.
#[derive(Debug, Clone, PartialEq)]
pub struct MatrixCell {
pub agent: String,
pub model: String,
/// 1-based rank within the run (1 = the winning cell).
pub rank: usize,
pub score: f64,
pub tokens_per_s: f64,
pub latency_ms: f64,
pub cost_usd: f64,
pub mcp_tool_calls: i64,
/// How many task rows landed in this cell.
pub task_count: usize,
/// All this cell's task rows answered without error.
pub ok: bool,
}
/// The full agent × model grid for a run: the sorted axis labels and the cell at
/// each coordinate, with the overall winner highlighted by `rank == 1`. This is
/// the shape both the 🏆 viz grid and `bakeoff leaderboard` render, and what
/// `state_json` exposes (LAW #6).
#[derive(Debug, Clone, PartialEq)]
pub struct MatrixGrid {
/// Distinct agents (rows of the grid), sorted.
pub agents: Vec<String>,
/// Distinct models (columns of the grid), sorted.
pub models: Vec<String>,
/// One cell per occupied `(agent, model)` coordinate.
pub cells: Vec<MatrixCell>,
}
impl MatrixGrid {
/// The winning cell (rank 1), if any.
pub fn winner(&self) -> Option<&MatrixCell> {
self.cells.iter().find(|c| c.rank == 1)
}
/// Look up a cell by coordinate.
pub fn cell(&self, agent: &str, model: &str) -> Option<&MatrixCell> {
self.cells
.iter()
.find(|c| c.agent == agent && c.model == model)
}
}
/// Build the agent × model [`MatrixGrid`] for a run's rows. Each `(agent, model)`
/// coordinate aggregates its task rows (mean score / tok/s / latency / cost, sum
/// of mcp calls, AND of `ok`), then the cells are ranked by the SAME order
/// `leaderboard` uses (score → tok/s → latency → cost) so the grid winner is the
/// leaderboard winner.
pub fn matrix_grid(rows: &[AgentModelRunRow]) -> MatrixGrid {
use std::collections::BTreeSet;
let mut by_cell: BTreeMap<(String, String), Vec<&AgentModelRunRow>> = BTreeMap::new();
for r in rows {
by_cell.entry(r.cell()).or_default().push(r);
}
let mut cells: Vec<MatrixCell> = by_cell
.into_iter()
.map(|((agent, model), group)| {
let n = group.len().max(1) as f64;
let mean = |f: &dyn Fn(&AgentModelRunRow) -> f64| -> f64 {
group.iter().map(|r| f(r)).sum::<f64>() / n
};
MatrixCell {
agent,
model,
rank: 0, // filled after ranking
score: mean(&|r| r.score),
tokens_per_s: mean(&|r| r.tokens_per_s),
latency_ms: mean(&|r| r.latency_ms),
cost_usd: mean(&|r| r.cost_usd),
mcp_tool_calls: group.iter().map(|r| r.mcp_tool_calls).sum(),
task_count: group.len(),
ok: group.iter().all(|r| r.ok),
}
})
.collect();
// Rank cells with the SAME comparator the leaderboard uses.
cells.sort_by(|a, b| {
b.ok.cmp(&a.ok)
.then(
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
b.tokens_per_s
.partial_cmp(&a.tokens_per_s)
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
a.latency_ms
.partial_cmp(&b.latency_ms)
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(
a.cost_usd
.partial_cmp(&b.cost_usd)
.unwrap_or(std::cmp::Ordering::Equal),
)
.then(a.agent.cmp(&b.agent))
.then(a.model.cmp(&b.model))
});
for (i, c) in cells.iter_mut().enumerate() {
c.rank = i + 1;
}
let agents: BTreeSet<String> = cells.iter().map(|c| c.agent.clone()).collect();
let models: BTreeSet<String> = cells.iter().map(|c| c.model.clone()).collect();
MatrixGrid {
agents: agents.into_iter().collect(),
models: models.into_iter().collect(),
cells,
}
}
// ─── rendering: JSON + human view ────────────────────────────────────────
fn esc(s: &str) -> String {
let mut o = String::with_capacity(s.len() + 2);
for c in s.chars() {
match c {
'"' => o.push_str("\\\""),
'\\' => o.push_str("\\\\"),
'\n' => o.push_str("\\n"),
'\t' => o.push_str("\\t"),
'\r' => o.push_str("\\r"),
c => o.push(c),
}
}
o
}
/// Serialize rows to a stable JSON array (one object per answer). Pure string
/// assembly so the CLI stays dependency-light (mirrors `release_events`).
pub fn rows_to_json(rows: &[AgentModelRunRow]) -> String {
let mut s = String::from("[\n");
for (i, r) in rows.iter().enumerate() {
let ts_rfc = Utc
.timestamp_micros(r.ts_micros)
.single()
.map(|d| d.to_rfc3339())
.unwrap_or_default();
let error = match &r.error {
None => "null".to_string(),
Some(e) => format!("\"{}\"", esc(e)),
};
s.push_str(&format!(
" {{\"run_id\": \"{}\", \"ts\": \"{}\", \"agent\": \"{}\", \"model\": \"{}\", \
\"prompt_id\": \"{}\", \"prompt\": \"{}\", \"output\": \"{}\", \
\"latency_ms\": {:.3}, \"tokens_in\": {}, \"tokens_out\": {}, \
\"tokens_per_s\": {:.3}, \"score\": {:.3}, \"cost_usd\": {:.4}, \
\"mcp_tool_calls\": {}, \"ok\": {}, \"error\": {}}}{}\n",
esc(&r.run_id),
esc(&ts_rfc),
esc(&r.agent),
esc(&r.model),
esc(&r.prompt_id),
esc(&r.prompt),
esc(&r.output),
r.latency_ms,
r.tokens_in,
r.tokens_out,
r.tokens_per_s,
r.score,
r.cost_usd,
r.mcp_tool_calls,
r.ok,
error,
if i + 1 < rows.len() { "," } else { "" },
));
}
s.push(']');
s
}
/// Human-readable leaderboard for one run's rows: ranked rows, fastest first.
pub fn render_leaderboard(rows: &[AgentModelRunRow]) -> String {
if rows.is_empty() {
return "(no bake-off runs recorded)\n".to_string();
}
let board = leaderboard(rows);
let mut out = String::new();
out.push_str(
" # agent/model score tok/s latency cost mcp status\n",
);
for e in &board {
out.push_str(&format!(
" {:<2} {:<28} {:>6.3} {:>8.1} {:>8.0}ms ${:>6.4} {:>4} {}\n",
e.rank,
truncate(&e.cell_label(), 28),
e.score,
e.tokens_per_s,
e.latency_ms,
e.cost_usd,
e.mcp_tool_calls,
if e.ok { "✓" } else { "✗" },
));
}
out
}
/// Human-readable agent × model **matrix** for a run's rows: a grid with agents
/// down the rows and models across the columns, each cell showing its score and
/// rank (the winner cell marked `*`). Complements [`render_leaderboard`] (the
/// flat ranking) with the cross-product view the matrix is named for.
pub fn render_matrix(rows: &[AgentModelRunRow]) -> String {
if rows.is_empty() {
return "(no bake-off runs recorded)\n".to_string();
}
let grid = matrix_grid(rows);
let mut out = String::new();
// Header: model columns.
out.push_str(&format!(" {:<16}", "agent \\ model"));
for m in &grid.models {
out.push_str(&format!(" {:>14}", truncate(m, 14)));
}
out.push('\n');
for a in &grid.agents {
out.push_str(&format!(" {:<16}", truncate(a, 16)));
for m in &grid.models {
match grid.cell(a, m) {
Some(c) => {
let win = if c.rank == 1 { "*" } else { " " };
let status = if c.ok { "" } else { "✗" };
out.push_str(&format!(" {win}{:>5.3}#{:<2}{status:>3}", c.score, c.rank));
}
None => out.push_str(&format!(" {:>14}", "·")),
}
}
out.push('\n');
}
if let Some(w) = grid.winner() {
out.push_str(&format!(
" winner: {}/{} (score {:.3}, {:.1} tok/s)\n",
w.agent, w.model, w.score, w.tokens_per_s
));
}
out
}
/// `bakeoff leaderboard` / `bakeoff show` rendered as the uniform
/// [`crate::cli_outcome::CommandOutcome`] — the shared DATA FACE for the CLI
/// (`run_bakeoff_cmd`, fat) and its thin twin (`Viz.BakeoffResults` RPC), and by
/// parity the viz 🏆 Leaderboard tab. The `rows` arrive ALREADY fetched from the
/// canonical source (the warehouse `query_agent_model_runs` in fat mode, the RPC in
/// thin mode); this fn only SHAPES + renders them — it never fetches (CommandOutcome
/// is presentation, never a transport). `matrix` toggles the grid vs the flat ranked
/// board for the `human` rendering. `ok ⟺ ≥1 row` (RAGNARÖK: an empty leaderboard is
/// RED, never silently green). AUT9 — see `.nornir/cli-command-contract.md`.
pub fn leaderboard_outcome(
reference: &str,
rows: &[AgentModelRunRow],
matrix: bool,
) -> crate::cli_outcome::CommandOutcome {
use crate::cli_outcome::CommandOutcome;
if rows.is_empty() {
let scope = if reference.is_empty() {
"the most recent run".to_string()
} else {
format!("`{reference}`")
};
return CommandOutcome::fail(
"bakeoff leaderboard",
format!(
"no bake-off runs recorded for {scope} — run `nornir bakeoff run --task …` or `nornir bakeoff demo` first"
),
);
}
let human = if matrix {
render_matrix(rows)
} else {
render_leaderboard(rows)
};
let data = serde_json::json!({
"reference": reference,
"rows": rows,
});
CommandOutcome::ok("bakeoff leaderboard", data, human.trim_end().to_string())
}
fn truncate(s: &str, n: usize) -> String {
if s.chars().count() <= n {
s.to_string()
} else {
let head: String = s.chars().take(n.saturating_sub(1)).collect();
format!("{head}…")
}
}
/// Abbreviate a run id for compact display (first 8 chars of a UUID, else whole).
pub fn short_run(run_id: &str) -> String {
if run_id.len() > 12 {
format!("{}…", &run_id[..8])
} else {
run_id.to_string()
}
}
/// Convenience: a fresh run id (UUIDv4 string).
pub fn new_run_id() -> String {
Uuid::new_v4().to_string()
}
// ─── model-call layer (mockable) ─────────────────────────────────────────
/// One model's answer to a prompt, with the economics the row needs. The
/// caller (real or mock) returns this; [`run_bakeoff`] stamps it into a row.
#[derive(Debug, Clone, PartialEq)]
pub struct ModelAnswer {
pub output: String,
pub latency_ms: f64,
pub tokens_in: i64,
pub tokens_out: i64,
/// Decode throughput. `0.0` ⇒ [`run_bakeoff`] derives it from
/// `tokens_out / (latency_ms/1000)`.
pub tokens_per_s: f64,
/// Quality score in `0.0..=1.0` (the backend may leave this `0.0`).
pub score: f64,
/// USD cost of the call (`0.0` = free/local).
pub cost_usd: f64,
/// MCP tool calls the agent made (`0` if none / not tracked).
pub mcp_tool_calls: i64,
}
impl ModelAnswer {
/// A minimal answer with no economics beyond latency/tokens (cost-free,
/// no MCP calls). Convenience for the ollama/mock paths that don't price.
pub fn basic(
output: impl Into<String>,
latency_ms: f64,
tokens_in: i64,
tokens_out: i64,
tokens_per_s: f64,
score: f64,
) -> Self {
ModelAnswer {
output: output.into(),
latency_ms,
tokens_in,
tokens_out,
tokens_per_s,
score,
cost_usd: 0.0,
mcp_tool_calls: 0,
}
}
}
/// Pluggable bake-off backend. A bake-off calls `call(agent, model, prompt)`
/// once per `(agent, model)` matrix cell; the impl is the only thing that
/// touches an actual LLM / agent harness, so a test injects [`MockCaller`] and
/// never needs a running ollama or Claude.
///
/// The `agent` is the harness/caller identity (claude / kilo / a local-LLM
/// driver) — the matrix's first axis. An impl that ignores `agent` (e.g. plain
/// ollama, which has no agent concept) simply runs `model` regardless; the row
/// still records which agent column it was placed in.
pub trait ModelCaller {
/// Run `prompt` for the `(agent, model)` cell, returning the answer +
/// economics, or an error (recorded as a red `ok=false` row, never aborting
/// the bake-off).
fn call(&self, agent: &str, model: &str, prompt: &str) -> Result<ModelAnswer>;
}
/// Canned-output caller for tests + offline use: maps each `(agent, model)`
/// matrix cell to a fixed [`ModelAnswer`]. A cell absent from the map returns an
/// error (so the bake-off records a red row, exercising the failure path).
///
/// `with(model, …)` registers a cell under the [`AGENT_UNSET`] agent (the
/// single-axis shorthand, so legacy callers keep working); `with_cell(agent,
/// model, …)` registers a specific matrix cell.
#[derive(Debug, Clone, Default)]
pub struct MockCaller {
answers: BTreeMap<(String, String), ModelAnswer>,
}
impl MockCaller {
pub fn new() -> Self {
Self::default()
}
/// Register the answer for the `(AGENT_UNSET, model)` cell. Chainable.
pub fn with(self, model: impl Into<String>, answer: ModelAnswer) -> Self {
self.with_cell(AGENT_UNSET, model, answer)
}
/// Register the answer for the `(agent, model)` matrix cell. Chainable.
pub fn with_cell(
mut self,
agent: impl Into<String>,
model: impl Into<String>,
answer: ModelAnswer,
) -> Self {
self.answers.insert((agent.into(), model.into()), answer);
self
}
}
impl ModelCaller for MockCaller {
fn call(&self, agent: &str, model: &str, _prompt: &str) -> Result<ModelAnswer> {
self.answers
.get(&(agent.to_string(), model.to_string()))
.cloned()
.ok_or_else(|| anyhow!("mock: no canned answer for cell `{agent}/{model}`"))
}
}
/// Real ollama backend: POSTs to `{host}/api/generate` (default
/// `http://localhost:11434`) and reads the non-streaming response's
/// `response` + `prompt_eval_count` / `eval_count` / `eval_duration`. Uses the
/// already-vendored `ureq` client (no new dependency). Errors (host down, model
/// not pulled, …) surface as a red row, never aborting the bake-off.
pub struct OllamaCaller {
host: String,
}
impl OllamaCaller {
/// New caller against `host` (e.g. `http://localhost:11434`). The
/// `$OLLAMA_HOST` env var, when set, overrides a `None`.
pub fn new(host: Option<String>) -> Self {
let host = host
.or_else(|| std::env::var("OLLAMA_HOST").ok())
.unwrap_or_else(|| "http://localhost:11434".to_string());
Self {
host: host.trim_end_matches('/').to_string(),
}
}
}
impl ModelCaller for OllamaCaller {
fn call(&self, _agent: &str, model: &str, prompt: &str) -> Result<ModelAnswer> {
let url = format!("{}/api/generate", self.host);
let body = serde_json::json!({
"model": model,
"prompt": prompt,
"stream": false,
});
let body_str = serde_json::to_string(&body)?;
let started = std::time::Instant::now();
// `ureq` here is built without its `json` feature, so POST the JSON as a
// string with an explicit Content-Type and parse the reply by hand.
let resp = crate::http_agent().post(&url)
.header("Content-Type", "application/json")
.send(&body_str)
.map_err(|e| anyhow!("ollama POST {url} failed: {e}"))?;
let txt = resp
.into_body()
.read_to_string()
.map_err(|e| anyhow!("ollama response not readable: {e}"))?;
let v: serde_json::Value =
serde_json::from_str(&txt).map_err(|e| anyhow!("ollama response not JSON: {e}"))?;
let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
let output = v
.get("response")
.and_then(|x| x.as_str())
.unwrap_or("")
.to_string();
let tokens_in = v
.get("prompt_eval_count")
.and_then(|x| x.as_i64())
.unwrap_or(0);
let tokens_out = v.get("eval_count").and_then(|x| x.as_i64()).unwrap_or(0);
// ollama reports eval_duration in nanoseconds; prefer it for tok/s.
let tokens_per_s = match v.get("eval_duration").and_then(|x| x.as_i64()) {
Some(ns) if ns > 0 => tokens_out as f64 / (ns as f64 / 1e9),
_ => 0.0,
};
Ok(ModelAnswer::basic(
output,
latency_ms,
tokens_in,
tokens_out,
tokens_per_s,
0.0,
))
}
}
/// Run a bake-off: call `caller` once per model in `models` with `prompt`, and
/// build one [`AgentModelRunRow`] per model (all sharing `run_id` + `ts_micros`).
/// A per-model error is captured as a red `ok=false` row — the bake-off never
/// aborts because one model is missing. `tokens_per_s` is derived from
/// `tokens_out / latency` when the backend didn't supply it.
///
/// This is the pure pipeline the CLI and the tests share; it does NOT touch the
/// warehouse (the caller appends the returned rows), so a test can run a full
/// bake-off with a [`MockCaller`] and assert the rows before any I/O.
pub fn run_bakeoff(
caller: &dyn ModelCaller,
run_id: &str,
ts_micros: i64,
prompt_id: &str,
prompt: &str,
models: &[String],
) -> Vec<AgentModelRunRow> {
// A single-agent bake-off is the 1×N matrix under the AGENT_UNSET agent.
run_bakeoff_matrix(
caller,
run_id,
ts_micros,
prompt_id,
prompt,
&[AGENT_UNSET.to_string()],
models,
)
}
/// Run a bake-off **matrix**: call `caller` once per `(agent, model)` cell of the
/// `agents × models` cross-product with `prompt`, building one
/// [`AgentModelRunRow`] per cell (all sharing `run_id` + `ts_micros`). A
/// per-cell error is captured as a red `ok=false` row — the matrix never aborts
/// because one cell is missing. `tokens_per_s` is derived from
/// `tokens_out / latency` when the backend didn't supply it.
///
/// This is the pure pipeline the CLI and the tests share; it does NOT touch the
/// warehouse (the caller appends the returned rows), so a test runs a full
/// matrix with a [`MockCaller`] and asserts the rows before any I/O. Row order is
/// the cross-product in `(agent, model)` order — stable and testable.
pub fn run_bakeoff_matrix(
caller: &dyn ModelCaller,
run_id: &str,
ts_micros: i64,
prompt_id: &str,
prompt: &str,
agents: &[String],
models: &[String],
) -> Vec<AgentModelRunRow> {
let mut rows = Vec::with_capacity(agents.len() * models.len());
for agent in agents {
for model in models {
let row = match caller.call(agent, model, prompt) {
Ok(ans) => {
let tps = if ans.tokens_per_s > 0.0 {
ans.tokens_per_s
} else if ans.latency_ms > 0.0 {
ans.tokens_out as f64 / (ans.latency_ms / 1000.0)
} else {
0.0
};
AgentModelRunRow {
run_id: run_id.to_string(),
ts_micros,
agent: agent.clone(),
model: model.clone(),
prompt_id: prompt_id.to_string(),
prompt: prompt.to_string(),
output: ans.output,
latency_ms: ans.latency_ms,
tokens_in: ans.tokens_in,
tokens_out: ans.tokens_out,
tokens_per_s: tps,
score: ans.score,
ok: true,
error: None,
cost_usd: ans.cost_usd,
mcp_tool_calls: ans.mcp_tool_calls,
}
}
Err(e) => AgentModelRunRow {
run_id: run_id.to_string(),
ts_micros,
agent: agent.clone(),
model: model.clone(),
prompt_id: prompt_id.to_string(),
prompt: prompt.to_string(),
output: String::new(),
latency_ms: 0.0,
tokens_in: 0,
tokens_out: 0,
tokens_per_s: 0.0,
score: 0.0,
ok: false,
error: Some(format!("{e:#}")),
cost_usd: 0.0,
mcp_tool_calls: 0,
},
};
rows.push(row);
}
}
rows
}
/// Score a run's rows with an [`AnswerJudge`](super::codegen_judge::AnswerJudge),
/// **in place**: for every OK row, judge its `output` and replace the (usually
/// `0.0`) `score` with the verdict's. This is what wires the compile-and-test
/// [`CodegenJudge`](super::codegen_judge::CodegenJudge) into the bake-off so a
/// REAL ollama run gets a real quality score instead of being ranked by speed
/// alone.
///
/// Failed rows (`ok == false` — the model never answered) are left at `0.0`:
/// there is nothing to compile. A judge *infrastructure* error (e.g. the target
/// crate can't be copied) is non-fatal — the row keeps its prior score and the
/// error is returned in the per-row report so the caller can surface it — the
/// bake-off never aborts because the judge tripped.
///
/// Returns one [`JudgeReport`] per row judged (the OK rows), in row order, so
/// the CLI can print each cell's verdict. Pure (no warehouse I/O): the caller
/// persists the re-scored rows.
pub fn score_rows_with_judge(
rows: &mut [AgentModelRunRow],
judge: &dyn super::codegen_judge::AnswerJudge,
) -> Vec<JudgeReport> {
let mut reports = Vec::new();
for row in rows.iter_mut() {
if !row.ok {
continue; // no answer to compile/test
}
let report = match judge.judge(&row.output) {
Ok(verdict) => {
row.score = verdict.score;
JudgeReport {
agent: row.agent.clone(),
model: row.model.clone(),
score: verdict.score,
verdict: Some(verdict),
error: None,
}
}
Err(e) => JudgeReport {
agent: row.agent.clone(),
model: row.model.clone(),
score: row.score,
verdict: None,
error: Some(format!("{e:#}")),
},
};
reports.push(report);
}
reports
}
/// One cell's judging outcome, returned by [`score_rows_with_judge`] so the CLI
/// can print per-cell verdicts. `verdict` is `None` (and `error` set) when the
/// judge's own infrastructure failed for that cell.
#[derive(Debug, Clone)]
pub struct JudgeReport {
pub agent: String,
pub model: String,
/// The score the row ended up with (the verdict's, or the prior score if the
/// judge errored).
pub score: f64,
pub verdict: Option<super::codegen_judge::Verdict>,
pub error: Option<String>,
}
/// The default bake-off agent set when `--agents` is omitted: a single
/// local-LLM agent (`AGENT_UNSET`) so a model-only bake-off behaves exactly as
/// before. The CLI passes `--agents` to widen the matrix.
pub fn default_agents() -> Vec<String> {
vec![AGENT_UNSET.to_string()]
}
/// The default bake-off model set: the embedding-model registry's ids. The
/// bake-off reuses this same registry (H5 requirement) so `--models` defaults
/// to "every model nornir knows about" when the caller passes none.
///
/// The registry (`src/vector/embed_registry.rs`) is feature-gated behind
/// `vector`; when that feature is off we fall back to the same ids the registry
/// declares so the default still names the known models.
#[cfg(feature = "vector")]
pub fn default_models() -> Vec<String> {
crate::vector::embed_registry::MODELS
.iter()
.map(|m| m.id.to_string())
.collect()
}
/// Fallback default model set when the `vector` feature (and thus the live
/// registry) is not compiled in. Kept in sync with
/// `src/vector/embed_registry.rs::MODELS`.
#[cfg(not(feature = "vector"))]
pub fn default_models() -> Vec<String> {
["jina-v2-base-code", "minilm-l6-v2", "bge-base-en-v1.5"]
.iter()
.map(|s| s.to_string())
.collect()
}
// ─── demo seeder ─────────────────────────────────────────────────────────
/// A realistic mock bake-off **matrix** for `nornir bakeoff demo` — a populated
/// agent × model leaderboard a fresh viz can show without a live LLM. Mirrors
/// `funnel demo`: pure, deterministic, warehouse-bound.
///
/// Crosses 3 agents (`claude`, `kilo`, `local-llm`) × 3 models
/// (`opus`, `sonnet`, `mistral`) over one task `fix-flaky-test`, with plausible
/// economics: hosted agents score higher but cost money and make MCP tool calls;
/// the local-llm agent is free + fast-but-weaker; one cell (`local-llm/opus`)
/// fails (the local driver can't run a hosted model) to exercise the red path.
/// `run_id` + `ts_micros` are passed in so the seeder is reproducible.
pub fn demo_matrix_rows(run_id: &str, ts_micros: i64) -> Vec<AgentModelRunRow> {
let task = "Fix the flaky test in src/foo.rs and explain the race.";
let prompt_id = "fix-flaky-test";
let caller = demo_caller();
let agents: Vec<String> = ["claude", "kilo", "local-llm"]
.iter()
.map(|s| s.to_string())
.collect();
let models: Vec<String> = ["opus", "sonnet", "mistral"]
.iter()
.map(|s| s.to_string())
.collect();
run_bakeoff_matrix(
&caller, run_id, ts_micros, prompt_id, task, &agents, &models,
)
}
/// The canned [`MockCaller`] behind [`demo_matrix_rows`] — one entry per
/// `(agent, model)` cell, except `local-llm/opus` which is intentionally absent
/// (→ a red failed cell, since a local driver can't run a hosted model).
fn demo_caller() -> MockCaller {
let mut m = MockCaller::new();
// (agent, model, output, latency_ms, tin, tout, tps, score, cost_usd, mcp)
let cells: &[(&str, &str, &str, f64, i64, i64, f64, f64, f64, i64)] = &[
(
"claude",
"opus",
"Patched the race with a Mutex; root cause: TOCTOU on the cache.",
4200.0,
1800,
520,
124.0,
0.97,
0.0840,
6,
),
(
"claude",
"sonnet",
"Added a lock around the shared map; the test was racing on init.",
2100.0,
1800,
480,
228.0,
0.91,
0.0210,
5,
),
(
"claude",
"mistral",
"Wrapped the counter in an atomic.",
1400.0,
1800,
300,
214.0,
0.74,
0.0000,
3,
),
(
"kilo",
"opus",
"Serialized the two writers; documented the ordering.",
4600.0,
1900,
540,
117.0,
0.93,
0.0880,
4,
),
(
"kilo",
"sonnet",
"Guarded the lazy-init with Once.",
2400.0,
1900,
460,
191.0,
0.88,
0.0220,
4,
),
(
"kilo",
"mistral",
"Added a sleep to dodge the race.",
1600.0,
1900,
280,
175.0,
0.55,
0.0000,
2,
),
// local-llm/opus is intentionally missing → failed cell.
(
"local-llm",
"sonnet",
"Used a channel to sync the workers.",
900.0,
1700,
240,
266.0,
0.69,
0.0000,
0,
),
(
"local-llm",
"mistral",
"Made the field volatile-ish via Arc.",
700.0,
1700,
200,
285.0,
0.58,
0.0000,
0,
),
];
for &(agent, model, out, lat, tin, tout, tps, score, cost, mcp) in cells {
m = m.with_cell(
agent,
model,
ModelAnswer {
output: out.to_string(),
latency_ms: lat,
tokens_in: tin,
tokens_out: tout,
tokens_per_s: tps,
score,
cost_usd: cost,
mcp_tool_calls: mcp,
},
);
}
m
}
// ─── column helpers ──────────────────────────────────────────────────────
fn col_str<'a>(b: &'a RecordBatch, idx: usize) -> Result<&'a StringArray> {
b.column(idx)
.as_any()
.downcast_ref::<StringArray>()
.ok_or_else(|| anyhow!("agent_model_runs col {idx} is not StringArray"))
}
fn col_f64<'a>(b: &'a RecordBatch, idx: usize) -> Result<&'a Float64Array> {
b.column(idx)
.as_any()
.downcast_ref::<Float64Array>()
.ok_or_else(|| anyhow!("agent_model_runs col {idx} is not Float64Array"))
}
fn col_i64<'a>(b: &'a RecordBatch, idx: usize) -> Result<&'a Int64Array> {
b.column(idx)
.as_any()
.downcast_ref::<Int64Array>()
.ok_or_else(|| anyhow!("agent_model_runs col {idx} is not Int64Array"))
}
fn col_bool<'a>(b: &'a RecordBatch, idx: usize) -> Result<&'a BooleanArray> {
b.column(idx)
.as_any()
.downcast_ref::<BooleanArray>()
.ok_or_else(|| anyhow!("agent_model_runs col {idx} is not BooleanArray"))
}
fn col_ts<'a>(b: &'a RecordBatch, idx: usize) -> Result<&'a TimestampMicrosecondArray> {
b.column(idx)
.as_any()
.downcast_ref::<TimestampMicrosecondArray>()
.ok_or_else(|| anyhow!("agent_model_runs col {idx} is not TimestampMicrosecondArray"))
}
#[cfg(test)]
mod tests {
use super::*;
fn ans(out: &str, lat: f64, tin: i64, tout: i64, tps: f64, score: f64) -> ModelAnswer {
ModelAnswer::basic(out, lat, tin, tout, tps, score)
}
/// Priced answer (cost + mcp calls) for the matrix-economics tests.
fn ans_priced(
out: &str,
lat: f64,
tout: i64,
tps: f64,
score: f64,
cost: f64,
mcp: i64,
) -> ModelAnswer {
ModelAnswer {
output: out.to_string(),
latency_ms: lat,
tokens_in: 10,
tokens_out: tout,
tokens_per_s: tps,
score,
cost_usd: cost,
mcp_tool_calls: mcp,
}
}
fn sample_rows() -> Vec<AgentModelRunRow> {
vec![
AgentModelRunRow {
run_id: "runZ".into(),
ts_micros: 2_000,
agent: "claude".into(),
model: "fast-model".into(),
prompt_id: "p1".into(),
prompt: "2+2?".into(),
output: "4".into(),
latency_ms: 100.0,
tokens_in: 10,
tokens_out: 20,
tokens_per_s: 200.0,
score: 1.0,
ok: true,
error: None,
cost_usd: 0.0,
mcp_tool_calls: 0,
},
AgentModelRunRow {
run_id: "runZ".into(),
ts_micros: 2_000,
agent: "kilo".into(),
model: "slow-model".into(),
prompt_id: "p1".into(),
prompt: "2+2?".into(),
output: "four".into(),
latency_ms: 500.0,
tokens_in: 10,
tokens_out: 30,
tokens_per_s: 60.0,
score: 0.5,
ok: true,
error: None,
cost_usd: 0.02,
mcp_tool_calls: 3,
},
]
}
#[test]
fn leaderboard_outcome_empty_is_red_not_silently_green() {
let o = leaderboard_outcome("", &[], false);
assert_eq!(o.command, "bakeoff leaderboard");
assert!(!o.is_sannr(), "empty leaderboard must be RED (RAGNARÖK)");
assert!(o.human.contains("no bake-off runs"));
// a missing named reference is also red, and names the scope.
let o2 = leaderboard_outcome("runMISSING", &[], false);
assert!(!o2.is_sannr());
assert!(
o2.human.contains("runMISSING"),
"scope named in human: {}",
o2.human
);
}
#[test]
fn leaderboard_outcome_real_rows_are_sannr_with_data_and_render() {
let rows = sample_rows();
let o = leaderboard_outcome("runZ", &rows, false);
assert!(
o.is_sannr(),
"real bake-off rows are a true (sannr) outcome"
);
assert_eq!(o.command, "bakeoff leaderboard");
assert_eq!(o.data["reference"], serde_json::json!("runZ"));
let arr = o.data["rows"].as_array().unwrap();
assert_eq!(arr.len(), 2, "both cells surfaced as data");
assert_eq!(arr[0]["model"], serde_json::json!("fast-model"));
assert_eq!(arr[0]["agent"], serde_json::json!("claude"));
// flat leaderboard rendering by default; matrix flag flips to the grid.
assert!(
o.human.contains("agent/model"),
"flat board header: {}",
o.human
);
let grid = leaderboard_outcome("runZ", &rows, true);
assert!(grid.is_sannr());
assert!(
grid.human.contains("agent \\ model"),
"matrix grid header: {}",
grid.human
);
}
#[test]
fn run_bakeoff_builds_rows_and_records_failures() {
// Two models answer; one is absent from the mock → red row.
let caller = MockCaller::new()
.with("fast-model", ans("4", 100.0, 10, 20, 200.0, 1.0))
.with("slow-model", ans("four", 500.0, 10, 30, 0.0, 0.5));
let models = vec![
"fast-model".to_string(),
"slow-model".to_string(),
"missing-model".to_string(),
];
let rows = run_bakeoff(&caller, "runX", 1_000, "p1", "2+2?", &models);
assert_eq!(rows.len(), 3);
let fast = rows.iter().find(|r| r.model == "fast-model").unwrap();
assert_eq!(fast.output, "4");
assert!(fast.ok);
assert_eq!(fast.tokens_per_s, 200.0); // backend-supplied tok/s kept
assert_eq!(fast.tokens_out, 20);
// slow-model supplied tps=0 → derived: 30 tokens / 0.5s = 60 tok/s.
let slow = rows.iter().find(|r| r.model == "slow-model").unwrap();
assert!(slow.ok);
assert!(
(slow.tokens_per_s - 60.0).abs() < 1e-6,
"derived tok/s: {}",
slow.tokens_per_s
);
// missing-model → red, error recorded, no output.
let missing = rows.iter().find(|r| r.model == "missing-model").unwrap();
assert!(!missing.ok);
assert!(missing.error.as_ref().unwrap().contains("missing-model"));
assert_eq!(missing.output, "");
}
#[test]
fn leaderboard_ranks_ok_then_score_then_throughput() {
let rows = run_bakeoff(
&MockCaller::new()
.with("a", ans("x", 100.0, 5, 10, 100.0, 0.9))
.with("b", ans("x", 100.0, 5, 10, 300.0, 0.9)) // same score, faster
.with("c", ans("x", 100.0, 5, 10, 500.0, 0.5)), // higher tps but lower score
"runR",
1,
"p",
"q",
&["a".into(), "b".into(), "c".into(), "z".into()],
);
// z is missing → red, must sink to the bottom.
let board = leaderboard(&rows);
assert_eq!(board.len(), 4);
// score 0.9 beats 0.5; within 0.9, higher tps (b=300) beats a=100.
assert_eq!(board[0].model, "b");
assert_eq!(board[1].model, "a");
assert_eq!(board[2].model, "c");
// failed model last, marked not-ok.
assert_eq!(board[3].model, "z");
assert!(!board[3].ok);
assert_eq!(board[0].rank, 1);
assert_eq!(board[3].rank, 4);
}
#[test]
fn append_query_round_trip_groups_and_ranks() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
// A full bake-off, mock-driven (no live LLM), persisted + read back.
let caller = MockCaller::new()
.with("mistral", ans("Paris", 80.0, 12, 4, 50.0, 1.0))
.with("llama3", ans("paris", 200.0, 12, 4, 20.0, 0.8));
let run_id = "bake-1";
let rows = run_bakeoff(
&caller,
run_id,
12_345,
"capital-fr",
"Capital of France?",
&["mistral".into(), "llama3".into()],
);
wh.block_on(append_agent_model_runs(&wh, &rows)).unwrap();
// Read the run back; values round-trip exactly.
let got = wh
.block_on(query_agent_model_runs(
&wh,
&BakeoffSelector::Run(run_id.into()),
))
.unwrap();
assert_eq!(got.len(), 2);
let mistral = got.iter().find(|r| r.model == "mistral").unwrap();
assert_eq!(mistral.output, "Paris");
assert_eq!(mistral.tokens_in, 12);
assert_eq!(mistral.tokens_out, 4);
assert!((mistral.tokens_per_s - 50.0).abs() < 1e-6);
assert_eq!(mistral.score, 1.0);
assert!(mistral.ok);
assert_eq!(mistral.error, None);
// Leaderboard from the round-tripped rows: mistral wins (higher score).
let board = leaderboard(&got);
assert_eq!(board[0].model, "mistral");
assert_eq!(board[0].rank, 1);
assert_eq!(board[1].model, "llama3");
// Model scope crosses runs but filters by model.
let only_mistral = wh
.block_on(query_agent_model_runs(
&wh,
&BakeoffSelector::Model("mistral".into()),
))
.unwrap();
assert_eq!(only_mistral.len(), 1);
assert_eq!(only_mistral[0].model, "mistral");
// JSON is well-formed and carries the leaderboard values.
let json = rows_to_json(&got);
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed.as_array().unwrap().len(), 2);
// Human leaderboard names the winning model.
let board_txt = render_leaderboard(&got);
assert!(board_txt.contains("mistral"), "leaderboard: {board_txt}");
}
#[test]
fn matrix_crosses_agents_and_models_and_records_a_failed_cell() {
// 2 agents × 2 models = 4 cells, one missing → red cell.
let caller = MockCaller::new()
.with_cell(
"claude",
"opus",
ans_priced("A", 4000.0, 500, 125.0, 0.95, 0.08, 6),
)
.with_cell(
"claude",
"mistral",
ans_priced("B", 1200.0, 300, 250.0, 0.70, 0.0, 2),
)
.with_cell(
"local",
"mistral",
ans_priced("C", 700.0, 200, 285.0, 0.60, 0.0, 0),
);
// local/opus is absent → failed cell.
let agents = vec!["claude".to_string(), "local".to_string()];
let models = vec!["opus".to_string(), "mistral".to_string()];
let rows = run_bakeoff_matrix(&caller, "m1", 10, "task-x", "do x", &agents, &models);
assert_eq!(rows.len(), 4, "full 2x2 cross-product");
// Every row carries its agent axis.
assert!(
rows.iter()
.all(|r| r.agent == "claude" || r.agent == "local")
);
let claude_opus = rows
.iter()
.find(|r| r.agent == "claude" && r.model == "opus")
.unwrap();
assert!(claude_opus.ok);
assert_eq!(claude_opus.score, 0.95);
assert!((claude_opus.cost_usd - 0.08).abs() < 1e-9);
assert_eq!(claude_opus.mcp_tool_calls, 6);
let local_opus = rows
.iter()
.find(|r| r.agent == "local" && r.model == "opus")
.unwrap();
assert!(!local_opus.ok, "local/opus must be the failed cell");
assert!(local_opus.error.as_ref().unwrap().contains("local/opus"));
}
#[test]
fn matrix_grid_ranks_cells_and_picks_winner() {
let caller = MockCaller::new()
// claude/opus: top score, slow, costly → still the winner (score first).
.with_cell(
"claude",
"opus",
ans_priced("A", 4000.0, 500, 125.0, 0.95, 0.08, 6),
)
// kilo/sonnet: lower score but fast + cheap → second.
.with_cell(
"kilo",
"sonnet",
ans_priced("B", 2000.0, 400, 200.0, 0.88, 0.02, 4),
)
// local/mistral: fastest but weakest → third.
.with_cell(
"local",
"mistral",
ans_priced("C", 700.0, 200, 285.0, 0.58, 0.0, 0),
);
let agents = vec!["claude".into(), "kilo".into(), "local".into()];
let models = vec!["opus".into(), "sonnet".into(), "mistral".into()];
// Only the 3 registered cells answer; the other 6 fail (red).
let rows = run_bakeoff_matrix(&caller, "m2", 20, "task-y", "do y", &agents, &models);
assert_eq!(rows.len(), 9, "3x3 cross-product");
let grid = matrix_grid(&rows);
assert_eq!(grid.agents, vec!["claude", "kilo", "local"]);
assert_eq!(grid.models, vec!["mistral", "opus", "sonnet"]);
assert_eq!(grid.cells.len(), 9, "one cell per coordinate");
// The winner is the highest-score OK cell, by score then tok/s.
let w = grid.winner().expect("a winner");
assert_eq!((w.agent.as_str(), w.model.as_str()), ("claude", "opus"));
assert_eq!(w.rank, 1);
// Ranks: claude/opus (0.95) > kilo/sonnet (0.88) > local/mistral (0.58)
// > the 6 failed cells.
let co = grid.cell("claude", "opus").unwrap();
let ks = grid.cell("kilo", "sonnet").unwrap();
let lm = grid.cell("local", "mistral").unwrap();
assert_eq!(co.rank, 1);
assert_eq!(ks.rank, 2);
assert_eq!(lm.rank, 3);
assert!(co.ok && ks.ok && lm.ok);
// A failed cell sits below all three OK cells.
let failed = grid.cell("local", "opus").unwrap();
assert!(!failed.ok);
assert!(failed.rank > 3, "failed cell ranks after every ok cell");
// The flat leaderboard agrees with the grid winner.
let board = leaderboard(&rows);
assert_eq!(board[0].agent, "claude");
assert_eq!(board[0].model, "opus");
assert_eq!(board[0].cell_label(), "claude/opus");
// The matrix renders the winner.
let txt = render_matrix(&rows);
assert!(txt.contains("winner: claude/opus"), "matrix:\n{txt}");
}
#[test]
fn demo_matrix_is_populated_and_round_trips() {
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
let rows = demo_matrix_rows("demo-run", 99);
// 3 agents × 3 models = 9 cells; one is the failed local-llm/opus.
assert_eq!(rows.len(), 9);
assert_eq!(
rows.iter().filter(|r| !r.ok).count(),
1,
"exactly one red cell"
);
wh.block_on(append_agent_model_runs(&wh, &rows)).unwrap();
let got = wh
.block_on(query_agent_model_runs(
&wh,
&BakeoffSelector::Run("demo-run".into()),
))
.unwrap();
assert_eq!(got.len(), 9);
let grid = matrix_grid(&got);
assert_eq!(grid.agents, vec!["claude", "kilo", "local-llm"]);
assert_eq!(grid.models, vec!["mistral", "opus", "sonnet"]);
// claude/opus is the highest-scoring cell → winner.
let w = grid.winner().unwrap();
assert_eq!((w.agent.as_str(), w.model.as_str()), ("claude", "opus"));
// Cost + mcp survived the round trip.
let co = grid.cell("claude", "opus").unwrap();
assert!(co.cost_usd > 0.0, "hosted cell carries a cost");
assert!(co.mcp_tool_calls > 0, "hosted cell made mcp tool calls");
// A free local cell costs nothing.
let lm = grid.cell("local-llm", "mistral").unwrap();
assert_eq!(lm.cost_usd, 0.0);
// The Agent selector filters by agent across the run.
let only_claude = wh
.block_on(query_agent_model_runs(
&wh,
&BakeoffSelector::Agent("claude".into()),
))
.unwrap();
assert_eq!(only_claude.len(), 3, "claude × 3 models");
assert!(only_claude.iter().all(|r| r.agent == "claude"));
}
/// The pre-matrix (13-column) `agent_model_runs` schema, exactly as an
/// older nornir binary created it on the server — before `agent` /
/// `cost_usd` / `mcp_tool_calls` (fields 14-16) were added. Used by the
/// schema-evolution regression test to reproduce a stale server table.
fn legacy_13_col_schema() -> iceberg::spec::Schema {
use iceberg::spec::{NestedField, PrimitiveType, Schema, Type};
Schema::builder()
.with_schema_id(0)
.with_fields(vec![
Arc::new(NestedField::required(
1,
"run_id",
Type::Primitive(PrimitiveType::String),
)),
Arc::new(NestedField::required(
2,
"ts_micros",
Type::Primitive(PrimitiveType::Timestamptz),
)),
Arc::new(NestedField::required(
3,
"model",
Type::Primitive(PrimitiveType::String),
)),
Arc::new(NestedField::required(
4,
"prompt_id",
Type::Primitive(PrimitiveType::String),
)),
Arc::new(NestedField::required(
5,
"prompt",
Type::Primitive(PrimitiveType::String),
)),
Arc::new(NestedField::required(
6,
"output",
Type::Primitive(PrimitiveType::String),
)),
Arc::new(NestedField::required(
7,
"latency_ms",
Type::Primitive(PrimitiveType::Double),
)),
Arc::new(NestedField::required(
8,
"tokens_in",
Type::Primitive(PrimitiveType::Long),
)),
Arc::new(NestedField::required(
9,
"tokens_out",
Type::Primitive(PrimitiveType::Long),
)),
Arc::new(NestedField::required(
10,
"tokens_per_s",
Type::Primitive(PrimitiveType::Double),
)),
Arc::new(NestedField::required(
11,
"score",
Type::Primitive(PrimitiveType::Double),
)),
Arc::new(NestedField::required(
12,
"ok",
Type::Primitive(PrimitiveType::Boolean),
)),
Arc::new(NestedField::optional(
13,
"error",
Type::Primitive(PrimitiveType::String),
)),
])
.build()
.unwrap()
}
/// REGRESSION (0.4.22/0.4.23 `Telemetry.SubmitBakeoff` bug): the server's
/// `agent_model_runs` table is a stale 13-column schema (created before the
/// matrix axis + economics columns), but a current row produces 16 columns —
/// the append failed with `number of columns(16) must match number of
/// fields(13)`. The fix evolves the table forward (Iceberg add-column) on
/// append. This test reproduces the stale table, appends a full 16-column
/// row, and asserts the table evolved AND the new-column values read back.
#[test]
fn stale_13col_table_evolves_to_16_on_append() {
use iceberg::Catalog;
use iceberg::TableCreation;
use iceberg::spec::{PartitionSpec, Transform};
let dir = tempfile::tempdir().unwrap();
let wh = IcebergWarehouse::open(dir.path()).unwrap();
let ident = wh.table_ident(TABLE_AGENT_MODEL_RUNS);
// Replace the freshly-created canonical (16-col) table with the stale
// 13-col one a pre-matrix server would hold — same `run_id` partitioning.
wh.block_on(async {
let cat = wh.catalog();
cat.drop_table(&ident).await.unwrap();
let schema = legacy_13_col_schema();
let spec = PartitionSpec::builder(Arc::new(schema.clone()))
.add_partition_field("run_id", "run_id", Transform::Identity)
.unwrap()
.build()
.unwrap()
.into_unbound();
let creation = TableCreation::builder()
.name(ident.name().to_string())
.schema(schema)
.partition_spec(spec)
.build();
cat.create_table(ident.namespace(), creation).await.unwrap();
// Sanity: the table really is 13 columns before we append.
let t = cat.load_table(&ident).await.unwrap();
assert_eq!(t.metadata().current_schema().as_struct().fields().len(), 13);
});
// Append a full matrix row (16 columns: agent + cost_usd + mcp_tool_calls).
let caller = MockCaller::new().with_cell(
"claude",
"opus",
ans_priced("Fixed it.", 4000.0, 500, 125.0, 0.95, 0.08, 6),
);
let rows = run_bakeoff_matrix(
&caller,
"evolve-run",
777,
"task-z",
"do z",
&["claude".to_string()],
&["opus".to_string()],
);
assert_eq!(rows.len(), 1);
// The bug: this append used to fail with the column-count mismatch.
wh.block_on(append_agent_model_runs(&wh, &rows)).unwrap();
// The table evolved to the full 16-column schema.
wh.block_on(async {
let t = wh.catalog().load_table(&ident).await.unwrap();
let names: Vec<&str> = t
.metadata()
.current_schema()
.as_struct()
.fields()
.iter()
.map(|f| f.name.as_str())
.collect();
assert_eq!(names.len(), 16, "schema evolved 13 → 16 columns");
assert!(names.contains(&"agent"));
assert!(names.contains(&"cost_usd"));
assert!(names.contains(&"mcp_tool_calls"));
});
// The row reads back with the NEW columns' values intact.
let got = wh
.block_on(query_agent_model_runs(
&wh,
&BakeoffSelector::Run("evolve-run".into()),
))
.unwrap();
assert_eq!(got.len(), 1);
let r = &got[0];
assert_eq!(r.agent, "claude");
assert_eq!(r.model, "opus");
assert!(
(r.cost_usd - 0.08).abs() < 1e-9,
"evolved cost_usd reads back"
);
assert_eq!(r.mcp_tool_calls, 6, "evolved mcp_tool_calls reads back");
assert!(r.ok);
// Idempotent: a second append against the now-current schema is a no-op
// migration (no panic, row count grows to 2).
let rows2 = run_bakeoff_matrix(
&caller,
"evolve-run-2",
888,
"task-z",
"do z",
&["claude".to_string()],
&["opus".to_string()],
);
wh.block_on(append_agent_model_runs(&wh, &rows2)).unwrap();
let all = wh
.block_on(query_agent_model_runs(&wh, &BakeoffSelector::All))
.unwrap();
assert_eq!(all.len(), 2);
}
#[test]
fn legacy_single_axis_rows_default_agent_unset() {
// A model-only bake-off (no agents) lands every row under AGENT_UNSET.
let caller = MockCaller::new().with("m", ans("x", 100.0, 5, 10, 100.0, 0.9));
let rows = run_bakeoff(&caller, "r", 1, "p", "q", &["m".into()]);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].agent, AGENT_UNSET);
assert_eq!(rows[0].cost_usd, 0.0);
assert_eq!(rows[0].mcp_tool_calls, 0);
}
/// The codegen judge wired into the bake-off: a mock matrix whose cells emit
/// real code answers (a GOOD body, a NON-COMPILING body, and a failed cell),
/// scored through a real `CodegenJudge` against a scratch crate. Asserts the
/// re-scored rows carry the judge's verdicts (NOT the caller's 0.0): the good
/// cell lands 1.0, the non-compiling cell 0.0, and the failed cell stays
/// 0.0/red. This is the "mock caller + codegen spec → real (non-0.0) scores"
/// LAW-test.
#[test]
fn matrix_scored_by_codegen_judge_lands_real_scores() {
use super::super::codegen_judge::{CodegenJudge, CodegenSpec};
// A scratch crate whose lib.rs the answer overwrites, with two baked-in
// acceptance tests (one always-pass, one asserting add(2,2)==4).
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::write(
root.join("Cargo.toml"),
"[package]\nname=\"jcrate\"\nversion=\"0.0.0\"\nedition=\"2021\"\n\n[lib]\npath=\"src/lib.rs\"\n",
)
.unwrap();
std::fs::create_dir_all(root.join("src")).unwrap();
std::fs::write(
root.join("src/lib.rs"),
"pub fn add(_a:i32,_b:i32)->i32{0}\n",
)
.unwrap();
let accept = "\n#[cfg(test)]\nmod t{use super::*;#[test]fn ok(){assert!(true);}#[test]fn tgt(){assert_eq!(add(2,2),4);}}\n";
let good = format!("pub fn add(a:i32,b:i32)->i32{{a+b}}{accept}");
let bad = format!("pub fn add(_a:i32,_b:i32)->i32{{\"x\"}}{accept}");
// 2 agents × 1 model: claude/m answers GOOD, local/m answers NON-COMPILING.
let caller = MockCaller::new()
.with_cell("claude", "m", ans(&good, 1000.0, 10, 50, 50.0, 0.0))
.with_cell("local", "m", ans(&bad, 800.0, 10, 40, 50.0, 0.0));
// "kilo" is absent → a red failed cell (no answer to judge).
let agents = vec![
"claude".to_string(),
"kilo".to_string(),
"local".to_string(),
];
let models = vec!["m".to_string()];
let mut rows = run_bakeoff_matrix(&caller, "jrun", 1, "task", "do it", &agents, &models);
assert_eq!(rows.len(), 3);
// Before judging, every score is the caller's 0.0.
assert!(
rows.iter().all(|r| r.score == 0.0),
"caller left scores at 0.0"
);
let judge = CodegenJudge::new(CodegenSpec::write_file(root.to_path_buf(), "src/lib.rs"));
let reports = score_rows_with_judge(&mut rows, &judge);
// Two OK cells got judged; the red cell was skipped.
assert_eq!(reports.len(), 2, "only the two OK cells are judged");
let claude = rows.iter().find(|r| r.agent == "claude").unwrap();
assert!(claude.ok);
assert_eq!(claude.score, 1.0, "GOOD answer → real 1.0 score (was 0.0)");
let local = rows.iter().find(|r| r.agent == "local").unwrap();
assert!(local.ok, "the cell answered (ok); it just doesn't compile");
assert_eq!(local.score, 0.0, "NON-COMPILING answer → 0.0");
let kilo = rows.iter().find(|r| r.agent == "kilo").unwrap();
assert!(!kilo.ok, "kilo never answered → red");
assert_eq!(kilo.score, 0.0);
// The leaderboard now ranks the GOOD cell first on its real score (not
// speed): claude/m (1.0) beats local/m (0.0).
let board = leaderboard(&rows);
assert_eq!(
(board[0].agent.as_str(), board[0].model.as_str()),
("claude", "m")
);
assert_eq!(board[0].score, 1.0);
}
}