src/vector/fallback.rs
//! Say out loud WHY embedding is about to run on the CPU.
//!
//! A silent CPU fallback is a **defect**, not a graceful degradation. On this
//! constellation's RTX 4090 the CUDA embedder was measured ~67× faster than the
//! pure-Rust tract path; falling back without a word means an operator reads a
//! perfectly green log — `embedding 19 member(s) with tract-onnx (CPU, pure
//! Rust)` — while `ORT_DYLIB_PATH` sits unused in the unit file and the GPU idles.
//! That exact line ran on oden's FAT `dwarves-server` for hours on 2026-08-04 at
//! 3–4 chunk/s, and the only visible symptom was the duplicates lens returning 0
//! clusters. Nothing in the log said the word "GPU".
//!
//! So: when the box (or the operator's environment) clearly ASKED for a GPU and
//! we ended up on CPU anyway, [`diagnose`] returns a loud, named reason. When
//! nothing suggests a GPU was ever intended — a genuine CPU server — it returns
//! `None`, because tract IS the right answer there and silence is honest.
//!
//! ## Shape (LAW: inject-assert)
//!
//! [`diagnose`] is a **pure function over [`FallbackFacts`]**, deliberately the
//! same shape as [`super::select::decide`] over its `Probe`. A test injects the
//! facts — including `ort_compiled: false`, the cfg case that by definition
//! cannot be reached from within a build that has ort compiled in — and asserts
//! the message. That is the only way the tract-only branch is ever provable from
//! a default (both-backends) test build.
use std::fmt::Write as _;
/// Stable cause tags. Each verdict carries EXACTLY one, so a log grep — and the
/// tests below — can tell the four causes apart without matching prose. A single
/// shared template that merely mentions every keyword would still be the silent
/// degradation wearing a warning label, so the tags are asserted mutually
/// exclusive: naming one means NOT naming the others.
pub const NO_ORT_FEATURE: &str = "no-embed-ort-feature";
/// `libonnxruntime.so` absent or not loadable (often: unreadable by the service user).
pub const NO_DYLIB: &str = "onnxruntime-not-loadable";
/// onnxruntime loaded but no CUDA/ROCm execution provider is ready.
pub const NO_PROVIDER: &str = "no-gpu-execution-provider";
/// Every probe green yet a CPU backend was selected — a bug in the selector.
pub const SELECTOR_BUG: &str = "selector-bug";
/// Every probe green, the ort GPU session was attempted, and the execution
/// provider REFUSED to register. Emitted by [`super::select::load_backend`],
/// which is the only place that can observe it — usually an onnxruntime built
/// against a different CUDA major than the installed runtime.
pub const EP_REGISTRATION_FAILED: &str = "gpu-ep-registration-failed";
/// All five tags, for exclusivity assertions.
pub const ALL_CAUSE_TAGS: [&str; 5] = [
NO_ORT_FEATURE,
NO_DYLIB,
NO_PROVIDER,
SELECTOR_BUG,
EP_REGISTRATION_FAILED,
];
/// The runtime facts a CPU-fallback verdict is drawn from. [`live`] fills this
/// from the real probes; a test injects it directly.
#[derive(Debug, Clone, Default)]
pub struct FallbackFacts {
/// The backend actually selected runs on a GPU (ort CUDA or ROCm).
pub selected_is_gpu: bool,
/// The `embed-ort` feature is compiled into THIS binary. When false there is
/// no GPU backend at all and no environment variable can change that.
pub ort_compiled: bool,
/// A usable `libonnxruntime.so` was found and loaded.
pub onnxruntime: bool,
/// NVIDIA driver + CUDA runtime + cuDNN all present.
pub cuda_ready: bool,
/// AMD driver + HIP runtime + MIOpen all present.
pub rocm_ready: bool,
/// Evidence that a GPU was INTENDED on this box, human-readable. `None` ⇒
/// nothing on the box asks for one, so a CPU embedder is not a fallback.
pub gpu_intent: Vec<String>,
/// **The CPU is where the operator ASKED to be** — an honoured explicit
/// `$NORNIR_EMBED_BACKEND` pin to a non-GPU backend, or a static-table model that
/// has no graph for any GPU to run.
///
/// Added 2026-08-14, because `Backend::OrtCpu` made the old heuristic wrong.
/// [`live_gpu_intent`] treats `ORT_DYLIB_PATH` as "the loudest declaration there
/// is" that a GPU was wanted — and it WAS, until setting that variable became the
/// way to ask for ort on the CPU. Without this fact, deliberately pinning the
/// measured-4.18×-FASTER backend printed "⚠ GPU EMBEDDING SKIPPED — running on the
/// CPU embedder, which was measured ~67× slower", which is false twice over: it is
/// not the tract embedder, and it is not slower. Observed in production on `dwarf`
/// the first time ort-CPU served a real populate.
pub deliberate_cpu: bool,
/// The backend that was actually selected, for the message to name honestly rather
/// than assuming "CPU" means tract.
pub selected_label: Option<String>,
/// Optional detailed preflight report to append (the `vector doctor` text).
pub detail: Option<String>,
}
/// Pure verdict: `Some(loud reason)` when this box wanted a GPU and did not get
/// one, `None` when the CPU backend is the honest answer.
///
/// The message always names (a) that the GPU was skipped, (b) the concrete
/// evidence the operator expected one, (c) WHICH check failed, and (d) what to
/// do about it. A reason that does not name the failing check is the same silent
/// degradation wearing a warning label.
pub fn diagnose(f: &FallbackFacts) -> Option<String> {
if f.selected_is_gpu {
return None;
}
if f.deliberate_cpu {
// The operator PINNED a non-GPU backend (or selected a model with no graph).
// That is not a fallback, it is the configuration, and warning about it is the
// same crime in the other direction: crying degradation over a deliberate — and
// on this hardware, measurably faster — choice. Silence is honest here for the
// identical reason it is honest for a GPU-less box below.
return None;
}
if f.gpu_intent.is_empty() {
// No GPU asked for anywhere: tract CPU is correct, say nothing.
return None;
}
let cause: String = if !f.ort_compiled {
format!(
"[{NO_ORT_FEATURE}] this binary was built WITHOUT the `embed-ort` feature — it \
carries NO GPU backend at all, so no environment variable can move it off the \
CPU. Rebuild with `nornir/embed-ort` (for the fat forge: `cargo build -p dwarves \
--bin dwarves-server --features embed-all,read-fat --profile release`, whose \
`embed-nornir` now forwards it)."
)
} else if !f.onnxruntime {
format!(
"[{NO_DYLIB}] no usable `libonnxruntime.so` could be dlopen'ed (it must exist, be \
readable BY THE SERVICE USER, and export `OrtGetApiBase`). Set `ORT_DYLIB_PATH` \
to a real one or drop it in /opt/nornir/cuda."
)
} else if !f.cuda_ready && !f.rocm_ready {
format!(
"[{NO_PROVIDER}] onnxruntime loaded but NO GPU execution provider is ready — the \
NVIDIA driver (`libcuda.so.1`), the CUDA runtime (`libcudart.so.12`) and cuDNN 9 \
must ALL resolve, or the AMD ROCm equivalents. See the preflight below."
)
} else {
// Every probe says ready yet a GPU backend was not selected: that is a
// selector bug, and it must be reported as one rather than shrugged off.
format!(
"[{SELECTOR_BUG}] every GPU probe reports READY yet a CPU backend was selected — \
this is a selector bug, not a configuration problem. Report it."
)
};
let mut s = String::new();
let _ = write!(
s,
"⚠ GPU EMBEDDING SKIPPED — running on the CPU embedder, which was measured \
~67× slower than the CUDA path on this constellation's RTX 4090.\n \
a GPU was expected here: {}\n cause: {}",
f.gpu_intent.join("; "),
cause
);
if let Some(d) = &f.detail {
let _ = write!(s, "\n{}", d.trim_end());
}
Some(s)
}
/// Evidence, from the live environment, that somebody meant for this box to
/// embed on a GPU. Empty ⇒ a CPU embedder is not a fallback, it is the plan.
///
/// Deliberately environment- and device-based rather than "is there a GPU": the
/// question is what the OPERATOR declared, and `ORT_DYLIB_PATH` in a unit file
/// is the loudest declaration there is.
pub fn live_gpu_intent() -> Vec<String> {
let mut out = Vec::new();
for key in ["ORT_DYLIB_PATH", "NORNIR_CUDA_LIBS"] {
if let Some(v) = std::env::var_os(key) {
if !v.is_empty() {
out.push(format!(
"{key}={} is set in this process's environment",
v.to_string_lossy()
));
}
}
}
if std::path::Path::new("/dev/nvidia0").exists() {
out.push("an NVIDIA device node (/dev/nvidia0) is present on this host".to_string());
}
if std::path::Path::new("/dev/kfd").exists() {
out.push("an AMD compute device node (/dev/kfd) is present on this host".to_string());
}
out
}
/// Fill [`FallbackFacts`] from the real probes on this machine and diagnose.
/// Cheap (the probes are cached); safe on any box, GPU or not.
pub fn live() -> FallbackFacts {
let gpu_intent = live_gpu_intent();
// Both backends compiled → the runtime selector exists and knows everything.
#[cfg(all(feature = "embed-tract", feature = "embed-ort"))]
{
let probe = super::select::live_probe();
// ⚠ ONE AUTHORITY. This used to call `select::decide(&probe)` directly, which is
// only the HARDWARE half of the policy — it ignored the model kind and any
// `$NORNIR_EMBED_BACKEND` pin, so the diagnosis described a backend that was not
// the one loading. `chosen_backend()` is the same function `select::load` uses,
// so the facts now describe what actually ran.
let backend = super::select::chosen_backend();
let deliberate_cpu = !backend.is_gpu() && super::select::cpu_was_requested();
let detail = if gpu_intent.is_empty() || deliberate_cpu {
None
} else {
Some(super::cuda::preflight().1)
};
FallbackFacts {
selected_is_gpu: backend.is_gpu(),
ort_compiled: true,
onnxruntime: probe.onnxruntime,
cuda_ready: probe.cuda_ready,
rocm_ready: probe.rocm_ready,
gpu_intent,
deliberate_cpu,
selected_label: Some(backend.label().to_string()),
detail,
}
}
// ort only: no selector; ort itself decides EP at session build.
#[cfg(all(feature = "embed-ort", not(feature = "embed-tract")))]
{
let (ready, report) = super::cuda::preflight();
FallbackFacts {
selected_is_gpu: ready,
ort_compiled: true,
onnxruntime: super::cuda::onnxruntime_dylib().is_some(),
cuda_ready: ready,
rocm_ready: false,
gpu_intent,
// No `select` module in this build, so no pin mechanism to honour; a static
// model is still expressible though, and it is not a fallback either.
deliberate_cpu: !ready && super::embed_registry::selected_kind_is_static(),
selected_label: None,
detail: Some(report),
}
}
// tract only: THE case that bit oden. There is no GPU path in this binary.
#[cfg(all(feature = "embed-tract", not(feature = "embed-ort")))]
{
FallbackFacts {
selected_is_gpu: false,
ort_compiled: false,
onnxruntime: false,
cuda_ready: false,
rocm_ready: false,
gpu_intent,
// Same reasoning as the ort-only arm: a static-table model asked for no
// graph, so running without one is the configuration, not a degradation.
deliberate_cpu: super::embed_registry::selected_kind_is_static(),
selected_label: None,
detail: None,
}
}
}
/// The live verdict: `Some(loud reason)` iff this box wanted a GPU and is about
/// to embed on the CPU anyway. Call it next to any "embedding with <backend>"
/// announcement.
pub fn live_reason() -> Option<String> {
// the forward passes run on a named worker ($NORNIR_EMBED_REMOTE): what this
// box's own GPU stack lacks is no fallback of the embed
#[cfg(feature = "wire-client")]
if super::remote::configured().is_some() {
return None;
}
diagnose(&live())
}
#[cfg(test)]
mod tests {
use super::*;
fn intent() -> Vec<String> {
vec!["ORT_DYLIB_PATH=/opt/nornir/cuda/libonnxruntime.so is set".to_string()]
}
/// A DELIBERATE non-GPU backend is SILENT, even with loud GPU intent on the box.
///
/// This is the production bug from 2026-08-14, as a test. `ORT_DYLIB_PATH` is both
/// the evidence `live_gpu_intent` reads as "a GPU was wanted" AND the variable you
/// must set to enable ort on the CPU — so the first real ort-CPU populate printed
/// "⚠ GPU EMBEDDING SKIPPED — running on the CPU embedder, which was measured ~67×
/// slower", which was wrong twice: not the tract embedder, and 4.18× FASTER than it.
///
/// Note what is held constant: intent is present, no GPU is ready, and every cause
/// tag would otherwise have something to say. Only `deliberate_cpu` differs — so
/// this cannot pass by accident.
#[test]
fn a_pinned_cpu_backend_is_not_a_fallback() {
let base = FallbackFacts {
selected_is_gpu: false,
ort_compiled: true,
onnxruntime: true,
cuda_ready: false,
rocm_ready: false,
gpu_intent: intent(),
deliberate_cpu: false,
selected_label: Some("ort (ONNX Runtime, CPU / MLAS)".into()),
detail: None,
};
// Without the flag this box looks exactly like a failed GPU box, and SHOULD warn.
let warned = diagnose(&base).expect("un-pinned CPU with GPU intent must warn");
assert!(warned.contains(NO_PROVIDER), "{warned}");
// With it, the same facts are a configuration, and the correct output is silence.
let pinned = FallbackFacts {
deliberate_cpu: true,
..base
};
assert_eq!(
diagnose(&pinned),
None,
"an explicitly pinned non-GPU backend must NOT be reported as a degradation"
);
}
/// Assert `msg` carries EXACTLY the expected cause tag and none of the other
/// three. This is the assertion that a shared "mentions every keyword"
/// template cannot satisfy — RED PROBE 2 (2026-08-04) produced exactly such a
/// template and slipped past a plain `contains("embed-ort")`, which is why the
/// tags exist at all.
fn assert_only_tag(msg: &str, want: &str) {
assert!(
msg.contains(want),
"must carry the [{want}] cause tag: {msg}"
);
for other in ALL_CAUSE_TAGS {
if other != want {
assert!(
!msg.contains(other),
"cause [{want}] must NOT also claim [{other}] — one message, one cause: {msg}"
);
}
}
}
/// RED-BEFORE-GREEN, the oden case: a binary with no `embed-ort` compiled in,
/// on a host that declares a GPU, MUST name the missing feature. This is the
/// branch a default (both-backends) test build can never reach at runtime —
/// injecting the facts is the only way it is ever proved.
#[test]
fn missing_ort_feature_is_named() {
let f = FallbackFacts {
selected_is_gpu: false,
ort_compiled: false,
gpu_intent: intent(),
..Default::default()
};
let msg = diagnose(&f).expect("a GPU-declaring host on CPU must produce a reason");
assert!(msg.contains("GPU EMBEDDING SKIPPED"), "must be loud: {msg}");
assert!(
msg.contains("embed-ort"),
"must name the missing feature: {msg}"
);
assert!(
msg.contains("ORT_DYLIB_PATH"),
"must quote the evidence: {msg}"
);
assert!(msg.contains("67×"), "must state the cost: {msg}");
assert_only_tag(&msg, NO_ORT_FEATURE);
}
/// Each failing probe gets its OWN cause. A single generic "GPU unavailable"
/// would be the same silent degradation with a warning label on it, so assert
/// the four causes are mutually exclusive — not merely unequal strings.
#[test]
fn each_failing_probe_names_a_different_cause() {
let base = FallbackFacts {
gpu_intent: intent(),
ort_compiled: true,
..Default::default()
};
let no_dylib = diagnose(&FallbackFacts {
onnxruntime: false,
..base.clone()
})
.unwrap();
assert!(no_dylib.contains("libonnxruntime.so"), "{no_dylib}");
assert!(
no_dylib.contains("SERVICE USER"),
"must hint at the perms trap: {no_dylib}"
);
assert_only_tag(&no_dylib, NO_DYLIB);
let no_ep = diagnose(&FallbackFacts {
onnxruntime: true,
..base.clone()
})
.unwrap();
assert!(no_ep.contains("execution provider"), "{no_ep}");
assert!(
no_ep.contains("libcuda.so.1"),
"must name the driver lib: {no_ep}"
);
assert_only_tag(&no_ep, NO_PROVIDER);
// All probes green but a CPU backend chosen anyway ⇒ selector bug, said so.
let bug = diagnose(&FallbackFacts {
onnxruntime: true,
cuda_ready: true,
..base.clone()
})
.unwrap();
assert!(bug.contains("selector bug"), "{bug}");
assert_only_tag(&bug, SELECTOR_BUG);
// The four causes must be genuinely different strings, not one template.
assert_ne!(no_dylib, no_ep);
assert_ne!(no_ep, bug);
assert_ne!(no_dylib, bug);
}
/// The silence side, and the reason this guard can go RED: a GPU box that
/// ACTUALLY selected a GPU must produce NOTHING. If `diagnose` ever returns
/// Some() here it would cry wolf on every healthy populate.
#[test]
fn a_working_gpu_says_nothing() {
let f = FallbackFacts {
selected_is_gpu: true,
ort_compiled: true,
onnxruntime: true,
cuda_ready: true,
gpu_intent: intent(),
..Default::default()
};
assert_eq!(diagnose(&f), None, "a healthy GPU run must be silent");
}
/// A genuine CPU-only server — no ORT_DYLIB_PATH, no device node — is NOT a
/// fallback and must stay silent even though `ort_compiled` is false. Without
/// this, every CPU box in the fleet would log a scary warning forever.
#[test]
fn a_real_cpu_box_is_not_a_fallback() {
let f = FallbackFacts {
ort_compiled: false,
gpu_intent: Vec::new(),
..Default::default()
};
assert_eq!(
diagnose(&f),
None,
"no GPU intent anywhere ⇒ CPU is the plan, not a fallback"
);
}
/// The live path must not panic on any box, and must agree with the pure
/// function over its own facts.
#[test]
fn live_is_self_consistent() {
let facts = live();
assert_eq!(
live_reason(),
diagnose(&facts),
"live_reason() == diagnose(live())"
);
// In a default test build ort IS compiled; assert the fact reports it, so
// this test fails if the feature forwarding is ever silently dropped.
#[cfg(feature = "embed-ort")]
assert!(
facts.ort_compiled,
"embed-ort is compiled here; facts must say so"
);
#[cfg(not(feature = "embed-ort"))]
assert!(
!facts.ort_compiled,
"embed-ort is NOT compiled here; facts must say so"
);
}
}