src/provenance.rs
//! **Where a test fixture CAME FROM** — the provenance record, and the guard that keeps
//! it true.
//!
//! # Why a fixture needs papers
//!
//! The owner, 2026-09-06: he will never create an account whose key lands in a build or in
//! git, so a collector's parser has to be proved against payloads that need no
//! credential — "read each provider HOMEPAGE and its API documentation, take the SAMPLE
//! PAYLOADS and public test endpoints they publish, commit them as fixtures with their
//! provenance (url + date fetched)".
//!
//! The reason the *provenance* is half the instruction and not a nicety: a fixture with
//! no papers is indistinguishable from one somebody wrote to make a test pass. This
//! crate already had 40-odd `SYNTHETIC_*` string literals, and they are honestly
//! labelled — [`SampleOrigin::Synthetic`](crate::sample::SampleOrigin::Synthetic) — but
//! a synthetic fixture can only ever prove that the parser accepts what we already
//! believed. It cannot find the field the provider actually sends. Only a real payload
//! can, and a real payload is only worth more than a synthetic one if you can still tell
//! which it is a month later.
//!
//! So every file under `tests/fixtures/provider/` has a row in `provenance.json` naming
//! the provider, the exact URL, the date it was fetched and the digest of the bytes —
//! and [`ProvenanceManifest::problems`] fails the build when a file has no row, a row
//! has no file, or the bytes no longer match the digest. That last one is the important
//! one: it means a fixture cannot be quietly "fixed" to keep a test green. Editing it
//! breaks the digest, and restoring the digest is a deliberate, reviewable act that says
//! in the diff that the provider's payload was replaced.
//!
//! # The manifest is DATA
//!
//! `provenance.json` is a JSON file, not a Rust table. It is written by the
//! `collector-fixtures` binary and read by the tests; nothing generates code from it.
use std::collections::BTreeMap;
use std::path::Path;
use serde::{Deserialize, Serialize};
/// The directory every provider fixture and its manifest live in, relative to the crate
/// root. One place, so the fetcher and the tests cannot disagree about where they are.
pub const FIXTURE_DIR: &str = "tests/fixtures/provider";
/// The manifest's file name inside [`FIXTURE_DIR`].
pub const MANIFEST_FILE: &str = "provenance.json";
/// **How this payload was obtained without a credential** — the distinction that decides
/// what a test may conclude from it.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Access {
/// A live endpoint the provider serves to anyone, no key: OpenSky's anonymous
/// `/states/all`, Nord Pool's public data portal slice. **Real data** — the
/// strongest kind of fixture, because the provider chose every field in it.
PublicKeylessEndpoint,
/// A payload printed in the provider's own API documentation as an example. Real in
/// the sense that the provider wrote it, but it may be abridged or idealised, so a
/// row count from one of these is not a measurement of anything.
DocumentationExample,
/// What the provider answers when the request is refused or malformed — an ENTSO-E
/// `Acknowledgement_MarketDocument`, a Trafikverket `ERROR` envelope.
///
/// This is a first-class fixture and not a consolation prize: the failure mode it
/// guards is a parser that reads a refusal as an empty result and reports "no rows
/// (window empty?)" — an unexplained zero laundered into an explained one, which is
/// exactly what RAGNARÖK exists to catch. It is the ONLY payload a credential-gated
/// endpoint will hand someone with no account, so for those collectors it is also
/// the only real provenance available at all.
ProviderErrorDocument,
}
impl Access {
/// The stable machine word (the manifest, the roster, a test name).
pub fn key(self) -> &'static str {
match self {
Access::PublicKeylessEndpoint => "public_keyless_endpoint",
Access::DocumentationExample => "documentation_example",
Access::ProviderErrorDocument => "provider_error_document",
}
}
/// Should a parse of this payload be expected to yield rows?
///
/// `false` for an error document, and that is the whole reason this method exists:
/// asserting "≥1 row" over every fixture would have forced an error document to be
/// either deleted or faked, and both lose the guard it carries.
pub fn expects_rows(self) -> bool {
!matches!(self, Access::ProviderErrorDocument)
}
}
/// One fixture's papers.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FixtureProvenance {
/// File name inside [`FIXTURE_DIR`].
pub file: String,
/// The `meta().name` of the collector whose parser this exercises.
pub collector: String,
/// The organisation that published it, in their own words.
pub provider: String,
/// The exact URL the bytes came from — a live endpoint, or the documentation page
/// the example was printed on. **Credential-free by construction**: nothing here was
/// fetched with a key, so no URL can carry one.
pub url: String,
/// `GET` / `POST` — a POST source records its body in `note`, because the request
/// shape is part of what makes the answer reproducible.
pub method: String,
/// The UTC date the fetch happened, `YYYY-MM-DD`. Dated because a live endpoint's
/// payload is a fact about a day, and LAW 7 says to date every figure taken from a
/// live source.
pub fetched_utc: String,
/// Byte length as fetched.
pub bytes: u64,
/// BLAKE3 of the bytes, hex — the house hash, same as znippy and skade.
pub blake3: String,
/// How it was obtained without a credential.
pub access: Access,
/// What it is and anything a reader needs in order to trust it: the query, the
/// abridgement, the reason this is an error document.
pub note: String,
}
/// **A collector for which no credential-free payload exists at all — checked, dated,
/// and written down.**
///
/// The other half of honesty. A roster that says only "unproven" for Bolagsverket
/// invites the next reader to go and try again, and they will find the same CAPTCHA.
/// Recording WHAT WAS TRIED and WHEN turns a gap into a fact, and a fact does not need
/// re-discovering every quarter.
///
/// It is deliberately not a permanent verdict: `checked_utc` dates it, because a
/// provider that publishes nothing today may publish an OpenAPI next year, and an
/// undated "impossible" is how a stale belief outlives its evidence.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Unobtainable {
/// The collector's `meta().name`.
pub collector: String,
/// The publishing organisation.
pub provider: String,
/// What was tried and what came back — concrete, quotable, and checkable by the
/// next reader.
pub reason: String,
/// UTC date this was last checked, `YYYY-MM-DD`.
pub checked_utc: String,
}
/// Every fixture's papers, as `provenance.json` holds them.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProvenanceManifest {
/// In file order — the fetcher keeps it sorted so a diff is readable.
pub fixtures: Vec<FixtureProvenance>,
/// Collectors whose providers publish nothing an anonymous reader can take.
#[serde(default)]
pub unobtainable: Vec<Unobtainable>,
}
impl ProvenanceManifest {
/// The recorded reason no credential-free payload exists for this collector.
pub fn unobtainable_for(&self, collector: &str) -> Option<&Unobtainable> {
self.unobtainable.iter().find(|u| u.collector == collector)
}
}
impl ProvenanceManifest {
/// Read the manifest from a fixture directory. `Ok(default)` when the directory has
/// no manifest yet (a first run of the fetcher); `Err` for a manifest that exists and
/// cannot be parsed, because a corrupt manifest must never read as an empty one.
pub fn load(dir: &Path) -> anyhow::Result<Self> {
let path = dir.join(MANIFEST_FILE);
if !path.exists() {
return Ok(Self::default());
}
let text = std::fs::read_to_string(&path)?;
Ok(serde_json::from_str(&text)?)
}
/// Write the manifest back, pretty-printed and sorted by file name.
pub fn save(&mut self, dir: &Path) -> anyhow::Result<()> {
self.fixtures.sort_by(|a, b| a.file.cmp(&b.file));
let text = serde_json::to_string_pretty(self)?;
std::fs::write(dir.join(MANIFEST_FILE), format!("{text}\n"))?;
Ok(())
}
/// The papers for one file.
pub fn get(&self, file: &str) -> Option<&FixtureProvenance> {
self.fixtures.iter().find(|f| f.file == file)
}
/// Insert or replace one fixture's papers.
pub fn upsert(&mut self, p: FixtureProvenance) {
match self.fixtures.iter_mut().find(|f| f.file == p.file) {
Some(slot) => *slot = p,
None => self.fixtures.push(p),
}
}
/// **Everything wrong with this fixture directory**, as operator-readable lines —
/// empty when the papers and the files agree.
///
/// Three failures, and each one is a way a fixture stops meaning what it claims:
///
/// * **a file with no row** — an undocumented payload; nobody can say whether the
/// provider sent it or an agent wrote it;
/// * **a row with no file** — papers for a fixture that is not there, so a test that
/// thinks it is covered is not;
/// * **a digest mismatch** — the bytes were edited after they were fetched, which
/// turns a provider's payload into a synthetic one wearing its name. This is the
/// one that matters most and the one a human eye never catches.
///
/// `digest` is injected rather than computed here so this module stays free of a
/// hashing dependency under the light `parsers` feature; the caller passes BLAKE3.
pub fn problems(
&self,
dir: &Path,
digest: impl Fn(&[u8]) -> String,
) -> anyhow::Result<Vec<String>> {
let mut out = Vec::new();
let mut on_disk: BTreeMap<String, Vec<u8>> = BTreeMap::new();
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().to_string();
if name == MANIFEST_FILE || !entry.file_type()?.is_file() {
continue;
}
on_disk.insert(name, std::fs::read(entry.path())?);
}
for (name, bytes) in &on_disk {
let Some(p) = self.get(name) else {
out.push(format!(
"{name}: a fixture with NO PROVENANCE. Nobody can say whether the \
provider sent these bytes or somebody wrote them — add a row to \
{MANIFEST_FILE} (or delete the file)"
));
continue;
};
if p.bytes != bytes.len() as u64 {
out.push(format!(
"{name}: provenance says {} bytes, the file is {}",
p.bytes,
bytes.len()
));
}
let actual = digest(bytes);
if actual != p.blake3 {
out.push(format!(
"{name}: DIGEST MISMATCH — the bytes have been edited since they \
were fetched from {}. provenance blake3 {}, file {}. A fixture that \
no longer matches its papers is a synthetic fixture wearing a \
provider's name; re-fetch it, or state in the diff why the payload \
was replaced.",
p.url, p.blake3, actual
));
}
}
// A collector cannot both have a credential-free payload and have none. This
// contradiction is easy to create — someone finds a fixture and forgets to
// delete the "impossible" row — and it makes the roster say the softer of two
// things.
for u in &self.unobtainable {
if self.fixtures.iter().any(|f| f.collector == u.collector) {
out.push(format!(
"{}: recorded as having NO credential-free payload, and a fixture is \
committed for it. Delete the `unobtainable` row.",
u.collector
));
}
if u.reason.trim().is_empty() || u.checked_utc.trim().is_empty() {
out.push(format!(
"{}: an `unobtainable` row needs both a reason and a checked_utc date \
— an undated 'impossible' is how a stale belief outlives its evidence",
u.collector
));
}
}
for p in &self.fixtures {
if !on_disk.contains_key(&p.file) {
out.push(format!(
"{}: provenance names a fixture that is not on disk, so every test \
that believes it is covered by {} is not",
p.file, p.file
));
}
}
Ok(out)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// The ONE parse dispatch
// ─────────────────────────────────────────────────────────────────────────────
/// **Read a fixture with the collector's REAL parser, by collector name.**
///
/// One writer, two callers: the `collector-fixtures` fetcher validates a payload through
/// this before it will commit it, and `tests/provider_fixtures.rs` sweeps every
/// committed fixture through the same function. Writing the dispatch twice would have
/// been the classic twin — the fetcher accepting a payload the tests then read
/// differently, which is a fixture that "works" only in the tool that made it.
///
/// It answers a ROW COUNT rather than a batch because that is all both callers need, and
/// because the collectors do not agree on an intermediate type: some yield a
/// `Vec<Struct>`, `RestCollector` yields a `RecordBatch` straight from
/// [`rest::parse_records`](crate::rest::parse_records). What they DO agree on is how many
/// records the provider's bytes contained.
///
/// `Err` is a real answer, not a failure of this function: a provider's refusal document
/// must be refused by name, and [`Access::expects_rows`] is what says which is wanted.
#[cfg(feature = "parsers")]
pub fn parse_rows(collector: &str, bytes: &[u8]) -> Result<usize, String> {
use crate::rest::RestCollector;
// The `RestCollector` family: the collector declares WHERE its records live and WHAT
// its columns are, and `rest::parse_records` is the single reader both the one-shot
// and the daemon face route through.
let rest = |make: fn() -> RestCollector| -> Result<usize, String> {
let v: serde_json::Value =
serde_json::from_slice(bytes).map_err(|e| format!("json: {e}"))?;
let c = make();
crate::rest::parse_records(&v, &c.records_pointer(), c.fields())
.map(|b| b.num_rows())
.map_err(|e| format!("{e:#}"))
};
let json = || -> Result<serde_json::Value, String> {
serde_json::from_slice(bytes).map_err(|e| format!("json: {e}"))
};
match collector {
"polisen-events" => rest(RestCollector::polisen),
"kolada-kpi" => rest(RestCollector::kolada),
"gleif-lei" => rest(RestCollector::gleif_lei),
"domstol-rattspraxis" => rest(RestCollector::domstol_rattspraxis),
"gbif-occurrences" => rest(RestCollector::gbif),
"worldbank-indicators" => rest(RestCollector::worldbank),
"cbs-odata" => rest(RestCollector::cbs_odata),
"metno-forecast" => rest(RestCollector::metno_forecast),
"riksdagen-dokument" => rest(RestCollector::riksdagen_dokument),
"trafikverket-trainpos" => {
let v = json()?;
let c = crate::trafikverket::TrafikverketCollector::train_positions();
crate::rest::parse_records(&v, &c.records_pointer(), c.fields())
.map(|b| b.num_rows())
.map_err(|e| format!("{e:#}"))
}
"nordpool" => crate::nordpool::parse_nordpool_json(&json()?)
.map(|p| p.len())
.map_err(|e| format!("{e:#}")),
"usgs-earthquakes" => crate::usgs::parse_usgs_geojson(&json()?)
.map(|q| q.len())
.map_err(|e| format!("{e:#}")),
"energy-charts-power" => crate::energy_charts::parse_energy_charts_json(&json()?)
.map(|r| r.len())
.map_err(|e| format!("{e:#}")),
"entsoe" | "entsoe_day_ahead" | "entsoe_generation" => {
crate::entsoe::parse_entsoe_xml(&String::from_utf8_lossy(bytes))
.map(|p| p.len())
.map_err(|e| format!("{e:#}"))
}
// OpenSky's `/states/all` is a bare array-of-arrays with no per-record object, so
// it has no `records_pointer` shape; the collector reads the `states` array
// positionally. Counting it here keeps the sweep honest without pretending it is
// a REST record list.
"opensky-flights" | "opensky-europe-bbox" | "opensky-tracks" => {
let v = json()?;
v.get("states")
.and_then(|s| s.as_array())
.map(|a| a.len())
.ok_or_else(|| "no `states` array".to_string())
}
other => Err(format!(
"no parse dispatch for collector `{other}` — a fixture was committed for a \
collector this function does not know how to read, so nothing is actually \
parsing it. Add an arm."
)),
}
}