src/bin/collector-fixtures.rs
//! **`collector-fixtures`** — fetch every provider fixture, with its papers.
//!
//! The owner's part (3), 2026-09-06: read each provider's own homepage and API
//! documentation, take the sample payloads and public test endpoints they publish,
//! commit them as fixtures with their provenance (url + date fetched), and test the
//! parsers against those — so there is real coverage and **no credential anywhere**.
//!
//! # Why this is a binary in the repo and not a shell one-liner
//!
//! LAW: pure Rust, no python, no shell scripts, no `Command::new`; fixtures are DATA,
//! not code. A `curl` in somebody's terminal history produces a fixture nobody can
//! reproduce and nobody can re-verify a year later. This tool is the reproduction: the
//! sources are declared as one table, the fetch is the crate's own HTTP path, and every
//! byte written gets a row in `provenance.json` carrying the URL, the UTC date, the
//! length and the BLAKE3 digest.
//!
//! It also VALIDATES before it writes: a payload that the collector's own parser cannot
//! read is refused, named, and not committed. A fixture that does not parse would
//! otherwise be discovered as a red test with no explanation days later.
//!
//! # It is never run by a test
//!
//! Tests read the committed bytes and never open a socket. This runs by hand, when a
//! provider changes its payload:
//!
//! ```text
//! cargo run --features daemon --bin collector-fixtures # fetch all
//! cargo run --features daemon --bin collector-fixtures -- nordpool # one, by slug
//! ```
//!
//! # No credential can enter here, by construction
//!
//! There is nowhere to put one. [`FixtureSource`] has no auth field, the fetch sends no
//! `Authorization` header and reads no environment variable, and every URL in the table
//! is one that answers an anonymous request. That is not a convention to remember — it
//! is the shape of the type.
use std::path::PathBuf;
use std::time::Duration;
use korp_collectors::provenance::{Access, FixtureProvenance, ProvenanceManifest, FIXTURE_DIR};
/// **One credential-free payload we can obtain, declared as data.**
///
/// Deliberately without an auth field, a header map or an env-var name — see the module
/// note. `body` is the only request shape beyond the URL, because Trafikverket's API
/// answers a POST and refuses a GET, and pretending otherwise would mean committing its
/// refusal as though it were its data.
struct FixtureSource {
/// Fixture file name (also the CLI slug, minus the extension).
file: &'static str,
/// The collector whose parser this proves, by `meta().name`.
collector: &'static str,
/// The publishing organisation, in their own words.
provider: &'static str,
/// The exact anonymous URL.
url: &'static str,
/// `Some(body)` posts it as `text/xml`; `None` is a plain GET.
body: Option<&'static str>,
/// How this is obtainable with no account.
access: Access,
/// What it is, and what a reader needs in order to trust it.
note: &'static str,
/// What the collector's parser must DO with these bytes.
///
/// Validation runs through [`korp_collectors::provenance::parse_rows`] — the ONE
/// dispatch, which `tests/provider_fixtures.rs` also sweeps every committed fixture
/// through. A second reader written in this file would be a fixture that "works"
/// only in the tool that made it.
expect: Expect,
}
/// One `RestCollector`-backed keyless source, since ten of them differ only in URL.
macro_rules! rest_fixture {
($file:literal, $collector:literal, $provider:literal, $url:literal, $min:literal, $note:literal) => {
FixtureSource {
file: $file,
collector: $collector,
provider: $provider,
url: $url,
body: None,
access: Access::PublicKeylessEndpoint,
note: $note,
expect: Expect::Rows($min),
}
};
}
/// **What the collector's parser must do with a fixture** — and the reason this is an
/// enum rather than a `min_rows: usize`.
///
/// The first draft said `min_rows: 0` for an error document, which quietly asserted that
/// a refusal parses to zero rows. Running it found that Trafikverket's ERROR envelope
/// makes `rest::parse_records` return `Err` (its records pointer is absent) — the RIGHT
/// answer, and one `min_rows: 0` would have called a failure. It also found the opposite
/// on ENTSO-E: an `Acknowledgement_MarketDocument` parsed to `Ok(0)` points, which the
/// collector then reports as "no TimeSeries points (window empty?)" — a refusal
/// laundered into an explained empty.
///
/// So "what should the parser do?" has two right answers and they are not the same, and
/// a fixture has to say which one it is.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Expect {
/// Parses, and yields at least this many rows.
Rows(usize),
/// **Must be REFUSED with a named error.** A provider's refusal document must never
/// parse to an empty-but-successful result, because an unexplained zero laundered
/// into an explained one is the exact defect RAGNARÖK exists to catch.
RefusedWithNamedError,
}
/// **The table.** One place, next to nothing else, so what we have and where it came
/// from is a single screen an operator can read.
fn sources() -> Vec<FixtureSource> {
vec![
FixtureSource {
file: "nordpool-dayahead-se3.json",
collector: "nordpool",
provider: "Nord Pool AS",
url: "https://dataportal-api.nordpoolgroup.com/api/DayAheadPrices\
?date=2026-09-05&market=DayAhead&deliveryArea=SE3¤cy=SEK",
body: None,
access: Access::PublicKeylessEndpoint,
note: "Nord Pool's public data portal serves a day-ahead slice with no \
credential, which the collector's own doc comment already records; \
the LICENSED api it declares NORDPOOL_API_TOKEN for is a different \
endpoint. SE3, SEK, one delivery day, 15-minute MTU — the modern \
`multiAreaEntries` shape, which is the branch parse_nordpool_json \
takes first.",
expect: Expect::Rows(24),
},
FixtureSource {
file: "opensky-states-nordic-bbox.json",
collector: "opensky-europe-bbox",
provider: "The OpenSky Network",
url: "https://opensky-network.org/api/states/all\
?lamin=55.0&lomin=10.0&lamax=60.0&lomax=20.0",
body: None,
access: Access::PublicKeylessEndpoint,
note: "OpenSky documents an anonymous tier (unauthenticated, bucketed by \
IP, most-recent state vectors at 10-second resolution) — no account, \
no OPENSKY_CLIENT_ID. A bounding box over the Nordic region so the \
payload is a few hundred aircraft rather than the whole sky. The \
18-element state-vector array order is the provider's, and it is \
precisely what a synthetic fixture cannot be trusted to have right.",
expect: Expect::Rows(1),
},
FixtureSource {
file: "entsoe-acknowledgement-auth-failed.xml",
collector: "entsoe",
provider: "ENTSO-E (European Network of Transmission System Operators for Electricity)",
url: "https://web-api.tp.entsoe.eu/api?documentType=A44\
&in_Domain=10Y1001A1001A46L&out_Domain=10Y1001A1001A46L\
&periodStart=202609050000&periodEnd=202609060000",
body: None,
access: Access::ProviderErrorDocument,
note: "ENTSO-E's Transparency Platform serves NO payload at all without a \
securityToken, and its API guide is behind a JS-rendered Postman \
collection and a 403 Zendesk, so this IEC 62325 \
Acknowledgement_MarketDocument (HTTP 401, Reason code 999 \
'Authentication failed.') is the only real ENTSO-E document \
obtainable with no account. It is worth committing on its own \
merits: it is the exact shape parse_entsoe_xml must NOT mistake for \
an empty result window.",
expect: Expect::RefusedWithNamedError,
},
FixtureSource {
file: "trafikverket-error-get-refused.json",
collector: "trafikverket-trainpos",
provider: "Trafikverket (the Swedish Transport Administration)",
url: "https://api.trafikinfo.trafikverket.se/v2/data.json",
body: None,
access: Access::ProviderErrorDocument,
note: "Trafikverket's API is POST-only and key-gated: with no \
TRAFIKVERKET_API_KEY there is no data document to be had at any URL. \
This is its RESPONSE/RESULT/ERROR envelope, which is the same \
envelope a key-bearing request gets back when the query is wrong — so \
it proves the collector reads the provider's error shape rather than \
reading an error as zero trains.",
expect: Expect::RefusedWithNamedError,
},
FixtureSource {
file: "riksdagen-dokumentlista.json",
collector: "riksdagen-dokument",
provider: "Sveriges riksdag (öppna data)",
url: "https://data.riksdagen.se/dokumentlista/\
?sok=&doktyp=mot&utformat=json&sz=5",
body: None,
access: Access::PublicKeylessEndpoint,
note: "Riksdagen's open-data API is keyless and documented as such. Five \
motions, the smallest page that still shows the dokumentlista \
envelope and repeated dokument entries.",
expect: Expect::Rows(1),
},
// ── The KEYLESS Swedish/EU open-data set. None of these needs an account at
// all, so a fixture for each is simply free real coverage — and it moves the
// collector out of `unproven` in the roster, which is where 48 of them sat
// when this file was written. Every URL is size-bounded on purpose: a fixture
// is a payload SHAPE, and a 200 kB one costs the tree without proving more.
rest_fixture!(
"polisen-events.json",
"polisen-events",
"Polismyndigheten (the Swedish Police Authority)",
"https://polisen.se/api/events",
50,
"Polisen's event feed is keyless and serves no page-size parameter, so this \
is the whole current window as the collector itself would read it."
),
rest_fixture!(
"kolada-kpi.json",
"kolada-kpi",
"Rådet för främjande av kommunala analyser (RKA)",
"https://api.kolada.se/v3/kpi?per_page=5",
5,
"Kolada's municipal KPI catalogue, keyless. Five KPIs — enough to carry the \
envelope and repeated entries."
),
rest_fixture!(
"gleif-lei-se.json",
"gleif-lei",
"GLEIF (Global Legal Entity Identifier Foundation)",
"https://api.gleif.org/api/v1/lei-records\
?filter%5Bentity.legalAddress.country%5D=SE&page%5Bsize%5D=3",
3,
"Three Swedish LEI records from GLEIF's keyless public API — the JSON:API \
envelope the collector's records pointer navigates."
),
rest_fixture!(
"domstol-rattspraxis.json",
"domstol-rattspraxis",
"Sveriges Domstolar",
"https://rattspraxis.etjanst.domstol.se/api/v1/publiceringar\
?sortorder=publiceringstid&pagesize=5",
5,
"Five published court decisions from the keyless rättspraxis service."
),
rest_fixture!(
"gbif-occurrences.json",
"gbif-occurrences",
"GBIF (Global Biodiversity Information Facility)",
"https://api.gbif.org/v1/occurrence/search?hasCoordinate=true&limit=5",
5,
"Five georeferenced occurrence records from GBIF's keyless API."
),
rest_fixture!(
"worldbank-population.json",
"worldbank-indicators",
"The World Bank",
"https://api.worldbank.org/v2/country/all/indicator/SP.POP.TOTL\
?format=json&per_page=5",
5,
"SP.POP.TOTL for five country-years. The World Bank's v2 JSON is a TWO-ELEMENT \
array whose first element is pagination and whose second is the data — a \
shape a hand-written fixture routinely gets wrong."
),
rest_fixture!(
"cbs-odata.json",
"cbs-odata",
"Centraal Bureau voor de Statistiek (Statistics Netherlands)",
"https://opendata.cbs.nl/ODataApi/odata/37296eng/TypedDataSet?$top=5",
5,
"Five rows of CBS table 37296eng through the keyless OData endpoint — the \
OData `value` envelope."
),
rest_fixture!(
"metno-locationforecast.json",
"metno-forecast",
"MET Norway (Meteorologisk institutt)",
"https://api.met.no/weatherapi/locationforecast/2.0/compact\
?lat=59.33&lon=18.07",
1,
"Stockholm. MET Norway is keyless but REQUIRES an identifying User-Agent, \
which this fetcher sends for every request — that is a courtesy header the \
provider documents, not a credential."
),
FixtureSource {
file: "usgs-quakes-all-hour.json",
collector: "usgs-earthquakes",
provider: "United States Geological Survey",
url: "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson",
body: None,
access: Access::PublicKeylessEndpoint,
note: "The USGS hourly summary feed, keyless — one hour of global \
seismicity as GeoJSON. Small by nature, and it exercises the \
geometry/coordinate path rather than a flat record list.",
expect: Expect::Rows(1),
},
FixtureSource {
file: "energy-charts-public-power-de.json",
collector: "energy-charts-power",
provider: "Fraunhofer ISE (Energy-Charts)",
url: "https://api.energy-charts.info/public_power?country=de",
body: None,
access: Access::PublicKeylessEndpoint,
note: "Energy-Charts is keyless. German public power generation by \
production type — a parallel-arrays shape (unix_seconds plus one \
series per type) rather than a record list.",
expect: Expect::Rows(1),
},
]
}
fn main() -> anyhow::Result<()> {
let only: Vec<String> = std::env::args().skip(1).collect();
let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(FIXTURE_DIR);
std::fs::create_dir_all(&dir)?;
let mut manifest = ProvenanceManifest::load(&dir)?;
let today = utc_today();
let client = tls_roots_blocking(reqwest::blocking::Client::builder())
.timeout(Duration::from_secs(30))
.user_agent("korp-collectors fixture fetcher (open data, anonymous)")
.build()?;
let mut wrote = 0usize;
let mut refused = 0usize;
for src in sources() {
if !only.is_empty() && !only.iter().any(|o| src.file.starts_with(o.as_str())) {
continue;
}
let url: String = src.url.split_whitespace().collect();
eprintln!("→ {} \n {}", src.file, url);
let resp = match &src.body {
Some(b) => client
.post(&url)
.header("Content-Type", "text/xml")
.body(*b)
.send(),
None => client.get(&url).send(),
};
let resp = match resp {
Ok(r) => r,
Err(e) => {
eprintln!(" REFUSED — transport: {e}");
refused += 1;
continue;
}
};
let status = resp.status();
let bytes = resp.bytes()?.to_vec();
// An error document is EXPECTED to be non-2xx; anything else must be a success,
// or we would be committing a 500 page as though it were data.
let want_error = src.access == Access::ProviderErrorDocument;
if status.is_success() == want_error {
eprintln!(
" REFUSED — HTTP {status} for access={} ({} bytes)",
src.access.key(),
bytes.len()
);
refused += 1;
continue;
}
// VALIDATE through the collector's own parser before anything is written. A
// fixture that does not behave as declared is never committed: it would surface
// days later as a red test with no explanation.
match (korp_collectors::provenance::parse_rows(src.collector, &bytes), src.expect) {
(Ok(rows), Expect::Rows(min)) if rows >= min => {
eprintln!(" ok — HTTP {status}, {} bytes, {rows} row(s)", bytes.len());
}
(Ok(rows), Expect::Rows(min)) => {
eprintln!(" REFUSED — parsed but only {rows} row(s), wanted >= {min}");
refused += 1;
continue;
}
(Err(e), Expect::RefusedWithNamedError) => {
eprintln!(
" ok — HTTP {status}, {} bytes, parser refuses it by name: {e}",
bytes.len()
);
}
(Ok(rows), Expect::RefusedWithNamedError) => {
eprintln!(
" REFUSED — the parser read this REFUSAL DOCUMENT as {rows} row(s) \
instead of erroring. That is a provider's 'no' being reported as an \
empty result. Fix the parser, then re-run."
);
refused += 1;
continue;
}
(Err(e), Expect::Rows(_)) => {
eprintln!(" REFUSED — the collector's parser cannot read it: {e}");
refused += 1;
continue;
}
}
std::fs::write(dir.join(src.file), &bytes)?;
manifest.upsert(FixtureProvenance {
file: src.file.to_string(),
collector: src.collector.to_string(),
provider: src.provider.to_string(),
url,
method: if src.body.is_some() { "POST" } else { "GET" }.to_string(),
fetched_utc: today.clone(),
bytes: bytes.len() as u64,
blake3: blake3_hex(&bytes),
access: src.access,
note: src.note.split_whitespace().collect::<Vec<_>>().join(" "),
});
wrote += 1;
}
manifest.save(&dir)?;
eprintln!("\n{wrote} fixture(s) written, {refused} refused. Manifest: {}", dir.display());
Ok(())
}
/// BLAKE3 hex — the house hash (znippy, skade, `acquire::Completeness::Blake3`).
fn blake3_hex(bytes: &[u8]) -> String {
blake3::hash(bytes).to_hex().to_string()
}
/// Today in UTC as `YYYY-MM-DD`, from the system clock with no date dependency.
///
/// A civil-date conversion written out rather than pulled in: `chrono`/`time` would be a
/// new dependency on a LEAF for one line of arithmetic used by a dev tool. The algorithm
/// is Howard Hinnant's `civil_from_days`, which is exact for every date this program can
/// see.
fn utc_today() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0);
let z = secs.div_euclid(86_400) + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
format!("{y:04}-{m:02}-{d:02}")
}
/// reqwest 0.13 with reqwest 0.12 `rustls-tls` behaviour (2026-10-02): the ring
/// provider and the bundled Mozilla roots (webpki), not 0.13's aws-lc-rs + OS
/// trust store. Roots added later on the builder are merged in, as in 0.12.
#[allow(dead_code)]
fn tls_roots_blocking(b: reqwest::blocking::ClientBuilder) -> reqwest::blocking::ClientBuilder {
let _ = rustls::crypto::ring::default_provider().install_default();
b.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
}