src/sample.rs
//! **Offline-safe real-or-fake sample data** — the `*_try_get_real_sample_data`
//! seam every collector's test + bench pulls from.
//!
//! Each function ATTEMPTS a **short-timeout live fetch** from the collector's real
//! open-data endpoint, VALIDATES the bytes actually parse with that collector's
//! own parser, and only then returns them tagged [`SampleOrigin::Real`]. If the
//! network is down, the endpoint 4xx/5xx/times-out, needs an absent credential,
//! or the bytes don't parse, it FALLS BACK to an embedded representative fixture
//! tagged [`SampleOrigin::Synthetic`]. So a test/bench that drives a sample:
//!
//! * **never hangs** — the fetch is bounded by [`REAL_TIMEOUT`];
//! * **never fails purely for want of a network** — the synthetic fixture is a
//! guaranteed-parseable stand-in;
//! * **is honest** — `origin.is_synthetic()` flags fabricated data explicitly.
//!
//! The URL recorded in [`SampleOrigin::Real`] is **credential-free**: a few of
//! these endpoints are gated by a token carried in the query string (ENTSO-E's
//! `securityToken`, Destatis GENESIS' `token`), and that string is rendered by
//! [`SampleOrigin::label`] into logs and durable test-matrix rows — so it goes
//! through [`redact_url`] first.
//!
//! Set `KORP_COLLECTORS_OFFLINE=1` to skip every live attempt (deterministic,
//! network-free CI); leave it unset to let the seam try real data first.
//!
//! The bodies are raw bytes in each collector's **canonical parse format** so the
//! same `Sample` drives both the INJECT/ASSERT test and the parse/normalize
//! bench off one offline-safe source.
use std::borrow::Cow;
use std::time::Duration;
/// The short per-attempt budget — a live fetch that hasn't answered within this
/// falls back to synthetic. Keeps `cargo test` / `cargo bench` bounded offline.
pub const REAL_TIMEOUT: Duration = Duration::from_millis(2000);
/// Query-parameter names whose VALUE is a credential. Several live sample
/// endpoints are gated by a token carried IN THE QUERY STRING (ENTSO-E's
/// `securityToken`, Destatis GENESIS' `token`), so the fetch URL holds a real
/// secret whenever the operator has exported one. Compared case-insensitively.
const SECRET_QUERY_PARAMS: &[&str] = &[
"securitytoken",
"token",
"access_token",
"accesstoken",
"api_key",
"apikey",
"key",
"subscription-key",
"password",
"secret",
];
/// The redaction placeholder substituted for a credential value.
const REDACTED: &str = "REDACTED";
/// Strip credentials out of `url` so it is safe to RECORD (matrix rows, logs,
/// [`SampleOrigin::Real`]).
///
/// Two carriers are scrubbed: any query parameter named in
/// [`SECRET_QUERY_PARAMS`] loses its value, and URL userinfo
/// (`https://user:pass@host`) loses its password. Everything else — scheme,
/// host, path, the non-secret query pairs, the fragment — is preserved so the
/// recorded URL still identifies WHICH endpoint the sample came from.
pub fn redact_url(url: &str) -> String {
// Split the fragment off first so a `#…` never lands inside a query value.
let (head, frag) = match url.split_once('#') {
Some((h, f)) => (h, Some(f)),
None => (url, None),
};
let (base, query) = match head.split_once('?') {
Some((b, q)) => (b, Some(q)),
None => (head, None),
};
let mut out = redact_userinfo(base);
if let Some(q) = query {
out.push('?');
let mut first = true;
for pair in q.split('&') {
if !first {
out.push('&');
}
first = false;
match pair.split_once('=') {
Some((name, value))
if !value.is_empty()
&& SECRET_QUERY_PARAMS.contains(&name.to_ascii_lowercase().as_str()) =>
{
out.push_str(name);
out.push('=');
out.push_str(REDACTED);
}
_ => out.push_str(pair),
}
}
}
if let Some(f) = frag {
out.push('#');
out.push_str(f);
}
out
}
/// Replace the password half of `scheme://user:pass@host/…` userinfo.
fn redact_userinfo(base: &str) -> String {
let Some((scheme, rest)) = base.split_once("://") else {
return base.to_string();
};
// Userinfo, if any, ends at the FIRST `@` and cannot span a `/`.
let authority_end = rest.find('/').unwrap_or(rest.len());
let Some(at) = rest[..authority_end].find('@') else {
return base.to_string();
};
let (userinfo, tail) = rest.split_at(at);
match userinfo.split_once(':') {
Some((user, pass)) if !pass.is_empty() => format!("{scheme}://{user}:{REDACTED}{tail}"),
_ => base.to_string(),
}
}
/// Where a [`Sample`] came from.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SampleOrigin {
/// A live fetch succeeded AND parsed — real open data pulled from `url`.
///
/// The URL is **credential-free**: it is passed through [`redact_url`] when
/// the sample is built, because this string is rendered by [`label`] into
/// durable, shareable test-matrix rows and logs.
///
/// [`label`]: SampleOrigin::label
Real { url: String },
/// The endpoint was unreachable / unparseable / credential-gated / offline —
/// an embedded representative fixture, clearly flagged **synthetic**.
Synthetic,
}
impl SampleOrigin {
/// Is this fabricated (fallback) data?
pub fn is_synthetic(&self) -> bool {
matches!(self, SampleOrigin::Synthetic)
}
/// Short tag for logs / matrix detail (`"real:https://…"` or `"synthetic"`).
pub fn label(&self) -> String {
match self {
SampleOrigin::Real { url } => format!("real:{url}"),
SampleOrigin::Synthetic => "synthetic".to_string(),
}
}
}
/// One representative sample for a collector: raw bytes in its canonical parse
/// format + provenance.
#[derive(Debug, Clone)]
pub struct Sample {
/// Real (live) or synthetic (fixture)?
pub origin: SampleOrigin,
/// Raw bytes — feed straight into the collector's parser.
pub body: Vec<u8>,
}
impl Sample {
/// A synthetic (fixture-backed) sample.
pub fn synthetic(body: impl Into<Vec<u8>>) -> Self {
Self {
origin: SampleOrigin::Synthetic,
body: body.into(),
}
}
/// Was this a fabricated fallback?
pub fn is_synthetic(&self) -> bool {
self.origin.is_synthetic()
}
/// Body as UTF-8 (lossy) — the text parsers (RDF/XML, JSON, ENTSO-E XML).
pub fn as_str(&self) -> Cow<'_, str> {
String::from_utf8_lossy(&self.body)
}
/// Body length in bytes.
pub fn len(&self) -> usize {
self.body.len()
}
/// Is the body empty?
pub fn is_empty(&self) -> bool {
self.body.is_empty()
}
}
/// `KORP_COLLECTORS_OFFLINE` set → never touch the network (synthetic only).
fn offline() -> bool {
std::env::var_os("KORP_COLLECTORS_OFFLINE").is_some()
}
/// A short-timeout blocking reqwest client. `None` if offline or the builder
/// fails — callers then fall back to synthetic. Must be called OUTSIDE a tokio
/// runtime (the sample seam is driven from sync tests + benches only).
fn client() -> Option<reqwest::blocking::Client> {
if offline() {
return None;
}
tls_roots_blocking(reqwest::blocking::Client::builder())
.connect_timeout(REAL_TIMEOUT)
.timeout(REAL_TIMEOUT)
.user_agent("korp-collectors-sample/0.1 (+https://git.gunnar.rs/nordisk)")
.build()
.ok()
}
/// GET `url` (optionally byte-range-capped so a huge dump doesn't blow the
/// budget). `None` on any failure — never panics, never blocks past the timeout.
fn http_get(url: &str, cap_bytes: Option<u64>) -> Option<Vec<u8>> {
let mut req = client()?.get(url);
if let Some(n) = cap_bytes {
req = req.header("range", format!("bytes=0-{}", n.saturating_sub(1)));
}
let resp = req.send().ok()?;
// A 206 (partial, from the Range) counts as success too.
if !(resp.status().is_success()) {
return None;
}
resp.bytes().ok().map(|b| b.to_vec())
}
/// GET `url` with extra request headers (byte-range-capped). Used by sources
/// whose v3 live path is credential-gated by a header key — OpenAQ v3 requires a
/// free `X-API-Key` (keyless now 401s). `None` on any failure.
fn http_get_headers(
url: &str,
cap_bytes: Option<u64>,
headers: &[(&str, &str)],
) -> Option<Vec<u8>> {
let mut req = client()?.get(url);
if let Some(n) = cap_bytes {
req = req.header("range", format!("bytes=0-{}", n.saturating_sub(1)));
}
for (k, v) in headers {
req = req.header(*k, *v);
}
let resp = req.send().ok()?;
if !resp.status().is_success() {
return None;
}
resp.bytes().ok().map(|b| b.to_vec())
}
/// GET `url` with HTTP **basic auth** (byte-range-capped). Used by sources whose
/// live path is username/password gated — OpenSky's `/tracks` historically wants
/// basic auth. `None` on any failure; never panics, never blocks past the budget.
fn http_get_basic(url: &str, cap_bytes: Option<u64>, user: &str, pass: &str) -> Option<Vec<u8>> {
let mut req = client()?.get(url).basic_auth(user, Some(pass));
if let Some(n) = cap_bytes {
req = req.header("range", format!("bytes=0-{}", n.saturating_sub(1)));
}
let resp = req.send().ok()?;
if !resp.status().is_success() {
return None;
}
resp.bytes().ok().map(|b| b.to_vec())
}
/// POST `body` (PxWeb selection / Trafikverket XML query). `None` on any failure.
fn http_post(url: &str, body: String, content_type: &str) -> Option<Vec<u8>> {
let resp = client()?
.post(url)
.header("content-type", content_type)
.body(body)
.send()
.ok()?;
if !resp.status().is_success() {
return None;
}
resp.bytes().ok().map(|b| b.to_vec())
}
/// The core real-or-fake decision: if `fetched` bytes validate, keep them as
/// [`SampleOrigin::Real`]; else fall back to the `synthetic` fixture.
fn real_or_synthetic(
url: &str,
fetched: Option<Vec<u8>>,
validate: impl Fn(&[u8]) -> bool,
synthetic: &[u8],
) -> Sample {
if let Some(bytes) = fetched {
if !bytes.is_empty() && validate(&bytes) {
// Redact HERE, at the single point the fetch URL becomes recorded
// identity — several of these URLs carry a live token in the query.
return Sample {
origin: SampleOrigin::Real {
url: redact_url(url),
},
body: bytes,
};
}
}
Sample::synthetic(synthetic.to_vec())
}
// ═══════════════════════════════════════════════════════════════════════════
// Embedded synthetic fixtures (representative, guaranteed-parseable).
// ═══════════════════════════════════════════════════════════════════════════
/// DCAT-AP-SE RDF/XML — two datasets (one with a distribution) → 2 registry rows.
pub const SYNTHETIC_DCAT_RDFXML: &str = r#"<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:dcat="http://www.w3.org/ns/dcat#"
xmlns:dct="http://purl.org/dc/terms/"
xmlns:foaf="http://xmlns.com/foaf/0.1/">
<dcat:Dataset rdf:about="https://data.example.se/dataset/befolkning">
<dct:title>Befolkning per kommun</dct:title>
<dct:publisher rdf:resource="https://data.example.se/org/scb"/>
<dct:license rdf:resource="http://creativecommons.org/publicdomain/zero/1.0/"/>
<dcat:distribution rdf:resource="https://data.example.se/dist/befolkning.csv"/>
</dcat:Dataset>
<dcat:Distribution rdf:about="https://data.example.se/dist/befolkning.csv">
<dcat:downloadURL rdf:resource="https://data.example.se/files/befolkning.csv"/>
<dct:format>CSV</dct:format>
<dcat:mediaType>text/csv</dcat:mediaType>
</dcat:Distribution>
<foaf:Agent rdf:about="https://data.example.se/org/scb">
<foaf:name>Statistiska centralbyrån</foaf:name>
</foaf:Agent>
<dcat:Dataset rdf:about="https://data.example.se/dataset/empty">
<dct:title>Tom dataset</dct:title>
</dcat:Dataset>
</rdf:RDF>"#;
/// PxWeb JSON-stat2 cube — 2×2×1 = 4 cells (region × year × ContentsCode).
pub const SYNTHETIC_PXWEB_JSONSTAT2: &str = r#"{
"version": "2.0", "class": "dataset",
"id": ["region", "year", "ContentsCode"],
"size": [2, 2, 1],
"dimension": {
"region": { "category": { "index": { "0114": 0, "0115": 1 } } },
"year": { "category": { "index": { "2020": 0, "2021": 1 } } },
"ContentsCode": { "category": { "index": { "BE0101N1": 0 } } }
},
"value": [100, 110, 200, 210]
}"#;
/// Eurostat SDMX-JSON 2.1 — 1 series dim (`geo`: SE, NO) × 1 observation dim
/// (`TIME_PERIOD`: 2021, 2022) → 4 observations, in the series/observations shape.
pub const SYNTHETIC_EUROSTAT_SDMX_JSON: &str = r#"{
"meta": {"id": "synthetic", "prepared": "2026-07-14T00:00:00Z"},
"data": {
"structures": [{
"dimensions": {
"series": [
{"id":"geo","name":"Geopolitical entity",
"values":[{"id":"SE","name":"Sweden"},{"id":"NO","name":"Norway"}]}
],
"observation": [
{"id":"TIME_PERIOD","name":"Period",
"values":[{"id":"2021"},{"id":"2022"}]}
]
}
}],
"dataSets": [{
"series": {
"0": {"observations": {"0": [100.0], "1": [110.0]}},
"1": {"observations": {"0": [50.0], "1": [55.0]}}
}
}]
}
}"#;
/// ENTSO-E `Publication_MarketDocument` — SE3 day-ahead, two `Point`s.
pub const SYNTHETIC_ENTSOE_XML: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<Publication_MarketDocument xmlns="urn:iec62325.351:tc57wg16:451-3:publicationdocument:7:0">
<TimeSeries>
<mRID>1</mRID>
<in_Domain.mRID codingScheme="A01">10Y1001A1001A46L</in_Domain.mRID>
<Period>
<timeInterval><start>2026-01-01T00:00Z</start><end>2026-01-01T02:00Z</end></timeInterval>
<resolution>PT60M</resolution>
<Point><position>1</position><quantity>42.5</quantity></Point>
<Point><position>2</position><quantity>43.0</quantity></Point>
</Period>
</TimeSeries>
</Publication_MarketDocument>"#;
/// ENTSO-E `Publication_MarketDocument` — SE3 DAY-AHEAD PRICES (`A44`), three
/// hourly `price.amount` points (the energy-market spot-price shape).
pub const SYNTHETIC_ENTSOE_A44_XML: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<Publication_MarketDocument xmlns="urn:iec62325.351:tc57wg16:451-3:publicationdocument:7:0">
<TimeSeries>
<mRID>1</mRID>
<in_Domain.mRID codingScheme="A01">10Y1001A1001A46L</in_Domain.mRID>
<out_Domain.mRID codingScheme="A01">10Y1001A1001A46L</out_Domain.mRID>
<currency_Unit.name>EUR</currency_Unit.name>
<price_Measure_Unit.name>MWH</price_Measure_Unit.name>
<Period>
<timeInterval><start>2026-01-01T00:00Z</start><end>2026-01-01T03:00Z</end></timeInterval>
<resolution>PT60M</resolution>
<Point><position>1</position><price.amount>42.5</price.amount></Point>
<Point><position>2</position><price.amount>39.9</price.amount></Point>
<Point><position>3</position><price.amount>55.1</price.amount></Point>
</Period>
</TimeSeries>
</Publication_MarketDocument>"#;
/// OpenSky `/tracks` — one aircraft's recorded flight PATH: 3 time-ascending
/// waypoints, each `[time, lat, lon, baro_altitude, true_track, on_ground]`.
pub const SYNTHETIC_OPENSKY_TRACKS_JSON: &str = r#"{
"icao24": "4b1815", "callsign": "SWR123",
"startTime": 1700000000, "endTime": 1700000120,
"path": [
[1700000000, 47.45, 8.55, 500.0, 87.3, true],
[1700000060, 47.48, 8.60, 1200.0, 88.0, false],
[1700000120, 47.52, 8.66, 2400.0, 89.1, false]
]
}"#;
/// Polisen events — a top-level JSON array, two incidents.
pub const SYNTHETIC_POLISEN_JSON: &str = r#"[
{"id":123,"datetime":"2026-06-30 12:00:00","name":"Trafikolycka","type":"Trafik",
"summary":"x","url":"https://polisen.se/x","location":{"name":"Stockholm","gps":"59.3,18.0"}},
{"id":124,"datetime":"2026-06-30 13:00:00","name":"Stöld","type":"Stöld",
"summary":"y","url":"https://polisen.se/y","location":{"name":"Uppsala","gps":"59.8,17.6"}}
]"#;
/// Kolada KPI catalogue — a `values` array, two KPIs.
pub const SYNTHETIC_KOLADA_JSON: &str = r#"{"count":2,"values":[
{"id":"N00002","title":"Andel invånare 0-19 år","operating_area":"Befolkning","description":"…"},
{"id":"N00003","title":"Andel invånare 65+ år","operating_area":"Befolkning","description":"…"}
]}"#;
/// SMHI metobs latest values — a `value` array, one observation.
pub const SYNTHETIC_SMHI_JSON: &str =
r#"{"value":[{"date":1751284800000,"value":"18.4","quality":"G"}]}"#;
/// Digitraffic AIS ship positions — a GeoJSON `FeatureCollection`, two moving-map rows.
///
/// **Copied from the live response**, not invented: two features taken verbatim from
/// `https://meri.digitraffic.fi/api/ais/v1/locations` on 2026-09-03 (that response
/// carried 1001 of them). It keeps the two traps the shape actually contains — the
/// position is `geometry.coordinates` as `[lon, lat]` (lon FIRST), and
/// `properties.timestamp` is the AIS second-of-minute (0..59) while
/// `properties.timestampExternal` is the epoch-ms receive time the business key uses.
pub const SYNTHETIC_AIS_JSON: &str = r#"{
"type": "FeatureCollection",
"dataUpdatedTime": "2026-09-03T21:46:20Z",
"features": [
{"mmsi": 265676640, "type": "Feature",
"geometry": {"type": "Point", "coordinates": [18.854418, 59.654993]},
"properties": {"mmsi": 265676640, "sog": 0.0, "cog": 360.0, "navStat": 0, "rot": -128,
"posAcc": true, "raim": true, "heading": 511,
"timestamp": 10, "timestampExternal": 1788471970217}},
{"mmsi": 230988040, "type": "Feature",
"geometry": {"type": "Point", "coordinates": [22.226532, 60.429507]},
"properties": {"mmsi": 230988040, "sog": 0.0, "cog": 0.0, "navStat": 5, "rot": -128,
"posAcc": true, "raim": true, "heading": 511,
"timestamp": 0, "timestampExternal": 1788471960031}}
]
}"#;
/// OGC WFS GML `FeatureCollection` — one Point feature (gml:pos) + one
/// LineString feature (gml:posList), two members → two rows.
pub const SYNTHETIC_WFS_GML: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<wfs:FeatureCollection xmlns:wfs="http://www.opengis.net/wfs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2"
xmlns:ad="http://example.se/ad">
<wfs:member>
<ad:AddressPoint gml:id="AP.1">
<ad:name>Sergels Torg</ad:name>
<ad:population>1234</ad:population>
<ad:geometry>
<gml:Point srsName="urn:ogc:def:crs:EPSG::4326"><gml:pos>59.3326 18.0649</gml:pos></gml:Point>
</ad:geometry>
</ad:AddressPoint>
</wfs:member>
<wfs:member>
<ad:Road gml:id="RD.7">
<ad:name>E4</ad:name>
<ad:geometry>
<gml:LineString srsName="urn:ogc:def:crs:EPSG::4326">
<gml:posList>59.10 18.00 59.20 18.05 59.30 18.10</gml:posList>
</gml:LineString>
</ad:geometry>
</ad:Road>
</wfs:member>
</wfs:FeatureCollection>"#;
/// Nord Pool day-ahead — a `multiAreaEntries` document, 2 periods × 3 SE zones.
pub const SYNTHETIC_NORDPOOL_JSON: &str = r#"{
"deliveryDateCET": "2026-07-13", "currency": "EUR",
"multiAreaEntries": [
{"deliveryStart":"2026-07-13T00:00:00Z","deliveryEnd":"2026-07-13T01:00:00Z",
"entryPerArea":{"SE1":42.5,"SE3":55.1,"SE4":60.2}},
{"deliveryStart":"2026-07-13T01:00:00Z","deliveryEnd":"2026-07-13T02:00:00Z",
"entryPerArea":{"SE1":40.0,"SE3":50.0,"SE4":58.0}}
]
}"#;
/// USGS earthquakes — a GeoJSON `FeatureCollection` with two `Point` features
/// (a shallow and a deep quake) → two flattened quake rows.
pub const SYNTHETIC_USGS_GEOJSON: &str = r#"{
"type": "FeatureCollection",
"metadata": {"generated": 1752400000000, "title": "USGS All Earthquakes, Past Hour", "count": 2},
"features": [
{"type":"Feature","id":"nc73012345",
"properties":{"mag":4.2,"place":"10km N of Reno, Nevada","time":1752400000000,"type":"earthquake",
"url":"https://earthquake.usgs.gov/earthquakes/eventpage/nc73012345"},
"geometry":{"type":"Point","coordinates":[-119.81,39.53,8.4]}},
{"type":"Feature","id":"us7000abcd",
"properties":{"mag":5.9,"place":"South of Fiji Islands","time":1752400500000,"type":"earthquake",
"url":"https://earthquake.usgs.gov/earthquakes/eventpage/us7000abcd"},
"geometry":{"type":"Point","coordinates":[178.12,-22.34,560.0]}}
]
}"#;
/// RSS 2.0 syndication feed — a channel with two `<item>`s (one CDATA body, one
/// `dc:creator`, categories) → two flattened feed rows.
pub const SYNTHETIC_RSS_XML: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Krisinformation</title>
<link>https://www.krisinformation.se</link>
<description>Official alerts</description>
<item>
<title>Flood warning, Dalälven</title>
<link>https://www.krisinformation.se/en/1</link>
<description><![CDATA[High water levels expected.]]></description>
<pubDate>Mon, 14 Jul 2026 08:00:00 GMT</pubDate>
<guid>https://www.krisinformation.se/en/1</guid>
<author>vmr@krisinformation.se</author>
<category>Weather</category>
<category>Flood</category>
</item>
<item>
<title>Power outage, Norrland</title>
<link>https://www.krisinformation.se/en/2</link>
<description>Restoration underway.</description>
<pubDate>Mon, 14 Jul 2026 09:30:00 GMT</pubDate>
<guid>tag:kris,2</guid>
<dc:creator>Elnät</dc:creator>
<category>Power</category>
</item>
</channel>
</rss>"#;
/// OpenAQ air-quality — a `results` envelope with two located readings (a `pm25`
/// and a `no2` station) → two flattened air-quality measurement rows.
pub const SYNTHETIC_OPENAQ_JSON: &str = r#"{
"meta": {"found": 2, "page": 1, "limit": 100},
"results": [
{"locationId": 8118, "location": "Stockholm Torkel Knutssonsgatan",
"parameter": "pm25", "value": 7.4, "unit": "µg/m³", "country": "SE", "city": "Stockholm",
"coordinates": {"latitude": 59.3162, "longitude": 18.0575},
"date": {"utc": "2026-07-14T08:00:00Z", "local": "2026-07-14T10:00:00+02:00"}},
{"locationId": 2178, "location": "Göteborg Femman",
"parameter": "no2", "value": 21.9, "unit": "µg/m³", "country": "SE", "city": "Göteborg",
"coordinates": {"latitude": 57.7075, "longitude": 11.9675},
"date": {"utc": "2026-07-14T08:00:00Z", "local": "2026-07-14T10:00:00+02:00"}}
]
}"#;
/// Energy-Charts public-power — a `{unix_seconds, production_types}` document:
/// two instants of the shared time axis × three production types (Solar, Wind
/// onshore, Nuclear) → six transposed `(time, type, power)` rows.
pub const SYNTHETIC_ENERGY_CHARTS_JSON: &str = r#"{
"unix_seconds": [1751328000, 1751328900],
"deprecated": false,
"production_types": [
{"name": "Solar", "data": [1200.5, 1810.0]},
{"name": "Wind onshore", "data": [ 640.0, 655.5]},
{"name": "Nuclear", "data": [4000.0, 4000.0]}
]
}"#;
/// Bolagsverket **Värdefulla datamängder** `POST /organisationer` — the
/// `{"organisationer":[…]}` envelope, two companies.
///
/// The record shape is copied from the API's OWN published example
/// (`organisations-aktiebolag-svar` in the `VärdefullaDatamängder` v1 OpenAPI, fetched
/// 2026-09-03), so this fixture proves the field pointers against the documented
/// document and not against a shape korp invented. The old fixture matched the DEAD
/// `naringslivsregistret` endpoint: flat `organisationsnummer`/`namn`/`adress.kommunkod`
/// keys that this API does not serve — including a `kommunkod` that does not exist in
/// it at all.
pub const SYNTHETIC_BOLAGSVERKET_JSON: &str = r#"{"organisationer":[
{"organisationsidentitet":{"identitetsbeteckning":"5560360793"},
"organisationsnamn":{"organisationsnamnLista":[
{"registreringsdatum":"1915-04-22","namn":"Volvo Aktiebolag",
"organisationsnamntyp":{"kod":"FORETAGSNAMN","klartext":"Företagsnamn"}}],
"fel":null,"dataproducent":"Bolagsverket"},
"registreringsland":{"kod":"SE-LAND","klartext":"Sverige"},
"organisationsform":{"kod":"AB","klartext":"Aktiebolag","fel":null,"dataproducent":"Bolagsverket"},
"juridiskForm":{"kod":"49","klartext":"Övriga aktiebolag"},
"verksamOrganisation":{"kod":"JA","fel":null,"dataproducent":"SCB"},
"postadressOrganisation":{"postadress":{"postnummer":"40508","utdelningsadress":"Volvo",
"land":"Sverige","postort":"Göteborg"},"fel":null,"dataproducent":"SCB"},
"verksamhetsbeskrivning":{"beskrivning":"Tillverkning av motorfordon","fel":null,
"dataproducent":"Bolagsverket"},
"organisationsdatum":{"registreringsdatum":"1915-04-22","fel":null,
"dataproducent":"Bolagsverket"},
"naringsgrenOrganisation":{"fel":null,"dataproducent":"Bolagsverket",
"sni":[{"kod":"29100","klartext":"Tillverkning av motorfordon"}]}},
{"organisationsidentitet":{"identitetsbeteckning":"5020171065"},
"organisationsnamn":{"organisationsnamnLista":[
{"registreringsdatum":"1943-07-28","namn":"Ikea AB",
"organisationsnamntyp":{"kod":"FORETAGSNAMN","klartext":"Företagsnamn"}}],
"fel":null,"dataproducent":"Bolagsverket"},
"registreringsland":{"kod":"SE-LAND","klartext":"Sverige"},
"organisationsform":{"kod":"AB","klartext":"Aktiebolag","fel":null,"dataproducent":"Bolagsverket"},
"juridiskForm":{"kod":"49","klartext":"Övriga aktiebolag"},
"verksamOrganisation":{"kod":"JA","fel":null,"dataproducent":"SCB"},
"postadressOrganisation":{"postadress":{"postnummer":"34381","utdelningsadress":"Box 700",
"land":"Sverige","postort":"Älmhult"},"fel":null,"dataproducent":"SCB"},
"verksamhetsbeskrivning":{"beskrivning":"Butikshandel med möbler","fel":null,
"dataproducent":"Bolagsverket"},
"organisationsdatum":{"registreringsdatum":"1943-07-28","fel":null,
"dataproducent":"Bolagsverket"},
"naringsgrenOrganisation":{"fel":null,"dataproducent":"Bolagsverket",
"sni":[{"kod":"47591","klartext":"Butikshandel med möbler"}]}}
]}"#;
/// GBIF occurrences — a `results` envelope with two located occurrences (a plant
/// and a bird), the WGS84 fix + taxon + eventDate → two occurrence point rows.
pub const SYNTHETIC_GBIF_JSON: &str = r#"{
"offset": 0, "limit": 20, "endOfRecords": false, "count": 2,
"results": [
{"key": 4501234567, "scientificName": "Bellis perennis L.", "kingdom": "Plantae",
"decimalLatitude": 59.3293, "decimalLongitude": 18.0686, "country": "SE",
"basisOfRecord": "HUMAN_OBSERVATION", "datasetKey": "ds-artportalen",
"eventDate": "2026-06-30T09:15:00"},
{"key": 4501234568, "scientificName": "Cyanistes caeruleus (Linnaeus, 1758)", "kingdom": "Animalia",
"decimalLatitude": 57.7089, "decimalLongitude": 11.9746, "country": "SE",
"basisOfRecord": "HUMAN_OBSERVATION", "datasetKey": "ds-ebird",
"eventDate": "2026-07-01T07:40:00"}
]
}"#;
/// World Bank indicators — the `[meta, data]` two-element envelope: paging meta,
/// then two population observations (SE 2021, SE 2022) → two statistics rows.
pub const SYNTHETIC_WORLDBANK_JSON: &str = r#"[
{"page": 1, "pages": 1, "per_page": 50, "total": 2},
[
{"indicator": {"id": "SP.POP.TOTL", "value": "Population, total"},
"country": {"id": "SE", "value": "Sweden"}, "countryiso3code": "SWE",
"date": "2022", "value": 10486941, "unit": "", "obs_status": "", "decimal": 0},
{"indicator": {"id": "SP.POP.TOTL", "value": "Population, total"},
"country": {"id": "SE", "value": "Sweden"}, "countryiso3code": "SWE",
"date": "2021", "value": 10415811, "unit": "", "obs_status": "", "decimal": 0}
]
]"#;
/// RESTCountries reference — a top-level array with two countries (SE, NO) →
/// two country reference rows (name, ISO codes, region, population, centroid).
pub const SYNTHETIC_RESTCOUNTRIES_JSON: &str = r#"[
{"name": {"common": "Sweden", "official": "Kingdom of Sweden"}, "cca2": "SE", "cca3": "SWE",
"region": "Europe", "subregion": "Northern Europe", "population": 10353442, "latlng": [62.0, 15.0]},
{"name": {"common": "Norway", "official": "Kingdom of Norway"}, "cca2": "NO", "cca3": "NOR",
"region": "Europe", "subregion": "Northern Europe", "population": 5379475, "latlng": [62.0, 10.0]}
]"#;
/// Mobility Database feed catalogue — a top-level array of feed descriptor
/// records (the `/v1/gtfs_feeds` shape): two GTFS feeds, each with a nested
/// `locations` array, `source_info.producer_url` and `latest_dataset.hosted_url`
/// → two feed-descriptor rows the GTFS collectors can be pointed at.
pub const SYNTHETIC_MOBILITY_DB_JSON: &str = r#"[
{"id":"mdb-1","data_type":"gtfs","status":"active","provider":"Storstockholms Lokaltrafik",
"feed_name":"SL","locations":[{"country_code":"SE","subdivision_name":"Stockholm","municipality":"Stockholm"}],
"source_info":{"producer_url":"https://opendata.samtrafiken.se/gtfs/sl/sl.zip"},
"latest_dataset":{"id":"mdb-1-202607","hosted_url":"https://files.mobilitydatabase.org/mdb-1/mdb-1-202607.zip"}},
{"id":"mdb-2","data_type":"gtfs","status":"active","provider":"Ruter",
"feed_name":"Ruter","locations":[{"country_code":"NO","subdivision_name":"Oslo","municipality":"Oslo"}],
"source_info":{"producer_url":"https://storage.googleapis.com/marduk-production/outbound/gtfs/rb_ruter-aggregated-gtfs.zip"},
"latest_dataset":{"id":"mdb-2-202607","hosted_url":"https://files.mobilitydatabase.org/mdb-2/mdb-2-202607.zip"}}
]"#;
/// ERDDAP tabledap `.json` `table` document — the self-describing shape:
/// `columnNames` + `columnTypes` + a `rows` matrix. Two NDBC-buoy rows (time,
/// lat, lon, wmo id, SST) → two typed Arrow rows.
pub const SYNTHETIC_ERDDAP_JSON: &str = r#"{
"table": {
"columnNames": ["time", "latitude", "longitude", "wmo_id", "sea_surface_temperature"],
"columnTypes": ["String", "double", "double", "int", "float"],
"rows": [
["2026-07-18T00:00:00Z", 55.02, 10.51, 6201, 17.8],
["2026-07-18T01:00:00Z", 55.02, 10.51, 6201, 17.6]
]
}
}"#;
/// EEA WISE water quality (Discodata) — a `results` array, two aggregated
/// water-quality rows (a nitrate + a phosphate determinand at located sites).
pub const SYNTHETIC_EEA_WISE_JSON: &str = r#"{
"results": [
{"monitoringSiteIdentifier": "SE_EU_1001", "countryCode": "SE",
"parameterWaterBodyCategory": "RW", "observedPropertyDeterminandLabel": "Nitrate",
"resultMeanValue": 2.35, "resultUom": "mg{NO3}/L", "phenomenonTimeReferenceYear": 2023,
"lat": 59.3162, "lon": 18.0575},
{"monitoringSiteIdentifier": "DE_EU_2002", "countryCode": "DE",
"parameterWaterBodyCategory": "LW", "observedPropertyDeterminandLabel": "Phosphate",
"resultMeanValue": 0.08, "resultUom": "mg{P}/L", "phenomenonTimeReferenceYear": 2023,
"lat": 52.52, "lon": 13.405}
]
}"#;
/// Destatis GENESIS catalogue/tables — a `List` of two statistical-table
/// descriptors (code, human title, time coverage).
pub const SYNTHETIC_DESTATIS_GENESIS_JSON: &str = r#"{
"Ident": {"Service": "catalogue", "Method": "tables"},
"Status": {"Code": 0, "Content": "erfolgreich"},
"List": [
{"Code": "12411-0001", "Content": "Bevölkerung: Bundesländer, Stichtag", "Time": "1990-2024"},
{"Code": "81000-0001", "Content": "Volkswirtschaftliche Gesamtrechnungen", "Time": "1991-2024"}
]
}"#;
/// CBS Open Data (Statistics Netherlands) OData TypedDataSet — a `value` array,
/// two typed population observations (period × region → value).
pub const SYNTHETIC_CBS_ODATA_JSON: &str = r#"{
"odata.metadata": "https://opendata.cbs.nl/ODataApi/odata/37296eng/$metadata#Cbs.OData.TypedDataSet",
"value": [
{"ID": 0, "Perioden": "2022JJ00", "RegioS": "NL01", "TotaleBevolking_1": 17590672},
{"ID": 1, "Perioden": "2023JJ00", "RegioS": "NL01", "TotaleBevolking_1": 17811291}
]
}"#;
/// A representative NATIONAL SDMX-JSON 2.1 message (INSEE / ONS / ISTAT / World
/// Bank all ride the shared flattener): `REF_AREA` (FR, IT) × `TIME_PERIOD`
/// (2021, 2022) → 4 observations. Provider-agnostic — the SAME body validates
/// through every national preset's parser.
pub const SYNTHETIC_NATIONAL_SDMX_JSON: &str = r#"{
"meta": {"id": "synthetic-national", "prepared": "2026-07-18T00:00:00Z"},
"data": {
"structures": [{
"dimensions": {
"series": [
{"id":"REF_AREA","name":"Reference area",
"values":[{"id":"FR","name":"France"},{"id":"IT","name":"Italy"}]}
],
"observation": [
{"id":"TIME_PERIOD","name":"Period",
"values":[{"id":"2021"},{"id":"2022"}]}
]
}
}],
"dataSets": [{
"series": {
"0": {"observations": {"0": [67.5], "1": [67.8]}},
"1": {"observations": {"0": [59.1], "1": [58.9]}}
}
}]
}
}"#;
/// Mobility Database GTFS-Realtime feed catalogue — a top-level array of RT
/// feed-descriptor records (the `/v1/gtfs_rt_feeds` shape): two GTFS-RT feeds
/// (a vehicle-positions + a trip-updates feed), each with `entity_types`, a
/// nested `locations` array and a `.pb` `source_info.producer_url` → two
/// feed-descriptor rows the gtfs-rt reader can be pointed at.
pub const SYNTHETIC_MOBILITY_DB_RT_JSON: &str = r#"[
{"id":"mdb-rt-1","data_type":"gtfs_rt","entity_types":["vp"],"status":"active",
"provider":"Storstockholms Lokaltrafik","feed_name":"SL VehiclePositions",
"locations":[{"country_code":"SE","subdivision_name":"Stockholm","municipality":"Stockholm"}],
"source_info":{"producer_url":"https://opendata.samtrafiken.se/gtfs-rt/sl/VehiclePositions.pb"}},
{"id":"mdb-rt-2","data_type":"gtfs_rt","entity_types":["tu"],"status":"active",
"provider":"Ruter","feed_name":"Ruter TripUpdates",
"locations":[{"country_code":"NO","subdivision_name":"Oslo","municipality":"Oslo"}],
"source_info":{"producer_url":"https://api.entur.io/realtime/v1/gtfs-rt/trip-updates"}}
]"#;
/// SMHI hydrological observations (WATER) — a `value` array (metobs-shaped), one
/// water-level/discharge reading.
pub const SYNTHETIC_SMHI_HYDRO_JSON: &str =
r#"{"value":[{"date":1751284800000,"value":"124.5","quality":"G"}]}"#;
/// Met.no Locationforecast (WEATHER) — a `properties.timeseries` array with two
/// forecast steps (temperature, wind, humidity, pressure, cloud) → two rows.
pub const SYNTHETIC_METNO_JSON: &str = r#"{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [10.7, 59.9, 5]},
"properties": {
"meta": {"updated_at": "2026-07-18T10:00:00Z", "units": {"air_temperature": "celsius"}},
"timeseries": [
{"time": "2026-07-18T11:00:00Z",
"data": {"instant": {"details": {
"air_temperature": 17.1, "wind_speed": 3.2, "wind_from_direction": 210.0,
"relative_humidity": 61.0, "air_pressure_at_sea_level": 1015.2, "cloud_area_fraction": 40.6}}}},
{"time": "2026-07-18T12:00:00Z",
"data": {"instant": {"details": {
"air_temperature": 18.4, "wind_speed": 3.8, "wind_from_direction": 205.0,
"relative_humidity": 57.0, "air_pressure_at_sea_level": 1014.8, "cloud_area_fraction": 55.0}}}}
]
}
}"#;
/// Trafikverket TrainPosition — a `<REQUEST>`→JSON response, one position row.
pub const SYNTHETIC_TRAFIKVERKET_JSON: &str = r#"{"RESPONSE":{"RESULT":[{"TrainPosition":[
{"Train":{"OperationalTrainNumber":"527"},"Position":{"WGS84":"POINT (16.18 59.61)"},
"TimeStamp":"2026-06-30T12:00:00Z","Bearing":270,"Speed":118,"Status":{"Active":true}}
]}]}}"#;
/// OSM admin features — the `admin_unit_from_tags` input, as a JSON array of
/// `{id, tags}` (tags = the OSM tag map). Four boundaries (SE ×3, NO ×1) + a
/// non-boundary road that the filter drops.
pub const SYNTHETIC_OSM_ADMIN_JSON: &str = r#"[
{"id":1,"tags":{"boundary":"administrative","admin_level":"7","name":"Solna","ISO3166-2":"SE-AB"}},
{"id":2,"tags":{"boundary":"administrative","admin_level":"7","name":"Sundbyberg","ISO3166-2":"SE-AB"}},
{"id":3,"tags":{"boundary":"administrative","admin_level":"4","name":"Stockholms län","ISO3166-2":"SE-AB"}},
{"id":4,"tags":{"boundary":"administrative","admin_level":"7","name":"Oslo","ISO3166-2":"NO-03"}},
{"id":5,"tags":{"highway":"primary"}}
]"#;
/// The bundled tiny real-OSM extract (4 nodes / 2 ways, 454-byte zstd PBF —
/// generated offline with osm-katana `xml2-pbf`). Real OSM bytes, no network.
#[cfg(feature = "osm")]
pub const SYNTHETIC_OSM_PBF: &[u8] = include_bytes!("fixtures/testland-latest.osm.pbf");
// ═══════════════════════════════════════════════════════════════════════════
// Per-collector real-or-fake sample sources.
// ═══════════════════════════════════════════════════════════════════════════
/// **DCAT-AP-SE** — attempt the nightly RDF/XML dump (byte-range-capped so a
/// multi-MB dump can't blow the timeout), validate it parses to ≥1 registry row,
/// else the synthetic catalogue.
pub fn dcat_try_get_real_sample_data() -> Sample {
let url = "https://admin.dataportal.se/all.rdf";
let fetched = http_get(url, Some(256 * 1024));
real_or_synthetic(
url,
fetched,
|b| {
crate::dcat::parse_dcat_rdfxml(b)
.map(|r| !r.is_empty())
.unwrap_or(false)
},
SYNTHETIC_DCAT_RDFXML.as_bytes(),
)
}
/// **PxWeb / JSON-stat2** — POST a JSON-stat2 selection to SCB, validate it
/// normalises to a non-empty cube, else the synthetic cube.
pub fn pxweb_try_get_real_sample_data() -> Sample {
let url = "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy";
// Minimal selection: latest year, one region, one measure, JSON-stat2 out.
let query = r#"{"query":[{"code":"Region","selection":{"filter":"item","values":["0114"]}},{"code":"Tid","selection":{"filter":"top","values":["1"]}}],"response":{"format":"json-stat2"}}"#;
let fetched = http_post(url, query.to_string(), "application/json");
real_or_synthetic(
url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::pxweb::parse_jsonstat2(&v).ok())
.map(|batch| batch.num_rows() > 0)
.unwrap_or(false)
},
SYNTHETIC_PXWEB_JSONSTAT2.as_bytes(),
)
}
/// **Eurostat SDMX-JSON 2.1** — attempt a live SDMX 2.1 REST query (open, no key;
/// byte-range-capped so a large cube can't blow the timeout), validate it flattens
/// to ≥1 observation row, else the synthetic 2×2 message. A small population cube
/// (`demo_pjan`, latest, SE) keeps the response tiny.
pub fn eurostat_try_get_real_sample_data() -> Sample {
let url = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/data/\
demo_pjan/A.NR.TOTAL.T.SE?format=jsondata&lastTimePeriod=1";
let fetched = http_get(url, Some(512 * 1024));
real_or_synthetic(
url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::sdmx::parse_sdmx_json(&v).ok())
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
SYNTHETIC_EUROSTAT_SDMX_JSON.as_bytes(),
)
}
/// A shared SDMX real-or-fake: GET `url` → validate it flattens to ≥1 observation
/// row via the shared [`crate::sdmx::parse_sdmx_json`], else the synthetic body.
fn sdmx_sample(url: &str, synthetic: &[u8]) -> Sample {
let fetched = http_get(url, Some(512 * 1024));
real_or_synthetic(
url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::sdmx::parse_sdmx_json(&v).ok())
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
/// **INSEE** (France BDM SDMX) — the concrete dataflow + series key is
/// deployment-specific, so absent `INSEE_SDMX_URL` this returns the synthetic
/// national SDMX message with no network; with the env var set it attempts that
/// SDMX-JSON query and validates it flattens to ≥1 observation, else falls back.
pub fn insee_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes();
match std::env::var("INSEE_SDMX_URL") {
Ok(url) if !url.is_empty() => sdmx_sample(&url, synthetic),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **ONS** (UK Beta SDMX) — dataflow-specific, so absent `ONS_SDMX_URL` it stays
/// synthetic; with the env var set it attempts that SDMX-JSON query, else falls back.
pub fn ons_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes();
match std::env::var("ONS_SDMX_URL") {
Ok(url) if !url.is_empty() => sdmx_sample(&url, synthetic),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **ISTAT** (Italy SDMX) — dataflow-specific, so absent `ISTAT_SDMX_URL` it stays
/// synthetic; with the env var set it attempts that SDMX-JSON query, else falls back.
pub fn istat_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes();
match std::env::var("ISTAT_SDMX_URL") {
Ok(url) if !url.is_empty() => sdmx_sample(&url, synthetic),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **World Bank** (SDMX) — dataflow-specific, so absent `WORLDBANK_SDMX_URL` it
/// stays synthetic; with the env var set it attempts that SDMX-JSON query, else
/// falls back.
pub fn worldbank_sdmx_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes();
match std::env::var("WORLDBANK_SDMX_URL") {
Ok(url) if !url.is_empty() => sdmx_sample(&url, synthetic),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **ENTSO-E** — needs a free token (`ENTSOE_TOKEN`); with one, pull a day-ahead
/// price document for SE3 and validate ≥1 point, else the synthetic document.
pub fn entsoe_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_ENTSOE_XML.as_bytes();
let token = std::env::var("ENTSOE_TOKEN").unwrap_or_default();
if token.is_empty() {
return Sample::synthetic(synthetic.to_vec());
}
let se3 = "10Y1001A1001A46L";
let url = format!(
"https://web-api.tp.entsoe.eu/api?securityToken={token}&documentType=A44\
&in_Domain={se3}&out_Domain={se3}&periodStart=202601010000&periodEnd=202601020000"
);
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
let xml = String::from_utf8_lossy(b);
crate::entsoe::parse_entsoe_xml(&xml)
.map(|p| !p.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
/// **ENTSO-E day-ahead PRICES** — needs a free token (`ENTSOE_TOKEN`); with one,
/// pull an `A44` day-ahead price document for SE3 and validate it parses to ≥1
/// price point (via the `price.amount` parser), else the synthetic A44 document.
pub fn entsoe_day_ahead_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_ENTSOE_A44_XML.as_bytes();
let token = std::env::var("ENTSOE_TOKEN").unwrap_or_default();
if token.is_empty() {
return Sample::synthetic(synthetic.to_vec());
}
let se3 = "10Y1001A1001A46L";
let url = format!(
"https://web-api.tp.entsoe.eu/api?securityToken={token}&documentType=A44\
&in_Domain={se3}&out_Domain={se3}&periodStart=202601010000&periodEnd=202601020000"
);
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
let xml = String::from_utf8_lossy(b);
crate::entsoe_day_ahead::parse_entsoe_day_ahead_xml(&xml)
.map(|p| !p.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
/// A shared REST real-or-fake: GET `url` → validate it maps to ≥1 row via the
/// preset's `records_pointer` + field map, else the synthetic body.
fn rest_sample(collector: &crate::rest::RestCollector, url: &str, synthetic: &[u8]) -> Sample {
rest_sample_headers(collector, url, &[], synthetic)
}
/// [`rest_sample`] with constant request headers — for a source that mandates an
/// identification header rather than a credential (Digitraffic's `Digitraffic-User`;
/// without it the gateway answers 502, measured 2026-09-03). One body, two entry
/// points: `rest_sample` is this with an empty header list.
fn rest_sample_headers(
collector: &crate::rest::RestCollector,
url: &str,
headers: &[(&str, &str)],
synthetic: &[u8],
) -> Sample {
let pointer = collector.records_pointer().to_string();
let fields = collector.fields().to_vec();
let fetched = if headers.is_empty() {
http_get(url, Some(512 * 1024))
} else {
http_get_headers(url, Some(512 * 1024), headers)
};
real_or_synthetic(
url,
fetched,
move |b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::rest::parse_records(&v, &pointer, &fields).ok())
.map(|batch| batch.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
/// **A CSV preset's sample** — the flat-file sibling of [`rest_sample`].
///
/// The validation is the parser's OWN sniff-and-parse over the fetched bytes, so a
/// source that changed its ENCODING (which is precisely the hazard here) falls back to
/// the fixture instead of landing garbage: a UTF-16 export served as UTF-8 would parse
/// into columns named `P\0u\0b\0…` and this rejects it.
#[cfg(feature = "csv")]
fn csv_sample(collector: &crate::csv::CsvCollector, url: &str, synthetic: &[u8]) -> Sample {
let _ = collector;
real_or_synthetic(
url,
http_get(url, Some(2 * 1024 * 1024)),
|b| {
let d = crate::csv::sniff_dialect(b);
crate::csv::parse_csv(b, &d)
.map(|(schema, batches)| {
let rows: usize = batches.iter().map(|x| x.num_rows()).sum();
// A column name carrying a NUL is the signature of a UTF-16 file
// read as UTF-8 — reject it rather than record it as "real".
let clean = schema.fields().iter().all(|f| !f.name().contains('\0'));
rows > 0 && clean
})
.unwrap_or(false)
},
synthetic,
)
}
/// **Polisen** events — a public no-auth JSON array (a strong real candidate).
pub fn polisen_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::polisen(),
"https://polisen.se/api/events",
SYNTHETIC_POLISEN_JSON.as_bytes(),
)
}
/// **Kolada** KPI catalogue — a public no-auth `values` array.
pub fn kolada_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::kolada(),
"https://api.kolada.se/v3/kpi?per_page=5",
SYNTHETIC_KOLADA_JSON.as_bytes(),
)
}
/// **SMHI** metobs — latest air-temperature values for a fixed station.
pub fn smhi_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::smhi_metobs(),
"https://opendata-download-metobs.smhi.se/api/version/1.0/parameter/1/station/98230/period/latest-hour/data.json",
SYNTHETIC_SMHI_JSON.as_bytes(),
)
}
/// **SMHI hydrological observations** (WATER) — public no-auth; latest water
/// readings for a fixed station-set. Falls back to the synthetic `value` fixture.
pub fn smhi_hydro_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::smhi_hydro(),
"https://opendata-download-hydroobs.smhi.se/api/version/latest/parameter/1/station/2237/period/latest-day/data.json",
SYNTHETIC_SMHI_HYDRO_JSON.as_bytes(),
)
}
/// **Met.no Locationforecast** (WEATHER) — public no-auth point forecast (an
/// identifying User-Agent is required; the sample client sends one). Oslo point.
pub fn metno_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::metno_forecast(),
"https://api.met.no/weatherapi/locationforecast/2.0/compact?lat=59.91&lon=10.75",
SYNTHETIC_METNO_JSON.as_bytes(),
)
}
/// **GBIF** species occurrences — public no-auth biodiversity search. Falls back
/// to the synthetic two-occurrence `results` envelope.
pub fn gbif_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::gbif(),
"https://api.gbif.org/v1/occurrence/search?country=SE&hasCoordinate=true&limit=20",
SYNTHETIC_GBIF_JSON.as_bytes(),
)
}
/// **World Bank** indicators — public no-auth; total population, latest years.
/// The `[meta, data]` envelope's element 1 is the observation array.
pub fn worldbank_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::worldbank(),
"https://api.worldbank.org/v2/country/SE/indicator/SP.POP.TOTL?format=json&per_page=5",
SYNTHETIC_WORLDBANK_JSON.as_bytes(),
)
}
/// **RESTCountries** — public no-auth country reference (name, ISO codes, region,
/// population, centroid). Falls back to the synthetic two-country array.
pub fn restcountries_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::restcountries(),
"https://restcountries.com/v3.1/all?fields=name,cca2,cca3,region,subregion,population,latlng",
SYNTHETIC_RESTCOUNTRIES_JSON.as_bytes(),
)
}
/// **Mobility Database** feed catalogue — the CC0 successor to TransitFeeds. The
/// catalogue API is bearer-gated: absent `MOBILITY_DB_TOKEN` it stays synthetic
/// (no network); with the token set it attempts `/v1/gtfs_feeds`, validates it
/// parses to ≥1 feed-descriptor row, else falls back to the synthetic two-feed
/// array.
pub fn mobility_database_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_MOBILITY_DB_JSON.as_bytes();
match std::env::var("MOBILITY_DB_TOKEN") {
Ok(tok) if !tok.is_empty() => {
let url = "https://api.mobilitydatabase.org/v1/gtfs_feeds?limit=20";
let bearer = format!("Bearer {tok}");
let fetched = http_get_headers(url, Some(512 * 1024), &[("Authorization", &bearer)]);
let c = crate::rest::RestCollector::mobility_database();
let pointer = c.records_pointer().to_string();
let fields = c.fields().to_vec();
real_or_synthetic(
url,
fetched,
move |b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::rest::parse_records(&v, &pointer, &fields).ok())
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **Mobility Database — GTFS-Realtime feed catalogue** — the CC0 successor to
/// TransitFeeds, bearer-gated: absent `MOBILITY_DB_TOKEN` it stays synthetic (no
/// network); with the token set it attempts `/v1/gtfs_rt_feeds`, validates it
/// parses to ≥1 RT feed-descriptor row, else falls back to the synthetic two-feed
/// array.
pub fn mobility_database_rt_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_MOBILITY_DB_RT_JSON.as_bytes();
match std::env::var("MOBILITY_DB_TOKEN") {
Ok(tok) if !tok.is_empty() => {
let url = "https://api.mobilitydatabase.org/v1/gtfs_rt_feeds?limit=20";
let bearer = format!("Bearer {tok}");
let fetched = http_get_headers(url, Some(512 * 1024), &[("Authorization", &bearer)]);
let c = crate::rest::RestCollector::mobility_database_rt();
let pointer = c.records_pointer().to_string();
let fields = c.fields().to_vec();
real_or_synthetic(
url,
fetched,
move |b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::rest::parse_records(&v, &pointer, &fields).ok())
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **EEA WISE water quality** (Discodata) — a public no-auth SQL REST endpoint
/// (a strong real candidate). Attempts a tiny `TOP 5` Waterbase query
/// (byte-range-capped), validates it maps to ≥1 row via the preset's
/// `/results` pointer + field map, else the synthetic two-determinand fixture.
/// Overridable via `EEA_WISE_URL`.
pub fn eea_wise_try_get_real_sample_data() -> Sample {
let default_url = "https://discodata.eea.europa.eu/sql?query=SELECT%20TOP%205%20*%20FROM%20%5BWISE_SOE%5D.%5Bv2r1%5D.%5BWaterbase_S_WISE6_AggregatedData%5D";
let url = std::env::var("EEA_WISE_URL").ok().filter(|u| !u.is_empty());
let url = url.as_deref().unwrap_or(default_url);
rest_sample(
&crate::rest::RestCollector::eea_wise(),
url,
SYNTHETIC_EEA_WISE_JSON.as_bytes(),
)
}
/// **Destatis GENESIS** table catalogue — the GENESIS-Online REST API is gated by
/// a free account token, so absent `DESTATIS_GENESIS_TOKEN` this returns the
/// synthetic two-table `List` with no network; with the token set it attempts the
/// `catalogue/tables` endpoint (token as a query param), validates it maps to ≥1
/// descriptor row, else falls back.
pub fn destatis_genesis_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_DESTATIS_GENESIS_JSON.as_bytes();
match std::env::var("DESTATIS_GENESIS_TOKEN") {
Ok(tok) if !tok.is_empty() => {
let url = format!(
"https://www-genesis.destatis.de/genesisWS/rest/2020/catalogue/tables?\
token={tok}&selection=*&pagelength=20&language=en"
);
rest_sample(
&crate::rest::RestCollector::destatis_genesis(),
&url,
synthetic,
)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **CBS Open Data** (Statistics Netherlands) OData — a public no-auth endpoint
/// (a strong real candidate). Attempts a small `TypedDataSet` page
/// (byte-range-capped) for a population table, validates it maps to ≥1 row via
/// the preset's `/value` pointer + field map, else the synthetic two-observation
/// fixture. Overridable via `CBS_ODATA_URL`.
pub fn cbs_odata_try_get_real_sample_data() -> Sample {
let default_url = "https://opendata.cbs.nl/ODataApi/odata/37296eng/TypedDataSet?%24top=5";
let url = std::env::var("CBS_ODATA_URL")
.ok()
.filter(|u| !u.is_empty());
let url = url.as_deref().unwrap_or(default_url);
rest_sample(
&crate::rest::RestCollector::cbs_odata(),
url,
SYNTHETIC_CBS_ODATA_JSON.as_bytes(),
)
}
/// **ERDDAP** — a public no-auth scientific-data server (a strong real
/// candidate). Attempts a tiny NOAA/CoastWatch tabledap `.json` query (a single
/// station, one column set, byte-range-capped so a busy dataset can't blow the
/// timeout), validates it decodes to ≥1 typed row, else the synthetic
/// two-buoy `table` document. Overridable via `ERDDAP_URL`.
pub fn erddap_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_ERDDAP_JSON.as_bytes();
// A small, stable CoastWatch NDBC query (latest hour, a few columns).
let default_url = "https://coastwatch.pfeg.noaa.gov/erddap/tabledap/cwwcNDBCMet.json?\
station,time,latitude,longitude,wtmp&time%3E=now-2hours";
let url = std::env::var("ERDDAP_URL").ok().filter(|u| !u.is_empty());
let url = url.as_deref().unwrap_or(default_url);
let fetched = http_get(url, Some(512 * 1024));
real_or_synthetic(
url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::erddap::parse_erddap_json(&v).ok())
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
/// A synthetic single-message GRIB2 (ERA5-like global lat/lon field) built by the
/// [`crate::gribnc`] encoder — a REAL GRIB2 byte stream, no committed blob. The
/// decoder recovers its header metadata (parameter, reference time, grid, bbox).
#[cfg(feature = "gribnc")]
pub fn synthetic_grib2() -> Vec<u8> {
crate::gribnc::build_grib2_message(&crate::gribnc::GribSpec {
discipline: 0,
parameter_category: 0,
parameter_number: 0,
year: 2026,
month: 7,
day: 18,
hour: 0,
ni: 360,
nj: 181,
lat_first: 90.0,
lon_first: 0.0,
lat_last: -90.0,
lon_last: 359.0,
})
}
/// **DWD ICON** open data — public no-auth GRIB2 forecast fields (a real
/// candidate). Attempts the caller-supplied `DWD_ICON_URL` GRIB2 file
/// (byte-range-capped so a full field can't blow the timeout), validates it
/// decodes to ≥1 GRIB2 message header, else the synthetic single-message GRIB2.
/// Absent the env var it stays synthetic (the ICON path/run/step is
/// deployment-specific).
#[cfg(feature = "gribnc")]
pub fn dwd_icon_try_get_real_sample_data() -> Sample {
let synthetic = synthetic_grib2();
match std::env::var("DWD_ICON_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
crate::gribnc::parse_grib2(b)
.map(|m| !m.is_empty())
.unwrap_or(false)
},
&synthetic,
)
}
_ => Sample::synthetic(synthetic),
}
}
/// **Copernicus CDS** — the retrieve API is key-gated (`CDSAPI_KEY`) and runs an
/// async job before a download URL exists, so the sample stays synthetic unless
/// `CDS_SAMPLE_URL` names a direct GRIB/NetCDF file; then it validates the
/// container header decodes to ≥1 catalog row, else falls back. Never hits the
/// network without the env var (deterministic, no credential needed offline).
#[cfg(feature = "gribnc")]
pub fn copernicus_cds_try_get_real_sample_data() -> Sample {
let synthetic = synthetic_grib2();
match std::env::var("CDS_SAMPLE_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(1024 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
crate::gribnc::parse_weather_bytes(b, "cds")
.map(|r| !r.is_empty())
.unwrap_or(false)
},
&synthetic,
)
}
_ => Sample::synthetic(synthetic),
}
}
/// A synthetic 2×2 **simple-packed** GRIB2 field carrying real integer values
/// (`build_grib2_simple`) — a REAL packed byte stream, no committed blob, that
/// round-trips through the VALUE decode (`parse_weather_values`). The ERA5 / CAMS
/// value collectors' offline stand-in.
#[cfg(feature = "gribnc")]
pub fn synthetic_grib2_values() -> Vec<u8> {
crate::gribnc::build_grib2_simple(
&crate::gribnc::GribSpec {
discipline: 0,
parameter_category: 0,
parameter_number: 0,
year: 2026,
month: 7,
day: 18,
hour: 0,
ni: 2,
nj: 2,
lat_first: 60.0,
lon_first: 10.0,
lat_last: 59.0,
lon_last: 11.0,
},
&[10, 20, 30, 40],
&[true; 4],
)
}
/// **Copernicus ERA5** climate reanalysis — the CDS `retrieve/v1` endpoint is
/// key-gated (`CDSAPI_KEY`) and runs an async job before a download URL exists,
/// so the sample stays synthetic unless `ERA5_SAMPLE_URL` names a direct
/// GRIB/NetCDF file; then it validates the container unpacks to ≥1 grid VALUE
/// row, else falls back to the synthetic simple-packed GRIB2. Never hits the
/// network without the env var (deterministic offline).
#[cfg(feature = "gribnc")]
pub fn copernicus_era5_try_get_real_sample_data() -> Sample {
let synthetic = synthetic_grib2_values();
match std::env::var("ERA5_SAMPLE_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(1024 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
crate::gribnc::parse_weather_values(b, None)
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
&synthetic,
)
}
_ => Sample::synthetic(synthetic),
}
}
/// **Copernicus CAMS** atmosphere / air-quality reanalysis — the ADS
/// `retrieve/v1` endpoint is key-gated (`ADSAPI_KEY`) with the same async-job
/// flow, so the sample stays synthetic unless `CAMS_SAMPLE_URL` names a direct
/// GRIB/NetCDF file; then it validates the container unpacks to ≥1 grid VALUE
/// row, else falls back to the synthetic simple-packed GRIB2.
#[cfg(feature = "gribnc")]
pub fn copernicus_cams_try_get_real_sample_data() -> Sample {
let synthetic = synthetic_grib2_values();
match std::env::var("CAMS_SAMPLE_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(1024 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
crate::gribnc::parse_weather_values(b, None)
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
&synthetic,
)
}
_ => Sample::synthetic(synthetic),
}
}
/// **OGC WFS** — attempt a live `GetFeature` (byte-range-capped so a large
/// FeatureCollection can't blow the budget) from a caller-supplied `WFS_URL`,
/// validate it parses to ≥1 feature, else the synthetic GML collection. Absent
/// the env var it stays synthetic (WFS endpoints/typeNames are deployment-specific).
pub fn wfs_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_WFS_GML.as_bytes();
match std::env::var("WFS_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
let xml = String::from_utf8_lossy(b);
crate::wfs::parse_wfs_gml(&xml)
.map(|f| !f.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **Nord Pool** day-ahead prices — licensed (paid / data agreement), so absent
/// `NORDPOOL_API_URL` it returns the synthetic 2-period × 3-zone fixture with no
/// network; with the env var set it attempts that URL and validates it flattens
/// to ≥1 price row, else falls back.
pub fn nordpool_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NORDPOOL_JSON.as_bytes();
match std::env::var("NORDPOOL_API_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::nordpool::parse_nordpool_json(&v).ok())
.map(|p| !p.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **USGS earthquakes** — a public, no-auth GeoJSON feed (a strong real
/// candidate): attempt the past-hour summary (byte-range-capped so a busy hour
/// can't blow the timeout), validate it flattens to ≥1 locatable quake row, else
/// the synthetic two-feature collection.
pub fn usgs_try_get_real_sample_data() -> Sample {
let url = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson";
let fetched = http_get(url, Some(512 * 1024));
real_or_synthetic(
url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::usgs::parse_usgs_geojson(&v).ok())
.map(|q| !q.is_empty())
.unwrap_or(false)
},
SYNTHETIC_USGS_GEOJSON.as_bytes(),
)
}
/// **Energy-Charts** generation mix — a public, no-auth CC-BY API (a strong real
/// candidate): attempt the German public-power document (byte-range-capped so a
/// long window can't blow the timeout), validate it transposes to ≥1 numeric
/// `(time, type)` power cell, else the synthetic three-type document.
pub fn energy_charts_try_get_real_sample_data() -> Sample {
let url = "https://api.energy-charts.info/public_power?country=de";
let fetched = http_get(url, Some(512 * 1024));
real_or_synthetic(
url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::energy_charts::parse_energy_charts_json(&v).ok())
.map(|r| !r.is_empty())
.unwrap_or(false)
},
SYNTHETIC_ENERGY_CHARTS_JSON.as_bytes(),
)
}
/// Synthesise a GTFS-Realtime `VehiclePositions` feed (a real protobuf
/// `FeedMessage`, header + two vehicles) with the [`crate::gtfs_rt::build_gtfs_rt`]
/// encoder — no committed binary blob. Round-trips through `parse_gtfs_rt`.
pub fn synthetic_gtfs_rt_pb() -> Vec<u8> {
use crate::gtfs_rt::VehicleFixture;
crate::gtfs_rt::build_gtfs_rt(&[
VehicleFixture {
entity_id: "veh-1",
vehicle_id: Some("9031001"),
trip_id: Some("trip-A"),
route_id: Some("17"),
latitude: 59.33107,
longitude: 18.05977,
bearing: Some(137.5),
speed: Some(11.4),
timestamp: Some(1_752_400_123),
stop_id: Some("9022001"),
},
VehicleFixture {
entity_id: "veh-2",
vehicle_id: Some("9031002"),
trip_id: Some("trip-B"),
route_id: Some("96"),
latitude: 57.70750,
longitude: 11.96750,
timestamp: Some(1_752_400_130),
..Default::default()
},
])
}
/// **GTFS-Realtime** vehicle positions — a binary protobuf feed. Public agency
/// feeds are usually no-auth but the exact `.pb` URL is deployment-specific, so
/// absent `GTFS_RT_URL` this returns the synthetic two-vehicle `FeedMessage` with
/// no network; with the env var set it attempts that URL (byte-capped) and
/// validates it decodes to ≥1 locatable vehicle, else falls back.
pub fn gtfs_rt_try_get_real_sample_data() -> Sample {
let synthetic = synthetic_gtfs_rt_pb();
match std::env::var("GTFS_RT_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(1024 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
crate::gtfs_rt::parse_gtfs_rt(b)
.map(|v| !v.is_empty())
.unwrap_or(false)
},
&synthetic,
)
}
_ => Sample::synthetic(synthetic),
}
}
/// **RSS / Atom feed** — public government/agency feeds are no-auth, and this one now
/// has a REAL default: regeringen.se's press-release feed
/// ([`crate::rss::REGERINGEN_PRESSMEDDELANDEN_FEED`]), keyless and with no agreement.
/// `RSS_FEED_URL` still overrides it.
///
/// It used to return the synthetic two-item channel unconditionally unless an operator
/// happened to set the env var, which meant the ONE generic collector korp configures
/// five real government feeds against was also the one whose pane showed a fixture. The
/// live feed answers 100 `<item>`s in 111 662 bytes (measured 2026-09-05), well inside
/// the 512 KB byte cap, and the validation is the same as before — it must flatten to
/// ≥1 row through this crate's own parser, or the synthetic wins.
pub fn rss_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_RSS_XML.as_bytes();
let url = match std::env::var("RSS_FEED_URL") {
Ok(u) if !u.is_empty() => u,
_ => crate::rss::REGERINGEN_PRESSMEDDELANDEN_FEED.to_string(),
};
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
let xml = String::from_utf8_lossy(b);
crate::rss::parse_feed(&xml)
.map(|i| !i.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
/// **OpenAQ** air quality — v3 needs a free self-serve API key carried as the
/// `X-API-Key` header, and the exact query is deployment-specific, so absent
/// `OPENAQ_API_URL` this returns the synthetic two-reading envelope with no
/// network; with the env var set it attempts that URL (byte-range-capped) and
/// validates it flattens to ≥1 measurement row, else falls back.
pub fn openaq_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_OPENAQ_JSON.as_bytes();
match std::env::var("OPENAQ_API_URL") {
Ok(url) if !url.is_empty() => {
// OpenAQ v3 gates on a free `X-API-Key`; send it from env when present
// (keyless requests 401). Secret stays in env, never hardcoded.
let key = std::env::var("OPENAQ_API_KEY").unwrap_or_default();
let fetched = if key.is_empty() {
http_get(&url, Some(512 * 1024))
} else {
http_get_headers(&url, Some(512 * 1024), &[("X-API-Key", &key)])
};
real_or_synthetic(
&url,
fetched,
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::openaq::parse_openaq_json(&v).ok())
.map(|m| !m.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **A REAL two-record GLEIF page**, read live from
/// `api.gleif.org/api/v1/lei-records?filter[entity.legalAddress.country]=SE` on
/// 2026-09-04 and trimmed to the fields the preset maps.
///
/// **`links` is EMPTY on purpose — the last-page shape.** Its first version carried the
/// real `links.next`, an ABSOLUTE url on `api.gleif.org`, and the registry sweep then
/// followed it: it served this fixture from a local port, read the `next`, and crawled
/// the LIVE API from page 2 to page 667 before a 400 stopped it. A fixture with an
/// absolute cursor drags every consumer to the wire. The pagination is proven where it
/// belongs — `tests/rest_pagination.rs`, over a three-page fixture whose urls are all
/// local.
///
/// The two orgnrs are hyphenated (`559586-2557`, `559556-1605`) exactly as GLEIF
/// serves them, which is the whole reason `orgnr` is a derived digits-only column: a
/// join against Bolagsverket's `orgnr` on the raw value matches NOTHING and says
/// nothing.
pub const SYNTHETIC_GLEIF_LEI_JSON: &str = r#"{"data":[
{"type":"lei-records","id":"9845001T0D83C4FAF687",
"attributes":{"lei":"9845001T0D83C4FAF687",
"entity":{"legalName":{"name":"Zesoz AB","language":"sv"},
"registeredAs":"559586-2557","status":"ACTIVE",
"legalForm":{"id":"XJHM","other":null},
"legalAddress":{"city":"STOCKHOLM","postalCode":"114 59","country":"SE"}},
"registration":{"status":"ISSUED","lastUpdateDate":"2026-09-03T20:25:35Z",
"managingLou":"529900T8BM49AURSDO55"},
"bic":null}},
{"type":"lei-records","id":"984500B85AS78A9DFF57",
"attributes":{"lei":"984500B85AS78A9DFF57",
"entity":{"legalName":{"name":"Magneten i Ume\u00e5 AB","language":"sv"},
"registeredAs":"559556-1605","status":"ACTIVE",
"legalForm":{"id":"XJHM","other":null},
"legalAddress":{"city":"UME\u00c5","postalCode":"903 20","country":"SE"}},
"registration":{"status":"ISSUED","lastUpdateDate":"2026-09-03T20:19:53Z",
"managingLou":"529900T8BM49AURSDO55"},
"bic":null}}],
"links":{}}"#;
/// **Two REAL Domstolsverket publications**, read live 2026-09-05 from
/// `rattspraxis.etjanst.domstol.se/api/v1/publiceringar` and kept verbatim except for
/// `innehall`, which is truncated to its first 120 characters with an ellipsis.
///
/// The truncation is the point of the fixture as much as the records are. The preset
/// maps `innehall` through [`crate::rest::FieldType::TextLen`] — it lands the character
/// COUNT and never the text — because one single judgment in the 610-publication slice
/// this pair came from carries **24 552 235** characters, 27.5 MB of that slice's 34.8
/// MB in one cell. A fixture that embedded a real `innehall` would be a 24 MB source
/// file; one that omitted the field entirely could not prove the column lands at all.
///
/// The document is a bare ARRAY, which is the endpoint's real shape (`records_pointer`
/// is `""`). Both records carry a multi-valued `nyckelordLista` (5 and 4 keywords) and a
/// `lagrumLista` of one and two statutes — the repeated fields that
/// [`crate::rest::FieldType::Joined`] exists for — and an EMPTY `bilagaLista`, so the
/// `bilaga_*` columns land NULL, which is what a referat without a PDF honestly is.
pub const SYNTHETIC_DOMSTOL_RATTSPRAXIS_JSON: &str = r#"[
{"id":"9962aaac-92a9-4422-8188-baa9c0bde0f6",
"gruppKorrelationsnummer":"cb8c0c8d-5396-4de4-b924-a4d46b20bb26",
"domstol":{"domstolKod":"HFD","domstolNamn":"Högsta förvaltningsdomstolen"},
"typ":"VAGLEDANDE_MEN_EJ_PREJUDICERANDE","publiceringsform":"REFERAT",
"avgorandedatum":"2026-06-10","publiceringstid":"2026-09-03T16:24:00",
"malNummerLista":["4566-25"],"referatNummerLista":["HFD 2026 ref. 31"],
"rattsomradeLista":["Skatt"],
"nyckelordLista":["Inkomstskatt","Inkomstslaget näringsverksamhet",
"Förhandsbesked om inkomstskatt","Uttag ur näringsverksamhet",
"Underprisöverlåtelse"],
"forarbeteLista":[],"litteraturLista":[],"hanvisadePubliceringarLista":[],
"europarattsligaAvgorandenLista":[],"bilagaLista":[],
"lagrumLista":[{"sfsNummer":"1999:1229","referens":"2 kap. 24 och 25 §§, 22 kap. 3 och 7 §§, 23 kap. 17 § inkomstskattelagen (1999:1229)"}],
"sammanfattning":"Aktier i ett fastighetsförvaltande företag som utgör lagertillgångar hos innehavaren enligt bestämmelserna om byggnadsrörelse har inte ansetts utgöra en verksamhetsgren. Förhandsbesked om inkomstskatt.",
"innehall":"<h1>Ref 31</h1><p><strong>Aktier i ett fastighetsförvaltande företag som utgör lagertillgångar hos innehavaren enligt b …"},
{"id":"76b6e0ac-3ace-456f-862f-65efb80217f7",
"gruppKorrelationsnummer":"8cfef5b9-26bb-4eb6-ac68-b1911c3d7b40",
"domstol":{"domstolKod":"HFD","domstolNamn":"Högsta förvaltningsdomstolen"},
"typ":"VAGLEDANDE_MEN_EJ_PREJUDICERANDE","publiceringsform":"REFERAT",
"avgorandedatum":"2026-06-05","publiceringstid":"2026-09-03T16:13:00",
"malNummerLista":["6143-25"],"referatNummerLista":["HFD 2026 ref. 30"],
"rattsomradeLista":["Socialtjänst"],
"nyckelordLista":["Ekonomiskt bistånd","Försörjningsstöd",
"Utredningsansvar","bevisbörda"],
"forarbeteLista":[],"litteraturLista":[],"hanvisadePubliceringarLista":[],
"europarattsligaAvgorandenLista":[],"bilagaLista":[],
"lagrumLista":[{"sfsNummer":"2017:900","referens":"23 § förvaltningslagen "},
{"sfsNummer":"2001:453","referens":"4 kap. 1 § socialtjänstlagen "}],
"sammanfattning":"Fråga om betydelsen av att ett fordon är registrerat i vägtrafikregistret på någon annan än den sökande vid bedömningen av rätten till ekonomiskt bistånd.",
"innehall":"<h1>Ref 30</h1><p><strong>Fråga om betydelsen av att ett fordon är registrerat i vägtrafikregistret på någon annan än de …"}
]"#;
/// **Two REAL rows of Finansinspektionen's `Insyn` export**, read live 2026-09-04 and
/// kept verbatim — header plus the two most recent transactions at the time.
///
/// Held as UTF-8 TEXT and re-encoded to UTF-16LE by
/// [`fi_insyn_try_get_real_sample_data`], because the ENCODING is the thing this
/// collector exists to get right: FI serves UTF-16LE with **no BOM**, and a fixture
/// stored as UTF-8 would test the parser against a file shaped unlike the source.
///
/// Note what the rows themselves carry: `Korrigering = Ja` on both, a decimal-comma
/// `15000,0` volume, an empty 23rd field after the trailing `;`, and an issuer named
/// ONLY by LEI (`549300XZE8FG96FTL804`) — no orgnummer anywhere, which is the whole
/// reason `gleif-lei` had to exist first.
pub const SYNTHETIC_FI_INSYN_CSV: &str = "Publiceringsdatum;Emittent;LEI-kod;Anmälningsskyldig;Person i ledande ställning;Befattning;Närstående;Korrigering;Beskrivning av korrigering;Är förstagångsrapportering;Är kopplad till aktieprogram;Karaktär;Instrumenttyp;Instrumentnamn;ISIN;Transaktionsdatum;Volym;Volymsenhet;Pris;Valuta;Handelsplats;Status;\n\
2026-09-04 15:09:17;Harmoney BidCo;64887HUYX14K9T80J308;Nicolai Chamizo;Nicolai Chamizo;Styrelseordförande;;Ja;Ändring av siffror, karäktar, och belopp;;;Förvärv;Obligation;Obligation;SE0027598178;2026-09-04 00:00:00;1000000,0;Antal;1,007;EUR;SEB ENSKILDA;Aktuell;\n\
2026-09-04 15:04:11;Viva Wine Group AB;549300XZE8FG96FTL804;Fano AB;Lars Ljungälv;Styrelseledamot;Ja;Ja;Närstående;;;Avyttring;Aktie;Viva Wine Group AB;SE0017084361;2026-08-31 00:00:00;15000,0;Antal;38,50;SEK;Utanför handelsplats;Aktuell;\n";
/// **Finansinspektionen insynsregistret** — a public keyless CSV export, so this
/// attempts a REAL fetch unconditionally and falls back to the two recorded rows.
///
/// The fallback is re-encoded to **UTF-16LE with no BOM**, which is what FI actually
/// serves: a fixture in UTF-8 would silently pass a parser that cannot read the source.
#[cfg(feature = "csv")]
pub fn fi_insyn_try_get_real_sample_data() -> Sample {
let synthetic: Vec<u8> = SYNTHETIC_FI_INSYN_CSV
.encode_utf16()
.flat_map(|u| u.to_le_bytes())
.collect();
csv_sample(
&crate::csv::CsvCollector::fi_insyn(),
crate::csv::FI_INSYN_URL,
&synthetic,
)
}
/// **GLEIF LEI records** — a public **keyless** CC0 feed, so this attempts a REAL
/// fetch unconditionally and falls back only on a transport/parse failure.
///
/// It asks for `page[size]=2` **and caps the crawl at ONE page**. Both matter, and the
/// second one is a bug this sample found in its own first version: `rest_sample` drives
/// the real `collect()`, and `collect()` now follows `links.next` (lane EC O4), so the
/// sample WENT CRAWLING — it reached `page[number]=137` and came back **HTTP 429 Too
/// Many Requests**. A sample exists to prove the parser against the source's real
/// shape; a sample that reads the whole register to do it is a load test wearing a
/// fixture's name.
///
/// That 429 is also the measurement that corrected this collector's rate cap: GLEIF
/// serves **no** `x-ratelimit-*` or `retry-after` header, and the limit is real anyway.
/// Absent headers are not an absent limit.
pub fn gleif_lei_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_GLEIF_LEI_JSON.as_bytes();
let url = crate::rest::GLEIF_SWEDEN_URL.replace("page%5Bsize%5D=200", "page%5Bsize%5D=2");
rest_sample(
&crate::rest::RestCollector::gleif_lei().with_max_pages(1),
&url,
synthetic,
)
}
/// **Domstolsverket rättspraxis** — public, keyless and open, so this attempts a REAL
/// fetch unconditionally and falls back only on a transport/parse failure.
///
/// It asks for `pagesize=2` rather than the preset's 100. The sample exists to prove the
/// PARSER against the source's real shape, and pulling 100 publications to do that would
/// drag up to 26 MB of `innehall` (measured: one page of the live crawl weighed 26.7 MB)
/// through every test run that has a network.
///
/// [`rest_sample`] issues one plain GET and never enters `collect`'s crawl, so this
/// cannot walk the register the way the gleif sample once did.
pub fn domstol_rattspraxis_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_DOMSTOL_RATTSPRAXIS_JSON.as_bytes();
let url = crate::rest::DOMSTOL_RATTSPRAXIS_URL.replace("pagesize=100", "pagesize=2");
rest_sample(
&crate::rest::RestCollector::domstol_rattspraxis(),
&url,
synthetic,
)
}
/// **Riksdagen's document list**, as a two-record synthetic — one proposition and one
/// betänkande, in `dokumentlista.dokument` shape with the fields the preset maps.
///
/// The two rows differ on the axis that matters for the graph: `organ` is a spelled-out
/// department for the proposition and a utskott abbreviation for the betänkande, which
/// is the mixed namespace the mapping's `limitation` names.
pub const SYNTHETIC_RIKSDAGEN_JSON: &str = r#"{"dokumentlista":{
"@traffar":"31812","@sidor":"160","@traff_till":"2",
"dokument":[
{"dok_id":"HD03305","id":"hd03305","doktyp":"prop","subtyp":"prop",
"titel":"En mer enhetlig reglering av tillsyn och kontroll på miljöbalkens område",
"rm":"2025/26","beteckning":"305","nummer":"305",
"organ":"Klimat- och näringslivsdepartementet",
"datum":"2026-09-01","publicerad":"2026-09-01","systemdatum":"2026-09-02 08:43:01",
"status":"","kalla":"prop-RIM",
"dokument_url_html":"//data.riksdagen.se/dokument/HD03305.html",
"dokumentstatus_url_xml":"//data.riksdagen.se/dokumentstatus/HD03305.xml",
"summary":"Regeringens proposition 2025/26:305"},
{"dok_id":"HD01MJU12","id":"hd01mju12","doktyp":"bet","subtyp":"bet",
"titel":"Miljö- och jordbruksutskottets betänkande","rm":"2025/26",
"beteckning":"MJU12","nummer":"12","organ":"MJU",
"datum":"2026-08-25","publicerad":"2026-08-25","systemdatum":"2026-08-25 10:00:00",
"status":"behandlad","kalla":"bet",
"dokument_url_html":"//data.riksdagen.se/dokument/HD01MJU12.html",
"dokumentstatus_url_xml":"//data.riksdagen.se/dokumentstatus/HD01MJU12.xml",
"summary":"Utskottets betänkande"}
]}}"#;
/// **Riksdagen dokumentlista** — public, keyless and no agreement, so this attempts a
/// REAL fetch unconditionally and falls back only on a transport/parse failure.
///
/// It asks for `sz=2` **and caps the crawl at ONE page**, for exactly the reason
/// [`gleif_lei_try_get_real_sample_data`] does: `rest_sample` drives the real field map
/// over the fetched body, and this source's cursor never terminates on its own — page 51
/// onwards repeats page 50's rows with HTTP 200 and a fresh `@nasta_sida` (measured
/// 2026-09-05). A sample that walked that would read 10 000 documents to prove a field
/// map.
pub fn riksdagen_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_RIKSDAGEN_JSON.as_bytes();
let url = crate::rest::RIKSDAGEN_DOKUMENTLISTA_URL.replace("sz=200", "sz=2");
rest_sample(
&crate::rest::RestCollector::riksdagen_dokument().with_max_pages(1),
&url,
synthetic,
)
}
/// **A PxWeb navigation listing** — two subject areas and one table, in SCB's real
/// shape, read live 2026-09-05 from `api.scb.se/OV0104/v1/doris/sv/ssd` and
/// `.../NV/NV0101`.
///
/// The mix is the point. `AA` and `NV` are folders (`type: l`) and carry **no**
/// `updated`; `FDBR07N` is a table (`type: t`) and carries one. A fixture of three
/// folders could not prove that `updated` lands, and a fixture of three tables could not
/// prove that a folder's NULL `updated` is the source's answer rather than the walk's.
pub const SYNTHETIC_PXWEB_LISTING_JSON: &str = r#"[
{"id":"AA","type":"l","text":"Ämnesövergripande statistik"},
{"id":"NV","type":"l","text":"Näringsverksamhet"},
{"id":"FDBR07N","type":"t","updated":"2025-12-02T08:00:00",
"text":"Företag (FDB) efter näringsgren SNI2007 och storleksklass. År 2008 - 2025"}
]"#;
/// **The PxWeb table TREE** — public and keyless, so this attempts a REAL fetch of the
/// declared root and falls back to the three-node listing above.
///
/// It fetches the ROOT LISTING only, never the walk. `collect()` on this collector is
/// one request per folder — a full SCB walk is hundreds — and a sample exists to prove
/// the parser against the source's real shape, not to crawl a national catalogue on
/// every test run that happens to have a network. The validation is
/// [`crate::pxweb::parse_listing`] itself: a document that is not an array of nodes with
/// ids falls back rather than landing empty rows.
pub fn pxweb_tables_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_PXWEB_LISTING_JSON.as_bytes();
let url = crate::pxweb::SCB_SSD_ROOT;
real_or_synthetic(
url,
http_get(url, Some(256 * 1024)),
|b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::pxweb::parse_listing(&v, "", 1).ok())
.map(|n| !n.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
/// **SCB Företagsregistret** — the collector that CANNOT RUN, so its sample is
/// synthetic and stays synthetic.
///
/// There is nothing to attempt. SCB publishes no endpoint (it is issued with the
/// credential) and no response layout, so a "try the real thing" branch here would ask a
/// placeholder host and time out on every test run. The fixture is an EMPTY array,
/// which is the only honest shape: the collector's field map is empty too, precisely
/// because nobody has seen the real document. When the key arrives, both change
/// together.
pub fn scb_foretagsregistret_try_get_real_sample_data() -> Sample {
Sample::synthetic(b"[]".to_vec())
}
/// **Bolagsverket** company registry — agreement-gated (needs a signed data
/// agreement + key), so absent `BOLAGSVERKET_API_URL` it returns the synthetic
/// two-company fixture with no network; with the env var it attempts that URL.
pub fn bolagsverket_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_BOLAGSVERKET_JSON.as_bytes();
match std::env::var("BOLAGSVERKET_API_URL") {
Ok(url) if !url.is_empty() => {
rest_sample(&crate::rest::RestCollector::bolagsverket(), &url, synthetic)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **Digitraffic AIS** ship positions — a public **keyless** feed, so this attempts a
/// REAL fetch unconditionally and only falls back on a transport/parse failure.
///
/// Before 2026-09-03 there was nothing to attempt: the preset's endpoint was the
/// placeholder `ais-feed.example`, and this function returned the fixture unless an
/// operator named a mirror in `KORP_AIS_FEED_URL`. Digitraffic needs no key — only the
/// mandatory `Digitraffic-User` identification header, which is why this goes through
/// [`rest_sample_headers`]. `KORP_AIS_FEED_URL` still overrides, for a local mirror.
pub fn ais_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_AIS_JSON.as_bytes();
let url = match std::env::var("KORP_AIS_FEED_URL") {
Ok(u) if !u.is_empty() => u,
_ => crate::rest::DIGITRAFFIC_AIS_URL.to_string(),
};
rest_sample_headers(
&crate::rest::RestCollector::digitraffic_ais(),
&url,
&[("Digitraffic-User", crate::rest::DIGITRAFFIC_USER)],
synthetic,
)
}
/// **OpenSky flight TRACK** — the `/tracks` endpoint is credential-gated (HTTP
/// basic auth). Absent `OPENSKY_USER` + `OPENSKY_PASS` it stays synthetic (no
/// network); with both set it attempts the recorded track for a fixed aircraft,
/// validates the `path[]` parses to ≥1 ordered waypoint via the preset's field
/// map, else falls back to the synthetic 3-waypoint trajectory.
pub fn opensky_tracks_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_OPENSKY_TRACKS_JSON.as_bytes();
let user = std::env::var("OPENSKY_USER").unwrap_or_default();
let pass = std::env::var("OPENSKY_PASS").unwrap_or_default();
if user.is_empty() || pass.is_empty() {
return Sample::synthetic(synthetic.to_vec());
}
let collector = crate::rest::RestCollector::opensky_tracks();
// `time=0` = the aircraft's most recent track; icao24 fixed to a busy example.
let url = "https://opensky-network.org/api/tracks/all?icao24=4b1815&time=0";
let pointer = collector.records_pointer().to_string();
let fields = collector.fields().to_vec();
let fetched = http_get_basic(url, Some(512 * 1024), &user, &pass);
real_or_synthetic(
url,
fetched,
move |b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::rest::parse_records(&v, &pointer, &fields).ok())
.map(|batch| batch.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
/// **Trafikverket** TrainPosition — needs a key (`TRAFIKVERKET_API_KEY`) injected
/// into the XML body; with one, POST + validate ≥1 position row, else synthetic.
pub fn trafikverket_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_TRAFIKVERKET_JSON.as_bytes();
let collector = crate::trafikverket::TrafikverketCollector::train_positions();
let key = std::env::var("TRAFIKVERKET_API_KEY").unwrap_or_default();
if key.is_empty() {
return Sample::synthetic(synthetic.to_vec());
}
let url = "https://api.trafikinfo.trafikverket.se/v2/data.json";
let body = crate::trafikverket::build_request(
&key,
"<QUERY objecttype=\"TrainPosition\" schemaversion=\"1.9\" limit=\"5\"/>",
);
let pointer = collector.records_pointer();
let fields = collector.fields().to_vec();
let fetched = http_post(url, body, "text/xml");
real_or_synthetic(
url,
fetched,
move |b| {
serde_json::from_slice::<serde_json::Value>(b)
.ok()
.and_then(|v| crate::rest::parse_records(&v, &pointer, &fields).ok())
.map(|batch| batch.num_rows() > 0)
.unwrap_or(false)
},
synthetic,
)
}
/// **OSM admin boundaries** — the pure tag-filter has no live PBF path under the
/// default (`parsers`) feature, so this is synthetic feature JSON (a future
/// `osm`-feature real fetch could fill it). Decode with [`parse_admin_features`].
pub fn osm_admin_try_get_real_sample_data() -> Sample {
Sample::synthetic(SYNTHETIC_OSM_ADMIN_JSON.as_bytes().to_vec())
}
/// Decode an OSM-admin sample body (`[{id, tags}, …]`) into the
/// `(osm_id, tags_json, geometry)` tuples `extract_admin_units` consumes. Tags
/// are re-serialised to the compact JSON string the tag-filter parses.
pub fn parse_admin_features(body: &[u8]) -> Vec<(i64, String, Option<Vec<u8>>)> {
let Ok(serde_json::Value::Array(items)) = serde_json::from_slice::<serde_json::Value>(body)
else {
return Vec::new();
};
items
.into_iter()
.filter_map(|it| {
let id = it.get("id")?.as_i64()?;
let tags = it.get("tags")?;
Some((id, tags.to_string(), None))
})
.collect()
}
/// A tiny synthetic GTFS `stops.txt` — two Stockholm stops, WGS84 lat/lon.
#[cfg(feature = "gtfs")]
pub const SYNTHETIC_GTFS_STOPS_CSV: &str = "stop_id,stop_name,stop_lat,stop_lon,zone_id,location_type\n\
9022001,T-Centralen,59.33107,18.05977,A,0\n\
9022002,Slussen,59.31970,18.07208,A,0\n";
/// A tiny synthetic GTFS `routes.txt` — two SL routes.
#[cfg(feature = "gtfs")]
pub const SYNTHETIC_GTFS_ROUTES_CSV: &str = "route_id,agency_id,route_short_name,route_long_name,route_type,route_color\n\
9011001001000000,SL,1,Blå linjen,1,0000FF\n\
9011001002000000,SL,2,Röd linjen,1,FF0000\n";
/// A tiny synthetic GTFS `trips.txt` — two trips over the routes above.
#[cfg(feature = "gtfs")]
pub const SYNTHETIC_GTFS_TRIPS_CSV: &str = "route_id,service_id,trip_id,trip_headsign,direction_id,shape_id\n\
9011001001000000,wk,trip-1,Hjulsta,0,shape-1\n\
9011001002000000,wk,trip-2,Ropsten,1,shape-2\n";
/// **GTFS** static feed — synthesise a DEFLATE feed zip (no committed binary
/// blob) with the [`crate::gtfs::build_zip`] helper, or attempt a live feed from
/// a caller-supplied `GTFS_FEED_URL` (byte-capped), validating it parses ≥1 stop.
///
/// The zip carries **all three** members the crate registers a collector for
/// (`stops`/`routes`/`trips`), not just `stops.txt`: a feed archive holding only
/// the member ONE of the three collectors reads is not an offline-safe sample for
/// the other two — they would have had no sample at all.
#[cfg(feature = "gtfs")]
pub fn gtfs_try_get_real_sample_data() -> Sample {
let synthetic = crate::gtfs::build_zip(&[
("stops.txt", 8, SYNTHETIC_GTFS_STOPS_CSV.as_bytes()),
("routes.txt", 8, SYNTHETIC_GTFS_ROUTES_CSV.as_bytes()),
("trips.txt", 8, SYNTHETIC_GTFS_TRIPS_CSV.as_bytes()),
]);
match std::env::var("GTFS_FEED_URL") {
Ok(url) if !url.is_empty() => {
let fetched = http_get(&url, Some(1024 * 1024));
let cols = crate::gtfs::GtfsCollector::stops();
let cols2 = cols.cols().to_vec();
real_or_synthetic(
&url,
fetched,
move |b| {
crate::gtfs::parse_gtfs_zip(b, "stops.txt", &cols2)
.map(|rb| rb.num_rows() > 0)
.unwrap_or(false)
},
&synthetic,
)
}
_ => Sample::synthetic(synthetic),
}
}
/// **OSM ways (basemap)** — the bundled tiny real-OSM extract, offline.
///
/// This used to open `https://download.geofabrik.de/europe/sweden-latest.osm.pbf`
/// with a 1 MiB byte cap. **Geofabrik is not an OSM source in this house** (the owner,
/// 2026-08-13): korp takes the OSM planet over BitTorrent (`korp::download` →
/// osm-katana/librqbit) and cuts every region out of it locally through the a→e
/// convert chain. Nothing downstream ever wanted a per-region HTTP extract.
///
/// It was also a network call that could not succeed: a 1 MiB PREFIX of a country
/// dump is a truncated PBF, so the parse validation failed and the fallback fired
/// essentially every time. Removing it costs no sample and one pointless request.
#[cfg(feature = "osm")]
pub fn osm_ways_try_get_real_sample_data() -> Sample {
Sample::synthetic(SYNTHETIC_OSM_PBF)
}
// ═══════════════════════════════════════════════════════════════════════════
// Sample sources for the collectors whose fixtures used to live only INSIDE a
// `#[cfg(test)] mod tests`. A fixture parked in a test module is invisible to
// the registry sweep, to the benches, and to any consumer — so a collector could
// (and several did) sit in `registry()` with a real parse test but NO
// offline-safe sample of its own. These promote the fixtures to the shared seam
// so `for_collector` can answer for EVERY registered collector.
// ═══════════════════════════════════════════════════════════════════════════
/// ENTSO-E `A75` generation-per-type — two `TimeSeries` (solar B16 + wind B19),
/// two points each. The per-`psrType` shape [`crate::entsoe`] does NOT parse.
pub const SYNTHETIC_ENTSOE_A75_XML: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<GL_MarketDocument xmlns="urn:iec62325.351:tc57wg16:451-6:generationloaddocument:3:0">
<TimeSeries>
<mRID>1</mRID>
<in_Domain.mRID codingScheme="A01">10Y1001A1001A46L</in_Domain.mRID>
<MktPSRType><psrType>B16</psrType></MktPSRType>
<Period>
<timeInterval><start>2026-01-01T00:00Z</start><end>2026-01-01T02:00Z</end></timeInterval>
<resolution>PT60M</resolution>
<Point><position>1</position><quantity>120.0</quantity></Point>
<Point><position>2</position><quantity>135.5</quantity></Point>
</Period>
</TimeSeries>
<TimeSeries>
<mRID>2</mRID>
<in_Domain.mRID codingScheme="A01">10Y1001A1001A46L</in_Domain.mRID>
<MktPSRType><psrType>B19</psrType></MktPSRType>
<Period>
<timeInterval><start>2026-01-01T00:00Z</start><end>2026-01-01T02:00Z</end></timeInterval>
<resolution>PT60M</resolution>
<Point><position>1</position><quantity>900.0</quantity></Point>
<Point><position>2</position><quantity>880.0</quantity></Point>
</Period>
</TimeSeries>
</GL_MarketDocument>"#;
/// OpenSky `/states/all` — the POSITIONAL-array state vector shape (two aircraft).
/// Shared by the plain and the Europe-bbox presets: same document, different query.
pub const SYNTHETIC_OPENSKY_STATES_JSON: &str = r#"{
"time": 1700000000,
"states": [
["4b1815","SWR123 ","Switzerland",1700000000,1700000000,8.55,47.45,10972.8,false,231.0,87.3,0.0,null,11277.6,"1000",false,0],
["3c6750","DLH456 ","Germany",1700000000,1700000000,11.78,48.35,3200.0,false,180.0,270.0,5.2,null,3300.0,"2000",false,0]
]
}"#;
/// AISStream.io — the nested `MetaData`/`Message.PositionReport` envelope (two ships).
pub const SYNTHETIC_AISSTREAM_JSON: &str = r#"{
"messages": [
{"MessageType":"PositionReport",
"MetaData":{"MMSI":265547200,"ShipName":"WASA EXPRESS","latitude":59.325,"longitude":18.072,"time_utc":"2026-07-13T10:00:00Z"},
"Message":{"PositionReport":{"Sog":12.3,"Cog":87.4,"TrueHeading":90,"NavigationalStatus":0}}},
{"MessageType":"PositionReport",
"MetaData":{"MMSI":219000512,"ShipName":"DANA","latitude":57.700,"longitude":11.960,"time_utc":"2026-07-13T10:05:00Z"},
"Message":{"PositionReport":{"Sog":0.0,"Cog":0.0,"TrueHeading":511,"NavigationalStatus":5}}}
]
}"#;
/// GBFS `free_bike_status` — the `data.bikes` dockless-vehicle array (two bikes).
pub const SYNTHETIC_GBFS_JSON: &str = r#"{
"last_updated": 1700000000, "ttl": 30, "version": "2.0",
"data": {"bikes": [
{"bike_id":"bike-001","lat":59.9139,"lon":10.7522,"is_reserved":0,"is_disabled":0,
"vehicle_type_id":"citybike","current_range_meters":0.0},
{"bike_id":"bike-002","lat":59.9200,"lon":10.7400,"is_reserved":1,"is_disabled":0,
"vehicle_type_id":"ebike","current_range_meters":12000.0}
]}
}"#;
/// OSM Overpass QL — an `elements` node array with a raw `tags` object (two POIs).
pub const SYNTHETIC_OVERPASS_JSON: &str = r#"{
"version": 0.6, "generator": "Overpass API",
"elements": [
{"type":"node","id":1000001,"lat":59.3293,"lon":18.0686,
"tags":{"amenity":"charging_station","name":"Sergels Torg","capacity":"4"}},
{"type":"node","id":1000002,"lat":57.7089,"lon":11.9746,
"tags":{"amenity":"charging_station","name":"Göteborg C"}}
]
}"#;
/// SMHI metobs station-SET — the `station` geometry array (two stations), the
/// point-table shape, distinct from [`SYNTHETIC_SMHI_JSON`]'s value readings.
pub const SYNTHETIC_SMHI_WEATHER_JSON: &str = r#"{
"key": "1", "title": "Lufttemperatur",
"station": [
{"key":"180960","name":"Nikkaluokta","owner":"SMHI","latitude":67.85,"longitude":19.02,"height":475.0,"active":true},
{"key":"159880","name":"Karesuando","owner":"SMHI","latitude":68.44,"longitude":22.45,"height":327.0,"active":false}
]
}"#;
/// Trafikverket NVDB `RoadGeometry` — WKT `LINESTRING` ways (two segments).
/// A different `objecttype` than [`SYNTHETIC_TRAFIKVERKET_JSON`]'s TrainPosition.
pub const SYNTHETIC_TRAFIKVERKET_ROADS_JSON: &str = r#"{"RESPONSE":{"RESULT":[{"RoadGeometry":[
{"Element_Id":123456,"RoadMainNumber":40,"RoadSubNumber":0,"Direction":2,"Length":812.5,"County":14,
"Geometry":{"WGS84":"LINESTRING (12.001 57.700, 12.003 57.702, 12.006 57.705)",
"SWEREF99TM":"LINESTRING (319000 6396000, 319120 6396210)"},
"ModifiedTime":"2026-07-10T08:00:00.000+02:00","TimeStamp":"2026-07-13T06:00:00.000+02:00"},
{"Element_Id":123457,"RoadMainNumber":27,"RoadSubNumber":0,"Direction":1,"Length":1503.0,"County":14,
"Geometry":{"WGS84":"LINESTRING (12.100 57.800, 12.104 57.803)",
"SWEREF99TM":"LINESTRING (325000 6407000, 325260 6407330)"},
"ModifiedTime":"2026-07-11T09:30:00.000+02:00","TimeStamp":"2026-07-13T06:00:00.000+02:00"}
]}]}}"#;
/// A tiny RFC-4180 CSV — header + two typed data rows (the [`crate::csv`]
/// two-pass sniff resolves `Int64`/`Utf8`/`Float64`).
#[cfg(feature = "csv")]
pub const SYNTHETIC_CSV: &str =
"id,kommun,befolkning,andel\n1,Solna,84000,0.81\n2,Sundbyberg,53000,0.77\n";
/// **ENTSO-E generation per type (A75)** — the same free `ENTSOE_TOKEN` route as
/// [`entsoe_try_get_real_sample_data`], but `documentType=A75`, validated
/// against the per-`psrType` parser; else the synthetic two-series document.
pub fn entsoe_generation_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_ENTSOE_A75_XML.as_bytes();
let token = std::env::var("ENTSOE_TOKEN").unwrap_or_default();
if token.is_empty() {
return Sample::synthetic(synthetic.to_vec());
}
let se3 = "10Y1001A1001A46L";
let url = format!(
"https://web-api.tp.entsoe.eu/api?securityToken={token}&documentType=A75\
&processType=A16&in_Domain={se3}&periodStart=202601010000&periodEnd=202601020000"
);
let fetched = http_get(&url, Some(512 * 1024));
real_or_synthetic(
&url,
fetched,
|b| {
let xml = String::from_utf8_lossy(b);
crate::entsoe_generation::parse_entsoe_generation_xml(&xml)
.map(|p| !p.is_empty())
.unwrap_or(false)
},
synthetic,
)
}
/// **OECD SDMX** — dataflow-specific, so absent `OECD_SDMX_URL` it stays
/// synthetic; with the env var set it attempts that SDMX-JSON query.
pub fn oecd_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes();
match std::env::var("OECD_SDMX_URL") {
Ok(url) if !url.is_empty() => sdmx_sample(&url, synthetic),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **ECB SDMX** — dataflow-specific, so absent `ECB_SDMX_URL` it stays synthetic;
/// with the env var set it attempts that SDMX-JSON query.
pub fn ecb_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes();
match std::env::var("ECB_SDMX_URL") {
Ok(url) if !url.is_empty() => sdmx_sample(&url, synthetic),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **OpenSky `/states/all`** — the live endpoint moved to OAuth2 client
/// credentials in 2025, so the anonymous GET is only attempted (and validated
/// against the positional field map) when `OPENSKY_STATES_URL` names a reachable
/// one; otherwise the synthetic two-aircraft snapshot, no network.
pub fn opensky_states_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_OPENSKY_STATES_JSON.as_bytes();
match std::env::var("OPENSKY_STATES_URL") {
Ok(url) if !url.is_empty() => {
rest_sample(&crate::rest::RestCollector::opensky(), &url, synthetic)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **AISStream.io** — the feed is a WebSocket stream needing an API key, so a
/// short-timeout GET only happens when `AISSTREAM_URL` names an HTTP mirror;
/// otherwise the synthetic nested `PositionReport` envelope.
pub fn aisstream_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_AISSTREAM_JSON.as_bytes();
match std::env::var("AISSTREAM_URL") {
Ok(url) if !url.is_empty() => rest_sample(
&crate::rest::RestCollector::aisstream(),
&url,
synthetic,
),
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **GBFS `free_bike_status`** — a public no-auth dockless-vehicle feed (Oslo
/// Bysykkel), validated against the city-bike field map, else the synthetic feed.
pub fn gbfs_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::gbfs_citybikes(),
"https://gbfs.urbansharing.com/oslobysykkel.no/free_bike_status.json",
SYNTHETIC_GBFS_JSON.as_bytes(),
)
}
/// **OSM Overpass** — the QL query is a POST body and every public instance rate
/// limits hard, so a live attempt happens only when `OVERPASS_URL` names a
/// prepared GET; otherwise the synthetic two-node `elements` document.
pub fn overpass_try_get_real_sample_data() -> Sample {
let synthetic = SYNTHETIC_OVERPASS_JSON.as_bytes();
match std::env::var("OVERPASS_URL") {
Ok(url) if !url.is_empty() => {
rest_sample(&crate::rest::RestCollector::osm_overpass(), &url, synthetic)
}
_ => Sample::synthetic(synthetic.to_vec()),
}
}
/// **SMHI station-set** (station GEOMETRY, not readings) — public no-auth,
/// validated against the point-table field map, else the synthetic station set.
pub fn smhi_weather_try_get_real_sample_data() -> Sample {
rest_sample(
&crate::rest::RestCollector::smhi_weather(),
"https://opendata-download-metobs.smhi.se/api/version/latest/parameter/1.json",
SYNTHETIC_SMHI_WEATHER_JSON.as_bytes(),
)
}
/// **Trafikverket NVDB `RoadGeometry`** — the key rides in the POST body, so
/// there is no short-timeout GET to attempt: always the synthetic WKT
/// `LINESTRING` ways document (same posture as
/// [`trafikverket_try_get_real_sample_data`] without a key).
pub fn trafikverket_roads_try_get_real_sample_data() -> Sample {
Sample::synthetic(SYNTHETIC_TRAFIKVERKET_ROADS_JSON.as_bytes().to_vec())
}
/// **CSV** — a LOCAL flat file: there is no endpoint to attempt, so the sample is
/// the embedded RFC-4180 document (offline by construction).
#[cfg(feature = "csv")]
pub fn csv_try_get_real_sample_data() -> Sample {
Sample::synthetic(SYNTHETIC_CSV.as_bytes().to_vec())
}
/// The COBOL copybook both flat-file samples below are laid out against
/// (`LRECL` 18: a 6-char id + a 12-char name). A caller passes it as
/// `params.copybook`.
#[cfg(feature = "mainframe")]
pub const SYNTHETIC_MAINFRAME_COPYBOOK: &str = "\
01 CUSTOMER-RECORD.
05 CUST-ID PIC X(6).
05 CUST-NAME PIC X(12).
";
/// The two 18-byte records, as text — shared by the EBCDIC and the ASCII sample
/// so both flat-file collectors decode the SAME logical rows.
#[cfg(feature = "mainframe")]
pub const SYNTHETIC_FIXED_WIDTH_TEXT: &str = "000001Solna 000002Sundbyberg ";
/// **Mainframe EBCDIC** — a LOCAL flat file: the sample is
/// [`SYNTHETIC_FIXED_WIDTH_TEXT`] run through the crate's own IBM-037 encoder
/// ([`crate::mainframe::utf8_to_ebcdic`], the inverse of the decoder under test),
/// so the bytes are real code-page output rather than a committed binary blob.
#[cfg(feature = "mainframe")]
pub fn mainframe_try_get_real_sample_data() -> Sample {
Sample::synthetic(crate::mainframe::utf8_to_ebcdic(
SYNTHETIC_FIXED_WIDTH_TEXT,
crate::mainframe::CodePage::Ibm037,
))
}
/// **ASCII fixed-width** — a LOCAL flat file: the same two `LRECL`-18 records as
/// [`mainframe_try_get_real_sample_data`], in ASCII rather than EBCDIC (the
/// fixed-width collector's DISPLAY-usage path).
#[cfg(feature = "mainframe")]
pub fn fixed_width_try_get_real_sample_data() -> Sample {
Sample::synthetic(SYNTHETIC_FIXED_WIDTH_TEXT.as_bytes().to_vec())
}
/// **Every registered collector's offline-safe sample, by `meta().name`.**
///
/// This is the seam that makes the real-or-fake law *checkable* instead of
/// aspirational: [`registry_descriptors`](crate::registry_descriptors) enumerates
/// the collectors, this resolves each to its sample, and the registry sweep test
/// asserts the two sets match. Add a collector without adding its sample here and
/// the sweep goes RED — the discovery-complete law, enforced.
///
/// Several collectors legitimately SHARE one sample source: the two ERDDAP server
/// presets read the same `table` document shape, the three GTFS member collectors
/// read the same feed zip, the plain and Europe-bbox OpenSky presets read the same
/// `/states` snapshot, and both TrainPosition presets read the same Trafikverket
/// envelope. Sharing is the point — one sample per ACCESS METHOD, like the
/// collectors themselves.
///
/// Returns `None` for a name that is not a registered collector.
pub fn for_collector(name: &str) -> Option<Sample> {
Some(match name {
"dcat-ap-se" => dcat_try_get_real_sample_data(),
"pxweb-jsonstat2" => pxweb_try_get_real_sample_data(),
"eurostat-sdmx" => eurostat_try_get_real_sample_data(),
"oecd-sdmx" => oecd_try_get_real_sample_data(),
"ecb-sdmx" => ecb_try_get_real_sample_data(),
"insee-sdmx" => insee_try_get_real_sample_data(),
"ons-sdmx" => ons_try_get_real_sample_data(),
"istat-sdmx" => istat_try_get_real_sample_data(),
"worldbank-sdmx" => worldbank_sdmx_try_get_real_sample_data(),
"entsoe" => entsoe_try_get_real_sample_data(),
"entsoe_generation" => entsoe_generation_try_get_real_sample_data(),
"entsoe_day_ahead" => entsoe_day_ahead_try_get_real_sample_data(),
"nordpool" => nordpool_try_get_real_sample_data(),
"energy-charts-power" => energy_charts_try_get_real_sample_data(),
"usgs-earthquakes" => usgs_try_get_real_sample_data(),
"rss-atom" => rss_try_get_real_sample_data(),
"openaq-air-quality" => openaq_try_get_real_sample_data(),
"gtfs-rt-vehicles" => gtfs_rt_try_get_real_sample_data(),
"polisen-events" => polisen_try_get_real_sample_data(),
"kolada-kpi" => kolada_try_get_real_sample_data(),
"smhi-metobs" => smhi_try_get_real_sample_data(),
"smhi-weather" => smhi_weather_try_get_real_sample_data(),
"digitraffic-ais" => ais_try_get_real_sample_data(),
"gleif-lei" => gleif_lei_try_get_real_sample_data(),
"domstol-rattspraxis" => domstol_rattspraxis_try_get_real_sample_data(),
"riksdagen-dokument" => riksdagen_try_get_real_sample_data(),
"pxweb-tables" => pxweb_tables_try_get_real_sample_data(),
"scb-foretagsregistret" => scb_foretagsregistret_try_get_real_sample_data(),
#[cfg(feature = "csv")]
"fi-insyn" => fi_insyn_try_get_real_sample_data(),
// Both OpenSky snapshot presets read the SAME `/states` document; only
// the bbox query differs.
"opensky-flights" | "opensky-europe-bbox" => opensky_states_try_get_real_sample_data(),
"opensky-tracks" => opensky_tracks_try_get_real_sample_data(),
"aisstream" => aisstream_try_get_real_sample_data(),
"gbfs-citybikes" => gbfs_try_get_real_sample_data(),
"osm-overpass" => overpass_try_get_real_sample_data(),
"bolagsverket-organisationer" => bolagsverket_try_get_real_sample_data(),
"gbif-occurrences" => gbif_try_get_real_sample_data(),
"worldbank-indicators" => worldbank_try_get_real_sample_data(),
"restcountries" => restcountries_try_get_real_sample_data(),
"smhi-hydroobs" => smhi_hydro_try_get_real_sample_data(),
"metno-forecast" => metno_try_get_real_sample_data(),
"mobility-database" => mobility_database_try_get_real_sample_data(),
"eea-wise-water" => eea_wise_try_get_real_sample_data(),
"destatis-genesis" => destatis_genesis_try_get_real_sample_data(),
"cbs-odata" => cbs_odata_try_get_real_sample_data(),
"mobility-database-rt" => mobility_database_rt_try_get_real_sample_data(),
// One ERDDAP `table` document shape serves every ERDDAP deployment.
"erddap-emodnet-physics" | "erddap-noaa-coastwatch" => erddap_try_get_real_sample_data(),
"wfs" => wfs_try_get_real_sample_data(),
// Both TrainPosition presets read the same envelope; RoadGeometry is a
// different objecttype and gets its own.
"trafikverket-trainpos" | "trafikverket_trains" => trafikverket_try_get_real_sample_data(),
"trafikverket-roads" => trafikverket_roads_try_get_real_sample_data(),
#[cfg(feature = "osm")]
"osm-geoparquet" => osm_ways_try_get_real_sample_data(),
"osm-admin-boundaries" => osm_admin_try_get_real_sample_data(),
#[cfg(feature = "gribnc")]
"copernicus-cds" => copernicus_cds_try_get_real_sample_data(),
#[cfg(feature = "gribnc")]
"dwd-icon" => dwd_icon_try_get_real_sample_data(),
#[cfg(feature = "gribnc")]
"copernicus-era5" => copernicus_era5_try_get_real_sample_data(),
#[cfg(feature = "gribnc")]
"copernicus-cams" => copernicus_cams_try_get_real_sample_data(),
#[cfg(feature = "csv")]
"csv" => csv_try_get_real_sample_data(),
// The three GTFS member collectors read members of ONE feed zip.
#[cfg(feature = "gtfs")]
"gtfs-stops" | "gtfs-routes" | "gtfs-trips" => gtfs_try_get_real_sample_data(),
#[cfg(feature = "mainframe")]
"mainframe-copybook" => mainframe_try_get_real_sample_data(),
#[cfg(feature = "mainframe")]
"fixed-width" => fixed_width_try_get_real_sample_data(),
_ => return None,
})
}
/// reqwest 0.13 with reqwest 0.12 `rustls-tls` behaviour (2026-10-02): the ring
/// provider and the bundled Mozilla roots (webpki), not 0.13's aws-lc-rs + OS
/// trust store. Roots added later on the builder are merged in, as in 0.12.
#[allow(dead_code)]
fn tls_roots_blocking(b: reqwest::blocking::ClientBuilder) -> reqwest::blocking::ClientBuilder {
let _ = rustls::crypto::ring::default_provider().install_default();
b.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
}
#[cfg(test)]
mod tests {
use super::*;
// ── the core real-or-fake decision (`real_or_synthetic`) ────────────────
// These drive the private decision fn directly with fabricated `fetched`
// bytes + a validator, so the seam's fallback logic is asserted WITHOUT any
// network at all.
#[test]
fn real_or_synthetic_keeps_valid_fetched_as_real() {
let s = real_or_synthetic(
"https://x/y",
Some(b"hello".to_vec()),
|_| true,
b"fallback",
);
assert_eq!(
s.origin,
SampleOrigin::Real {
url: "https://x/y".into()
}
);
assert_eq!(s.body, b"hello");
assert!(!s.is_synthetic());
}
#[test]
fn real_or_synthetic_falls_back_when_fetch_missing() {
let s = real_or_synthetic("https://x/y", None, |_| true, b"fallback");
assert!(s.is_synthetic());
assert_eq!(s.body, b"fallback");
}
#[test]
fn real_or_synthetic_falls_back_when_validation_fails() {
// Fetched bytes present but the collector's parser rejects them.
let s = real_or_synthetic(
"https://x/y",
Some(b"garbage".to_vec()),
|_| false,
b"fallback",
);
assert!(s.is_synthetic());
assert_eq!(s.body, b"fallback");
}
#[test]
fn real_or_synthetic_falls_back_on_empty_body_even_if_validator_true() {
// An empty 200 body is never `Real` (the `!is_empty()` guard).
let s = real_or_synthetic("https://x/y", Some(Vec::new()), |_| true, b"fallback");
assert!(s.is_synthetic());
assert_eq!(s.body, b"fallback");
}
// ── credential redaction of the RECORDED url ────────────────────────────
// The ENTSO-E / Destatis live sample URLs carry the operator's token in the
// query string. That URL becomes `SampleOrigin::Real{url}` → `label()` →
// the functional_status detail → a persisted test-matrix row, so the secret
// must never survive into the recorded identity.
#[test]
fn real_or_synthetic_redacts_query_token_from_recorded_url() {
let url = "https://web-api.tp.entsoe.eu/api?securityToken=s3cr3t-live-token\
&documentType=A44&in_Domain=10Y1001A1001A46L";
let s = real_or_synthetic(url, Some(b"<doc/>".to_vec()), |_| true, b"fallback");
let label = s.origin.label();
assert!(
!label.contains("s3cr3t-live-token"),
"token leaked into label: {label}"
);
assert!(
label.contains("securityToken=REDACTED"),
"not redacted: {label}"
);
// The endpoint identity survives — this is still a usable provenance tag.
assert!(
label.contains("web-api.tp.entsoe.eu/api"),
"endpoint lost: {label}"
);
assert!(
label.contains("documentType=A44"),
"non-secret param lost: {label}"
);
}
#[test]
fn real_or_synthetic_redacts_destatis_token_param() {
let url = "https://www-genesis.destatis.de/genesisWS/rest/2020/catalogue/tables?\
token=abc123&selection=*&pagelength=20&language=en";
let s = real_or_synthetic(url, Some(b"{}".to_vec()), |_| true, b"fallback");
let SampleOrigin::Real { url: recorded } = &s.origin else {
panic!("expected Real, got {:?}", s.origin);
};
assert!(!recorded.contains("abc123"), "token leaked: {recorded}");
assert_eq!(
recorded,
"https://www-genesis.destatis.de/genesisWS/rest/2020/catalogue/tables?\
token=REDACTED&selection=*&pagelength=20&language=en"
);
}
#[test]
fn redact_url_covers_the_known_secret_param_names_case_insensitively() {
for (raw, want) in [
(
"https://h/p?api_key=AKIA123",
"https://h/p?api_key=REDACTED",
),
("https://h/p?APIKEY=AKIA123", "https://h/p?APIKEY=REDACTED"),
(
"https://h/p?key=zz&other=1",
"https://h/p?key=REDACTED&other=1",
),
(
"https://h/p?access_token=t",
"https://h/p?access_token=REDACTED",
),
(
"https://h/p?subscription-key=t",
"https://h/p?subscription-key=REDACTED",
),
] {
assert_eq!(redact_url(raw), want, "for {raw}");
}
}
#[test]
fn redact_url_leaves_credential_free_urls_untouched() {
for url in [
"https://opendata.cbs.nl/ODataApi/odata/37296eng/TypedDataSet?%24top=5",
"https://h/p",
"https://h/p?a=1&b=2#frag",
// A valueless secret param has nothing to hide.
"https://h/p?token=",
] {
assert_eq!(redact_url(url), url, "rewrote a clean url: {url}");
}
}
#[test]
fn redact_url_scrubs_userinfo_password_and_keeps_the_fragment() {
assert_eq!(
redact_url("https://user:hunter2@h/p?a=1"),
"https://user:REDACTED@h/p?a=1"
);
// No password half → nothing to redact.
assert_eq!(redact_url("https://user@h/p"), "https://user@h/p");
// An `@` in the PATH is not userinfo.
assert_eq!(redact_url("https://h/p/a@b"), "https://h/p/a@b");
// The fragment is preserved and never parsed as a query pair.
assert_eq!(
redact_url("https://h/p?token=x#sec=y"),
"https://h/p?token=REDACTED#sec=y"
);
}
// ── Sample / SampleOrigin surface ───────────────────────────────────────
#[test]
fn sample_origin_labels_and_synthetic_flag() {
let real = SampleOrigin::Real {
url: "https://a".into(),
};
assert!(!real.is_synthetic());
assert_eq!(real.label(), "real:https://a");
assert!(SampleOrigin::Synthetic.is_synthetic());
assert_eq!(SampleOrigin::Synthetic.label(), "synthetic");
}
#[test]
fn synthetic_sample_body_accessors() {
let s = Sample::synthetic(b"abc".to_vec());
assert!(s.is_synthetic());
assert_eq!(s.len(), 3);
assert!(!s.is_empty());
assert_eq!(s.as_str(), "abc");
assert!(Sample::synthetic(Vec::new()).is_empty());
}
// ── every embedded synthetic fixture parses with its own parser ─────────
// This is the offline "fake-sample" contract: each `SYNTHETIC_*` fixture is
// a GUARANTEED-parseable stand-in — asserted here deterministically, with no
// live fetch, exactly mirroring the validators the live seam applies.
#[test]
fn synthetic_dcat_fixture_parses_into_registry() {
let rows = crate::dcat::parse_dcat_rdfxml(SYNTHETIC_DCAT_RDFXML.as_bytes()).expect("dcat");
assert_eq!(rows.len(), 2, "befolkning (1 dist) + empty (0 dist)");
crate::dcat::registry_batch(&rows).expect("registry batch");
}
#[test]
fn synthetic_pxweb_fixture_normalises_to_cube() {
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_PXWEB_JSONSTAT2).expect("json");
let batch = crate::pxweb::parse_jsonstat2(&v).expect("cube");
assert_eq!(batch.num_rows(), 4, "2×2×1 cells");
}
#[test]
fn synthetic_eurostat_sdmx_fixture_flattens_to_observations() {
let v: serde_json::Value =
serde_json::from_str(SYNTHETIC_EUROSTAT_SDMX_JSON).expect("json");
let rb = crate::sdmx::parse_sdmx_json(&v).expect("sdmx batch");
assert_eq!(rb.num_rows(), 4, "2 geo × 2 time observations");
}
#[test]
fn synthetic_entsoe_fixture_maps_to_points() {
let pts = crate::entsoe::parse_entsoe_xml(SYNTHETIC_ENTSOE_XML).expect("entsoe");
assert_eq!(pts.len(), 2);
assert_eq!(pts[0].bidding_zone.as_deref(), Some("SE3"));
}
fn rest_rows(preset: crate::rest::RestCollector, fixture: &str) -> usize {
let v: serde_json::Value = serde_json::from_str(fixture).expect("json");
crate::rest::parse_records(&v, preset.records_pointer(), preset.fields())
.expect("map")
.num_rows()
}
#[test]
fn synthetic_rest_fixtures_map_to_arrow() {
assert_eq!(
rest_rows(
crate::rest::RestCollector::polisen(),
SYNTHETIC_POLISEN_JSON
),
2
);
assert_eq!(
rest_rows(crate::rest::RestCollector::kolada(), SYNTHETIC_KOLADA_JSON),
2
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::smhi_metobs(),
SYNTHETIC_SMHI_JSON
),
1
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::digitraffic_ais(),
SYNTHETIC_AIS_JSON
),
2
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::bolagsverket(),
SYNTHETIC_BOLAGSVERKET_JSON
),
2
);
// The four zones + reference collectors added 2026-07-18.
assert_eq!(
rest_rows(
crate::rest::RestCollector::smhi_hydro(),
SYNTHETIC_SMHI_HYDRO_JSON
),
1,
"WATER"
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::metno_forecast(),
SYNTHETIC_METNO_JSON
),
2,
"WEATHER"
);
assert_eq!(
rest_rows(crate::rest::RestCollector::gbif(), SYNTHETIC_GBIF_JSON),
2,
"biodiversity"
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::worldbank(),
SYNTHETIC_WORLDBANK_JSON
),
2,
"worldbank [meta,data]"
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::restcountries(),
SYNTHETIC_RESTCOUNTRIES_JSON
),
2,
"country spine"
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::mobility_database(),
SYNTHETIC_MOBILITY_DB_JSON
),
2,
"GTFS feed catalogue"
);
// The three European open-data collectors added 2026-07-18.
assert_eq!(
rest_rows(
crate::rest::RestCollector::eea_wise(),
SYNTHETIC_EEA_WISE_JSON
),
2,
"EEA WISE water quality"
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::destatis_genesis(),
SYNTHETIC_DESTATIS_GENESIS_JSON
),
2,
"Destatis GENESIS tables"
);
assert_eq!(
rest_rows(
crate::rest::RestCollector::cbs_odata(),
SYNTHETIC_CBS_ODATA_JSON
),
2,
"CBS OData observations"
);
}
/// Typed-row contract for the three European open-data collectors: EEA WISE
/// lands the determinand + numeric mean value + WGS84 fix, Destatis GENESIS
/// the table code + title, CBS OData the period + numeric population value.
/// RED if any pointer/type mapping breaks.
#[test]
fn european_open_data_presets_decode_their_typed_rows() {
use arrow_array::{Array, Float64Array, Int64Array, StringArray};
// EEA WISE — water-quality rows (determinand, mean value, coordinate).
let c = crate::rest::RestCollector::eea_wise();
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_EEA_WISE_JSON).expect("json");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("eea wise");
assert_eq!(rb.num_rows(), 2);
let det = rb
.column_by_name("determinand")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("determinand");
assert_eq!(det.value(0), "Nitrate");
let mean = rb
.column_by_name("mean_value")
.and_then(|a| a.as_any().downcast_ref::<Float64Array>())
.expect("mean_value");
assert!(
(mean.value(0) - 2.35).abs() < 1e-9,
"numeric mean value decoded"
);
let year = rb
.column_by_name("year")
.and_then(|a| a.as_any().downcast_ref::<Int64Array>())
.expect("year");
assert_eq!(year.value(0), 2023);
// Destatis GENESIS — statistical-table descriptor rows.
let c = crate::rest::RestCollector::destatis_genesis();
let v: serde_json::Value =
serde_json::from_str(SYNTHETIC_DESTATIS_GENESIS_JSON).expect("json");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("destatis");
assert_eq!(rb.num_rows(), 2);
let code = rb
.column_by_name("table_code")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("table_code");
assert_eq!(
code.value(0),
"12411-0001",
"GENESIS table code from /List/*/Code"
);
// CBS OData — typed national-statistics observations.
let c = crate::rest::RestCollector::cbs_odata();
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_CBS_ODATA_JSON).expect("json");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("cbs");
assert_eq!(rb.num_rows(), 2);
let period = rb
.column_by_name("period")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("period");
assert_eq!(period.value(1), "2023JJ00");
let val = rb
.column_by_name("value")
.and_then(|a| a.as_any().downcast_ref::<Float64Array>())
.expect("value");
assert!(
(val.value(1) - 17811291.0).abs() < 1.0,
"population value decoded"
);
crate::functional_status(
"korp-collectors/european-open-data",
"eea_destatis_cbs_typed_rows",
true,
"EEA WISE + Destatis GENESIS + CBS OData → typed Arrow rows",
);
}
/// Destatis GENESIS is token-gated: absent `DESTATIS_GENESIS_TOKEN` the sample
/// stays synthetic with no network (the offline contract).
#[test]
fn destatis_genesis_sample_is_synthetic_without_token() {
if std::env::var_os("DESTATIS_GENESIS_TOKEN").is_none() {
let s = destatis_genesis_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_DESTATIS_GENESIS_JSON.as_bytes());
}
}
#[test]
fn synthetic_erddap_fixture_decodes_to_typed_table() {
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_ERDDAP_JSON).expect("json");
let rb = crate::erddap::parse_erddap_json(&v).expect("erddap table");
assert_eq!(rb.num_rows(), 2, "two ERDDAP table rows");
assert_eq!(rb.num_columns(), 5, "five self-typed columns");
}
#[cfg(feature = "gribnc")]
#[test]
fn synthetic_grib2_sample_decodes_to_catalog() {
let s = super::dwd_icon_try_get_real_sample_data();
// Offline (no DWD_ICON_URL) → the synthetic GRIB2, which must decode.
let msgs = crate::gribnc::parse_grib2(s.body.as_slice()).expect("decode grib2 sample");
assert_eq!(msgs.len(), 1, "one synthetic GRIB2 message");
assert_eq!(msgs[0].ni, 360);
assert_eq!(msgs[0].reference_time, "2026-07-18T00:00:00Z");
}
/// Deeper assertions on the new presets: the nested Met.no pointers decode the
/// forecast details, World Bank reads the `[meta, data]` element-1 array + the
/// nested indicator/country labels, and GBIF lands the taxon + WGS84 fix — so
/// each new live-capable collector is a real typed-row contract, not a stub.
#[test]
fn new_zone_presets_decode_their_typed_rows() {
use arrow_array::{Array, Float64Array, StringArray};
// WEATHER — Met.no timeseries → per-time forecast rows with decoded details.
let c = crate::rest::RestCollector::metno_forecast();
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_METNO_JSON).expect("json");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("metno");
assert_eq!(rb.num_rows(), 2);
let temp = rb
.column_by_name("air_temperature")
.and_then(|a| a.as_any().downcast_ref::<Float64Array>())
.expect("temp f64");
assert!(
(temp.value(0) - 17.1).abs() < 1e-9,
"instant.details.air_temperature decoded"
);
// World Bank — records live at element 1 of the [meta, data] envelope.
let c = crate::rest::RestCollector::worldbank();
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_WORLDBANK_JSON).expect("json");
let rb =
crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("worldbank");
assert_eq!(rb.num_rows(), 2);
let iso = rb
.column_by_name("countryiso3code")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("iso3");
assert_eq!(iso.value(0), "SWE", "ISO3 join spine");
let val = rb
.column_by_name("value")
.and_then(|a| a.as_any().downcast_ref::<Float64Array>())
.expect("value");
assert!(
(val.value(0) - 10486941.0).abs() < 1.0,
"population value decoded"
);
// GBIF — taxon + WGS84 fix.
let c = crate::rest::RestCollector::gbif();
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_GBIF_JSON).expect("json");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("gbif");
assert_eq!(rb.num_rows(), 2);
let lat = rb
.column_by_name("lat")
.and_then(|a| a.as_any().downcast_ref::<Float64Array>())
.expect("lat");
assert!(
(lat.value(0) - 59.3293).abs() < 1e-9,
"decimalLatitude decoded"
);
}
#[test]
fn synthetic_wfs_fixture_parses_into_features() {
let feats = crate::wfs::parse_wfs_gml(SYNTHETIC_WFS_GML).expect("wfs");
assert_eq!(feats.len(), 2, "point + linestring members");
assert!(
feats[0]
.geometry_wkt
.as_deref()
.unwrap()
.starts_with("POINT")
);
crate::wfs::features_batch(&feats).expect("features batch");
}
#[test]
fn synthetic_nordpool_fixture_flattens_to_prices() {
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_NORDPOOL_JSON).expect("json");
let prices = crate::nordpool::parse_nordpool_json(&v).expect("prices");
assert_eq!(prices.len(), 6, "2 periods × 3 SE zones");
crate::nordpool::prices_batch(&prices).expect("prices batch");
}
#[test]
fn synthetic_usgs_fixture_flattens_to_quakes() {
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_USGS_GEOJSON).expect("json");
let quakes = crate::usgs::parse_usgs_geojson(&v).expect("quakes");
assert_eq!(quakes.len(), 2, "two Point features → two quake rows");
crate::usgs::quakes_batch(&quakes).expect("quakes batch");
}
#[test]
fn synthetic_energy_charts_fixture_transposes_to_power_rows() {
let v: serde_json::Value =
serde_json::from_str(SYNTHETIC_ENERGY_CHARTS_JSON).expect("json");
let readings = crate::energy_charts::parse_energy_charts_json(&v).expect("readings");
assert_eq!(
readings.len(),
6,
"3 types × 2 instants → six transposed rows"
);
crate::energy_charts::power_batch(&readings).expect("power batch");
}
#[test]
fn synthetic_openaq_fixture_flattens_to_measurements() {
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_OPENAQ_JSON).expect("json");
let rows = crate::openaq::parse_openaq_json(&v).expect("measurements");
assert_eq!(rows.len(), 2, "two located readings → two measurement rows");
assert_eq!(rows[0].parameter, "pm25");
assert_eq!(rows[0].country.as_deref(), Some("SE"));
crate::openaq::measurements_batch(&rows).expect("measurements batch");
}
#[test]
fn synthetic_gtfs_rt_fixture_decodes_to_vehicle_rows() {
let pb = synthetic_gtfs_rt_pb();
let rows = crate::gtfs_rt::parse_gtfs_rt(&pb).expect("decode synthetic gtfs-rt");
assert_eq!(rows.len(), 2, "two vehicles in the synthetic feed");
assert_eq!(rows[0].vehicle_id.as_deref(), Some("9031001"));
assert_eq!(rows[0].route_id.as_deref(), Some("17"));
assert!((rows[0].latitude - 59.33107).abs() < 1e-3);
crate::gtfs_rt::vehicles_batch(&rows).expect("vehicles batch");
}
#[test]
fn synthetic_rss_fixture_flattens_to_feed_rows() {
let items = crate::rss::parse_feed(SYNTHETIC_RSS_XML).expect("rss");
assert_eq!(items.len(), 2, "two <item> → two feed rows");
assert_eq!(items[0].feed_title.as_deref(), Some("Krisinformation"));
assert_eq!(
items[0].categories,
vec!["Weather".to_string(), "Flood".to_string()]
);
crate::rss::feed_batch(&items).expect("feed batch");
}
#[test]
fn synthetic_trafikverket_fixture_maps_to_position_table() {
let c = crate::trafikverket::TrafikverketCollector::train_positions();
let v: serde_json::Value = serde_json::from_str(SYNTHETIC_TRAFIKVERKET_JSON).expect("json");
let batch = crate::rest::parse_records(&v, &c.records_pointer(), c.fields()).expect("map");
assert_eq!(batch.num_rows(), 1);
}
// ── parse_admin_features (the OSM-admin sample decoder) ──────────────────
#[test]
fn parse_admin_features_decodes_and_filters() {
let features = parse_admin_features(SYNTHETIC_OSM_ADMIN_JSON.as_bytes());
assert_eq!(
features.len(),
5,
"4 boundaries + 1 road (filtered downstream)"
);
let units = crate::osm_admin::extract_admin_units(features);
assert_eq!(units.len(), 4, "the highway=primary feature is dropped");
}
#[test]
fn parse_admin_features_is_empty_on_non_array() {
assert!(parse_admin_features(b"not json").is_empty());
assert!(parse_admin_features(b"{\"not\":\"an array\"}").is_empty());
}
// ── credential-gated samples stay Synthetic offline (deterministic) ──────
// ENTSO-E / AIS / Trafikverket only attempt a live fetch when their key env
// var is set; absent one (the CI norm) they return the embedded fixture with
// NO network — asserted here so the offline contract is pinned.
#[test]
fn entsoe_sample_is_synthetic_without_token() {
if std::env::var_os("ENTSOE_TOKEN").is_none() {
let s = entsoe_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_ENTSOE_XML.as_bytes());
}
}
#[test]
fn entsoe_day_ahead_sample_is_synthetic_without_token() {
if std::env::var_os("ENTSOE_TOKEN").is_none() {
let s = entsoe_day_ahead_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_ENTSOE_A44_XML.as_bytes());
// The offline fixture parses to ≥1 day-ahead price point.
let prices =
crate::entsoe_day_ahead::parse_entsoe_day_ahead_xml(SYNTHETIC_ENTSOE_A44_XML)
.expect("parse A44 fixture");
assert!(!prices.is_empty(), "A44 fixture yields ≥1 price point");
}
}
#[test]
fn opensky_tracks_sample_is_synthetic_without_credentials() {
if std::env::var_os("OPENSKY_USER").is_none() || std::env::var_os("OPENSKY_PASS").is_none()
{
let s = opensky_tracks_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_OPENSKY_TRACKS_JSON.as_bytes());
}
}
/// **The AIS sample now ATTEMPTS a real feed**, and either outcome is legal —
/// what is not legal is a body that no longer parses. Before 2026-09-03 the only
/// possible answer here was the fixture, because the preset's endpoint was a
/// placeholder; asserting `is_synthetic()` today would assert the network is down.
///
/// So this asserts the property that actually holds in both worlds: whatever comes
/// back parses through the collector's OWN pointer + field map to ≥1 row. A wrong
/// `records_pointer` (`/positions`, the dead feed's) reddens it on the fixture
/// alone, with no network at all.
#[test]
fn ais_sample_parses_through_the_digitraffic_field_map() {
let s = ais_try_get_real_sample_data();
let c = crate::rest::RestCollector::digitraffic_ais();
let v: serde_json::Value =
serde_json::from_slice(&s.body).expect("AIS sample body is JSON");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields())
.expect("AIS sample parses through the preset's field map");
assert!(
rb.num_rows() > 0,
"the AIS sample carried at least one position ({:?})",
s.origin
);
if s.is_synthetic() {
assert_eq!(s.body, SYNTHETIC_AIS_JSON.as_bytes());
assert_eq!(rb.num_rows(), 2, "the fixture carries two vessels");
}
}
#[test]
fn trafikverket_sample_is_synthetic_without_key() {
if std::env::var_os("TRAFIKVERKET_API_KEY").is_none() {
let s = trafikverket_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_TRAFIKVERKET_JSON.as_bytes());
}
}
#[test]
fn wfs_sample_is_synthetic_without_url() {
if std::env::var_os("WFS_URL").is_none() {
let s = wfs_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_WFS_GML.as_bytes());
}
}
#[test]
fn nordpool_sample_is_synthetic_without_api_url() {
if std::env::var_os("NORDPOOL_API_URL").is_none() {
let s = nordpool_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_NORDPOOL_JSON.as_bytes());
}
}
/// **The RSS sample has a real default now, so the claim to check is different.**
///
/// It used to be "synthetic unless `RSS_FEED_URL` is set", which is what made the
/// one generic collector korp configures five live government feeds against also
/// the one whose pane showed a fixture. The default is now regeringen.se's
/// press-release feed, so the two things worth asserting are that the OFFLINE
/// switch still wins outright, and that whatever comes back — live or fallback —
/// flattens through this crate's own parser to at least one row. Measured live
/// 2026-09-05: **111 662 bytes, 100 items, 100 distinct `guid`s**.
#[test]
fn rss_sample_is_synthetic_when_offline_and_parses_either_way() {
if offline() {
let s = rss_try_get_real_sample_data();
assert!(s.is_synthetic(), "KORP_COLLECTORS_OFFLINE must win outright");
assert_eq!(s.body, SYNTHETIC_RSS_XML.as_bytes());
return;
}
let s = rss_try_get_real_sample_data();
let items = crate::rss::parse_feed(&s.as_str()).expect("the sample is a feed");
assert!(
!items.is_empty(),
"a sample that flattens to nothing is not a sample — real_or_synthetic must \
have fallen back rather than handed over an unparseable body"
);
}
#[test]
fn gtfs_rt_sample_is_a_decodable_feed_without_url() {
if std::env::var_os("GTFS_RT_URL").is_none() {
let s = gtfs_rt_try_get_real_sample_data();
assert!(s.is_synthetic());
// The synthetic body is a real protobuf FeedMessage that decodes to 2 rows.
let rows = crate::gtfs_rt::parse_gtfs_rt(&s.body).expect("synthetic gtfs-rt decodes");
assert_eq!(rows.len(), 2, "two vehicles in the synthetic feed");
}
}
#[test]
fn openaq_sample_is_synthetic_without_api_url() {
if std::env::var_os("OPENAQ_API_URL").is_none() {
let s = openaq_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_OPENAQ_JSON.as_bytes());
}
}
#[test]
fn bolagsverket_sample_is_synthetic_without_api_url() {
if std::env::var_os("BOLAGSVERKET_API_URL").is_none() {
let s = bolagsverket_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_BOLAGSVERKET_JSON.as_bytes());
}
}
#[cfg(feature = "gtfs")]
#[test]
fn gtfs_sample_is_a_parseable_feed_zip_without_feed_url() {
if std::env::var_os("GTFS_FEED_URL").is_none() {
let s = gtfs_try_get_real_sample_data();
assert!(s.is_synthetic());
// The synthetic body is a real DEFLATE zip that parses to 2 stops.
let cols = crate::gtfs::GtfsCollector::stops();
let rb = crate::gtfs::parse_gtfs_zip(&s.body, "stops.txt", cols.cols())
.expect("synthetic GTFS zip parses");
assert_eq!(rb.num_rows(), 2, "two stops in the synthetic feed");
}
}
#[test]
fn osm_admin_sample_is_always_synthetic() {
// No live PBF path under the default feature → always the fixture.
let s = osm_admin_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_OSM_ADMIN_JSON.as_bytes());
}
// ── national SDMX (INSEE / ONS / ISTAT / World Bank) — 2026-07-18 ─────────
/// The national SDMX fixture flattens through the SHARED SDMX-JSON parser to
/// its 4 `(REF_AREA, TIME_PERIOD, value)` rows — the same body validates for
/// every national preset (zero new parsing), the offline sample contract.
#[test]
fn synthetic_national_sdmx_fixture_flattens_to_observations() {
use arrow_array::{Array, StringArray};
let v: serde_json::Value =
serde_json::from_str(SYNTHETIC_NATIONAL_SDMX_JSON).expect("json");
let rb = crate::sdmx::parse_sdmx_json(&v).expect("national sdmx batch");
assert_eq!(rb.num_rows(), 4, "2 REF_AREA × 2 TIME_PERIOD observations");
let area = rb
.column_by_name("REF_AREA")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("REF_AREA utf8");
assert_eq!(area.value(0), "FR");
}
/// Every national SDMX preset is dataflow-specific → synthetic (no network)
/// absent its override URL env var; the fixture flattens to 4 rows.
#[test]
fn national_sdmx_samples_are_synthetic_without_url() {
for (var, s) in [
("INSEE_SDMX_URL", insee_try_get_real_sample_data()),
("ONS_SDMX_URL", ons_try_get_real_sample_data()),
("ISTAT_SDMX_URL", istat_try_get_real_sample_data()),
(
"WORLDBANK_SDMX_URL",
worldbank_sdmx_try_get_real_sample_data(),
),
] {
if std::env::var_os(var).is_none() {
assert!(s.is_synthetic(), "{var} unset → synthetic");
assert_eq!(s.body, SYNTHETIC_NATIONAL_SDMX_JSON.as_bytes());
}
}
}
// ── Mobility Database GTFS-Realtime feed catalogue — 2026-07-18 ──────────
/// The GTFS-RT feed-catalogue fixture maps to its two RT feed-descriptor rows,
/// with `entity_type` decoded from `/entity_types/0` — the meta-collector row
/// that drives the gtfs-rt reader.
#[test]
fn synthetic_mobility_db_rt_fixture_maps_to_feed_rows() {
use arrow_array::{Array, StringArray};
let c = crate::rest::RestCollector::mobility_database_rt();
let v: serde_json::Value =
serde_json::from_str(SYNTHETIC_MOBILITY_DB_RT_JSON).expect("json");
let rb = crate::rest::parse_records(&v, c.records_pointer(), c.fields()).expect("rt feeds");
assert_eq!(rb.num_rows(), 2, "two GTFS-RT feed descriptors");
let et = rb
.column_by_name("entity_type")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("entity_type utf8");
assert_eq!(
et.value(0),
"vp",
"vehicle-positions entity type from /entity_types/0"
);
let url = rb
.column_by_name("latest_url")
.and_then(|a| a.as_any().downcast_ref::<StringArray>())
.expect("latest_url utf8");
assert!(
url.value(0).ends_with("VehiclePositions.pb"),
"the .pb feed URL that drives gtfs-rt"
);
}
#[test]
fn mobility_database_rt_sample_is_synthetic_without_token() {
if std::env::var_os("MOBILITY_DB_TOKEN").is_none() {
let s = mobility_database_rt_try_get_real_sample_data();
assert!(s.is_synthetic());
assert_eq!(s.body, SYNTHETIC_MOBILITY_DB_RT_JSON.as_bytes());
}
}
// ── Copernicus ERA5 / CAMS grid-value samples — 2026-07-18 ───────────────
/// The ERA5 / CAMS samples are async-job + key gated: absent their direct-URL
/// env var they stay synthetic (no network), and the synthetic body is a REAL
/// simple-packed GRIB2 that unpacks to 4 per-point grid VALUE rows.
#[cfg(feature = "gribnc")]
#[test]
fn era5_and_cams_samples_are_decodable_values_without_url() {
use arrow_array::{Array, Float64Array};
for (var, s) in [
(
"ERA5_SAMPLE_URL",
copernicus_era5_try_get_real_sample_data(),
),
(
"CAMS_SAMPLE_URL",
copernicus_cams_try_get_real_sample_data(),
),
] {
if std::env::var_os(var).is_none() {
assert!(s.is_synthetic(), "{var} unset → synthetic");
let rb = crate::gribnc::parse_weather_values(&s.body, None)
.expect("synthetic GRIB2 values decode");
assert_eq!(rb.num_rows(), 4, "2×2 grid → four value rows");
let val = rb
.column_by_name("value")
.and_then(|a| a.as_any().downcast_ref::<Float64Array>())
.expect("value f64");
assert!((val.value(0) - 10.0).abs() < 1e-9);
}
}
}
}