tests/pxweb_scb_live.rs
//! **The live proof of lane EC O6's SCB half** — against `api.scb.se`, keyless.
//!
//! `#[ignore]` because CI has no network and this crate's law is that tests never hang
//! for want of one. Run it deliberately:
//!
//! ```text
//! cargo test --test pxweb_scb_live -- --ignored --nocapture
//! ```
//!
//! It FAILS if SCB is unreachable rather than falling back to a fixture: the whole point
//! is to establish what the real API does, including the parts that fail.
use arrow_array::{Array, Float64Array, StringArray};
use korp_collectors::pxweb::{
PxWebCollector, PxWebTableTree, SCB_FDBR07N_DIMS, SCB_FDBR07N_URL, SCB_SSD_ROOT,
scb_foretag_sni_storleksklass_cfg,
};
use korp_collectors::sink::BatchSink;
use korp_collectors::{Bounds, Collector, CollectorCfg};
/// **The SNI × storleksklass × år aggregate — the keyless ceiling, fetched.**
///
/// `NV/NV0101/FDBR07N` is the cube that answers how many Swedish firms exist by industry
/// and by size, per year. 22 top-level SNI divisions × 9 size classes × 1 measure × 18
/// years = 3 564 cells, which is what this must land — and every one of the four
/// dimensions has to be a column, because the cube's business key IS its full
/// coordinate.
#[tokio::test]
#[ignore = "live network: SCB's public PxWeb Statistical Database"]
async fn the_scb_company_aggregate_cube_lands_every_cell() {
let cfg = scb_foretag_sni_storleksklass_cfg();
let sink = BatchSink::memory();
let stats = PxWebCollector
.collect(&cfg, Bounds::default(), &sink)
.await
.expect("SCB's public PxWeb API must answer — it needs no key");
let batches = sink.take_batches();
let b = batches.first().expect("one batch");
println!("rows landed : {}", stats.rows);
println!(
"columns : {:?}",
b.schema().fields().iter().map(|f| f.name().clone()).collect::<Vec<_>>()
);
assert_eq!(stats.rows, 3_564, "22 SNI × 9 storleksklass × 1 measure × 18 years");
assert_eq!(b.num_rows(), 3_564, "…as real Arrow rows, not a count");
for dim in SCB_FDBR07N_DIMS {
assert!(
b.column_by_name(dim).is_some(),
"`{dim}` is a column — the cube's identity is its full coordinate"
);
}
// The KEY the collector would upsert on names exactly those columns.
let key = PxWebCollector.business_key_for(&cfg);
assert!(
key.missing_from(&b.schema()).is_empty(),
"the business key names columns the LIVE batch really carries: {:?}",
key.columns
);
// The numbers are numbers, and the size classes are the register's own.
let v = b
.column_by_name("value")
.expect("value")
.as_any()
.downcast_ref::<Float64Array>()
.expect("value is f64");
let nulls = (0..v.len()).filter(|i| v.is_null(*i)).count();
let total_all: Vec<f64> = (0..v.len()).filter(|i| !v.is_null(*i)).map(|i| v.value(i)).collect();
println!("null cells : {nulls}/3564");
println!(
"value range : {:.0} … {:.0}",
total_all.iter().cloned().fold(f64::INFINITY, f64::min),
total_all.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
);
assert!(
total_all.iter().any(|x| *x > 100_000.0),
"the `A-U` × `SGR0` rows are hundreds of thousands of firms — a cube of small \
numbers would mean the selection landed the wrong measure"
);
let sk = b
.column_by_name("Storleksklass")
.expect("Storleksklass")
.as_any()
.downcast_ref::<StringArray>()
.expect("Utf8");
let classes: std::collections::BTreeSet<&str> = (0..sk.len()).map(|i| sk.value(i)).collect();
println!("size classes: {classes:?}");
assert_eq!(classes.len(), 9, "all nine size classes, SGR0..SGR8");
// **The ceiling, stated as an assertion.** There is no orgnr in this cube and there
// never will be — that is what the SCB Företagsregistret key would buy.
let cols: Vec<String> = b.schema().fields().iter().map(|f| f.name().clone()).collect();
assert!(
!cols.iter().any(|c| c.to_lowercase().contains("orgnr")
|| c.to_lowercase().contains("organisation")),
"aggregates, never an orgnr — the per-company register is a different, \
credential-gated SCB service: {cols:?}"
);
}
/// **Asking for the whole cube is over SCB's cell cap, and the failure is HTML.**
///
/// The measurement that chose the selection above. The full cube is 1 823 × 9 × 1 × 18 =
/// 295 326 cells against a documented cap of 150 000, and SCB answers **HTTP 403 with an
/// IIS HTML error page** — not a 400, not a JSON error. So the failure a caller sees is a
/// transport error or a JSON parse error, never a message saying "too many cells", which
/// is exactly why the selection is pinned in code rather than left to each caller.
#[tokio::test]
#[ignore = "live network: SCB's public PxWeb Statistical Database"]
async fn asking_for_the_whole_cube_is_refused_and_not_in_json() {
let cfg = CollectorCfg::new(SCB_FDBR07N_URL, "scb_foretag_all").with_params(
serde_json::json!({
"selection": [
{"code": "SNI2007", "selection": {"filter": "all", "values": ["*"]}},
{"code": "Storleksklass", "selection": {"filter": "all", "values": ["*"]}},
{"code": "ContentsCode", "selection": {"filter": "item", "values": ["000002YB"]}},
{"code": "Tid", "selection": {"filter": "all", "values": ["*"]}}
],
"dims": SCB_FDBR07N_DIMS,
}),
);
let sink = BatchSink::memory();
let err = PxWebCollector
.collect(&cfg, Bounds::default(), &sink)
.await
.expect_err(
"295 326 cells is over SCB's documented 150 000-cell per-call cap and MUST \
be refused — a success here would mean the cap moved, and the pinned \
selection should then be widened deliberately rather than by accident",
);
println!("over-cap failure: {err}");
assert!(
sink.take_batches().is_empty(),
"and nothing partial was landed"
);
}
/// **The SCB table tree, walked live.**
///
/// One branch (`/NV`, Näringsverksamhet) rather than the whole SSD: the full walk is
/// ~1 + 22 + one request per statistics product at 3 req/s, which is a job and not a
/// test. What this proves is that the walk really descends, that folders and tables come
/// back distinguishable, and that `NV/NV0101/FDBR07N` — the cube the test above fetched —
/// is reachable BY PATH from the catalogue.
#[tokio::test]
#[ignore = "live network: SCB's public PxWeb Statistical Database"]
async fn the_scb_table_tree_walks_live_and_reaches_the_company_cube() {
let cfg = CollectorCfg::new(format!("{SCB_SSD_ROOT}/NV"), "pxweb_tables");
let sink = BatchSink::memory();
let stats = PxWebTableTree
.collect(&cfg, Bounds::default(), &sink)
.await
.expect("SCB's public PxWeb API must answer");
let batches = sink.take_batches();
let b = batches.first().expect("one batch");
println!("rows landed : {}", stats.rows);
println!("reason : {:?}", stats.reason);
let path = b
.column_by_name("path")
.expect("path")
.as_any()
.downcast_ref::<StringArray>()
.expect("Utf8");
let ty = b
.column_by_name("node_type")
.expect("node_type")
.as_any()
.downcast_ref::<StringArray>()
.expect("Utf8");
let folders = (0..ty.len()).filter(|i| ty.value(*i) == "l").count();
let tables = (0..ty.len()).filter(|i| ty.value(*i) == "t").count();
println!("folders : {folders}");
println!("tables : {tables}");
assert!(folders >= 10, "Näringsverksamhet holds many statistics products");
assert!(tables >= 30, "…and each holds tables: {tables}");
let paths: std::collections::BTreeSet<&str> = (0..path.len()).map(|i| path.value(i)).collect();
assert!(
paths.contains("NV0101/FDBR07N"),
"the SNI × storleksklass cube is reachable BY PATH from the catalogue — which is \
the whole reason this collector exists"
);
// Every path distinct: the key is real, on live data.
assert_eq!(
paths.len(),
path.len(),
"every node's path is distinct — the business key holds against the live tree"
);
// `updated` on tables only, and it is the source's own answer.
let upd = b
.column_by_name("updated")
.expect("updated")
.as_any()
.downcast_ref::<StringArray>()
.expect("Utf8");
let folders_with_stamp = (0..ty.len()).filter(|i| ty.value(*i) == "l" && !upd.is_null(*i)).count();
let tables_with_stamp = (0..ty.len()).filter(|i| ty.value(*i) == "t" && !upd.is_null(*i)).count();
println!("tables with an `updated` stamp: {tables_with_stamp}/{tables}");
assert_eq!(folders_with_stamp, 0, "PxWeb serves no `updated` for a folder");
assert!(
tables_with_stamp > tables / 2,
"…and serves one for most tables: {tables_with_stamp}/{tables}"
);
}