tests/csv_utf16.rs
//! **A UTF-16 CSV must not be read as UTF-8** — lane EC O6.
//!
//! Finansinspektionen's insider register (`marknadssok.fi.se`, the `Insyn` export) is
//! the source that found this: measured live 2026-09-04 it answers **HTTP 200,
//! `text/csv`, 524 096 B, 23 `;`-delimited columns — and the bytes are UTF-16LE with
//! NO byte-order mark**.
//!
//! That combination defeats the sniffer in the worst possible way. `NUL` is a VALID
//! UTF-8 byte, so `std::str::from_utf8` SUCCEEDS on UTF-16LE ASCII text and the sniffer
//! answers `Utf8` — every character then arrives followed by a NUL, the delimiter count
//! on the first line is unchanged, and the file parses. It parses into garbage: column
//! names like `P\0u\0b\0…`, and no error anywhere.
//!
//! So this is not "an encoding we do not support yet". It is a silent corruption, and
//! the assertions below are on the DECODED TEXT rather than on a "did it parse" flag.
#![cfg(feature = "csv")]
use korp_collectors::csv::{parse_csv, sniff_dialect, CsvEncoding};
/// The real FI header, as FI serves it: UTF-16LE, no BOM.
fn fi_bytes() -> Vec<u8> {
let text = "Publiceringsdatum;Emittent;LEI-kod;Person i ledande ställning\n\
2026-09-04 15:09:17;Harmoney BidCo;64887HUYX14K9T80J308;Nicolai Chamizo\n";
text.encode_utf16().flat_map(|u| u.to_le_bytes()).collect()
}
/// **The sniffer names UTF-16LE, and does not call it UTF-8.**
///
/// The trap is asserted where it is REAL: a run of pure-ASCII UTF-16LE is genuinely
/// valid UTF-8, so "valid UTF-8 ⇒ UTF-8" reads it as UTF-8 and corrupts it silently.
/// (FI's own header is not a clean example of the trap — `ställning` puts an `E4` in
/// there, and a lone `E4` is not valid UTF-8 — which is why the two halves are asserted
/// separately rather than on one fixture. A trap statement that is only sometimes true
/// is not a trap statement.)
#[test]
fn utf16le_without_a_bom_is_not_mistaken_for_utf8() {
// THE TRAP, on bytes where it holds: ASCII-only UTF-16LE, no BOM.
let ascii_utf16: Vec<u8> = "Publiceringsdatum;Emittent;LEI-kod\n2026-09-04;X;Y\n"
.encode_utf16()
.flat_map(|u| u.to_le_bytes())
.collect();
assert!(
std::str::from_utf8(&ascii_utf16).is_ok(),
"ASCII-only UTF-16LE IS valid UTF-8 — a validity check cannot decide this, \
which is the whole reason the sniffer needed a shape rule"
);
let d = sniff_dialect(&ascii_utf16);
assert_eq!(d.encoding, CsvEncoding::Utf16Le, "sniffed: {:?}", d.encoding);
assert_eq!(d.delimiter, b';', "the delimiter survives the decode");
// …and FI's real header, which is UTF-16LE with Swedish characters in it.
let d = sniff_dialect(&fi_bytes());
assert_eq!(d.encoding, CsvEncoding::Utf16Le, "sniffed: {:?}", d.encoding);
assert_eq!(d.delimiter, b';');
}
/// **The decoded columns are the real ones** — the applied output, not a parse flag.
#[test]
fn a_utf16le_csv_decodes_to_the_columns_fi_actually_serves() {
let b = fi_bytes();
let (schema, batches) = parse_csv(&b, &sniff_dialect(&b)).expect("parse");
let names: Vec<String> = schema.fields().iter().map(|f| f.name().clone()).collect();
assert_eq!(
names,
vec![
"Publiceringsdatum".to_string(),
"Emittent".to_string(),
"LEI-kod".to_string(),
"Person i ledande ställning".to_string(),
],
"the column names must be the REAL ones — `P\\0u\\0b\\0…` is what a UTF-8 read \
of these bytes produces, and it produces it silently"
);
// The å in `ställning` proves the decode, not just the NUL-stripping.
assert!(names[3].contains('ä'));
let rows: usize = batches.iter().map(|b| b.num_rows()).sum();
assert_eq!(rows, 1, "one data row");
// And the LEI is intact and joinable — this is the column the whole O4/O6 chain
// exists for (`fi_insyn.lei` → `gleif_lei.lei` → `orgnr` → a Company node).
let b0 = &batches[0];
let lei = b0
.column_by_name("LEI-kod")
.expect("LEI-kod column")
.as_any()
.downcast_ref::<arrow_array::StringArray>()
.expect("utf8");
assert_eq!(lei.value(0), "64887HUYX14K9T80J308");
}
/// **A BOM'd UTF-16 file works too, in both byte orders** — and a genuine UTF-8 file is
/// still read as UTF-8, so the new rule cannot have swallowed the common case.
#[test]
fn both_byte_orders_are_read_and_plain_utf8_is_untouched() {
let text = "a;b\n1;2\n";
let le: Vec<u8> = std::iter::once(0xFEFFu16)
.chain(text.encode_utf16())
.flat_map(|u| u.to_le_bytes())
.collect();
let be: Vec<u8> = std::iter::once(0xFEFFu16)
.chain(text.encode_utf16())
.flat_map(|u| u.to_be_bytes())
.collect();
for (bytes, want) in [(&le, CsvEncoding::Utf16Le), (&be, CsvEncoding::Utf16Be)] {
let d = sniff_dialect(bytes);
assert_eq!(d.encoding, want, "{want:?}");
let (schema, batches) = parse_csv(bytes, &d).expect("parse");
let names: Vec<String> = schema.fields().iter().map(|f| f.name().clone()).collect();
assert_eq!(names, vec!["a".to_string(), "b".to_string()], "{want:?}");
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
}
// The common case is unchanged, including a Latin-1 file — the new rule must not
// have widened its way over either.
assert_eq!(sniff_dialect(b"a,b\n1,2\n").encoding, CsvEncoding::Utf8);
let latin1 = b"namn,ort\nB\xe5t,Ume\xe5\n";
assert_eq!(sniff_dialect(latin1).encoding, CsvEncoding::Latin1);
}