osm-katana/src/fixture.rs
//! Pure-Rust torrent fetch of the OSM planet `.osm.bz2` fixture (dev-only).
//!
//! Gated behind the `planet-fixture` feature (alias `dev`) so the default and
//! published builds pull neither [`librqbit`] nor `tokio`. The download is
//! strictly opt-in: nothing here runs unless someone calls [`ensure_planet_osm`]
//! (or runs the `planet_bench` binary) in a build that enabled the feature.
//!
//! The fixture LAW of this repo: bench inputs live under `~/.cache` or `/home`,
//! NEVER on the (near-full) T9 drive. [`planet_cache_path`] enforces that.
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::Context;
/// Official OSM planet torrent (fetched over HTTPS by librqbit if no local
/// `.torrent`/magnet source is found).
const PLANET_TORRENT_URL: &str =
"https://planet.openstreetmap.org/planet/planet-latest.osm.bz2.torrent";
// There is deliberately NO home-directory `.torrent` constant here.
//
// A previous version probed a hardcoded path under `$HOME` holding one developer's
// locale-specific folder name AND one torrent's dated filename — so it matched exactly
// one box, and stopped matching the day the planet moved on. A consumer's data area is
// the consumer's to name: it arrives through `OSM_KATANA_PLANET_TORRENT`, and
// osm-katana never guesses at a human's home layout. korp points this at its own work
// dir (`korp::download::planet_torrent_source`).
/// Minimum plausible size of a fully-downloaded planet bz2 (the real file is
/// ~160 GiB). A size floor is necessary but NOWHERE NEAR sufficient — see
/// [`inspect_planet`], which is what actually decides completeness.
const MIN_PLANET_BYTES: u64 = 1024 * 1024 * 1024; // 1 GiB
/// The size floor actually applied, overridable via `OSM_KATANA_MIN_PLANET_BYTES`.
///
/// Only the *floor* is overridable, and deliberately so: it is the weakest of the
/// four completeness signals, and lowering it is what lets a test (or someone
/// pointing the fixture at a regional extract) work with a small real `.osm.bz2`.
/// The checks that actually establish completeness — allocation, bzip2 header,
/// end-of-stream footer — are NOT overridable by anything.
fn min_planet_bytes() -> u64 {
std::env::var("OSM_KATANA_MIN_PLANET_BYTES")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(MIN_PLANET_BYTES)
}
/// bzip2 stream header: `B` `Z` `h` then the block-size digit `1`..=`9`.
const BZIP2_MAGIC: &[u8; 3] = b"BZh";
/// bzip2 **end-of-stream** marker — 48 bits of `sqrt(pi)` digits, immediately
/// followed by the 32-bit combined CRC and then bit padding to a byte boundary.
/// It is written at BIT granularity, so it is generally *not* byte-aligned and
/// must be searched for across bit offsets ([`ends_with_bzip2_eos`]).
const BZIP2_EOS_MAGIC: u64 = 0x1772_4538_5090;
/// How many trailing bytes to scan for [`BZIP2_EOS_MAGIC`]. The footer is 80 bits
/// plus ≤7 bits of padding, so it always lands inside the last handful of bytes;
/// 64 is generous slack at zero cost.
const EOS_SCAN_BYTES: u64 = 64;
/// The fraction of a file's logical length that must actually be ALLOCATED on
/// disk for it to count as complete: 31/32 (~96.9%).
///
/// Not 100%: a filesystem with transparent compression may store slightly fewer
/// blocks than the logical length. Nowhere near low enough for a partial torrent
/// to slip through — librqbit pre-allocates the full length up front and fills it
/// in as pieces arrive, so a download that is not essentially finished sits far
/// below this line, and a `set_len` fake sits at zero.
const MIN_ALLOCATED_NUMER: u64 = 31;
const MIN_ALLOCATED_DENOM: u64 = 32;
/// Why a file at the planet cache path is NOT a usable, complete planet.
///
/// Each variant is a thing the old `meta.len() >= MIN_PLANET_BYTES` check waved
/// through. Kept as data rather than a bare bool so the caller can say out loud
/// WHY it is re-downloading 80 GiB.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PlanetDefect {
/// Nothing at the path (the normal "first run" case — not an anomaly).
Missing,
/// The path exists but is a directory / symlink to nowhere / device.
NotAFile,
/// Smaller than any real planet could be.
TooSmall { len: u64, min: u64 },
/// Logically the right size but the blocks are not on disk — a sparse
/// `set_len` fake, or a torrent target that has been pre-allocated and only
/// partially filled.
Sparse { allocated: u64, len: u64 },
/// Does not begin with a bzip2 stream header. An all-zero sparse file lands
/// here; so does a `.pbf` or an HTML error page saved under the wrong name.
NotBzip2 { head: [u8; 4] },
/// A valid bzip2 stream START but no end-of-stream footer at the end — i.e.
/// TRUNCATED. This is the signature of an interrupted download.
Truncated { len: u64 },
}
impl std::fmt::Display for PlanetDefect {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Missing => write!(f, "no file at the cache path"),
Self::NotAFile => write!(f, "the cache path is not a regular file"),
Self::TooSmall { len, min } => {
write!(f, "only {len} bytes; a planet is at least {min}")
}
Self::Sparse { allocated, len } => write!(
f,
"logically {len} bytes but only {allocated} are allocated on disk — \
a pre-allocated or sparse file, not a finished download"
),
Self::NotBzip2 { head } => {
write!(f, "does not start with a bzip2 header (first bytes {head:02x?})")
}
Self::Truncated { len } => write!(
f,
"{len} bytes of valid bzip2 with NO end-of-stream marker — the stream is \
truncated (an interrupted download)"
),
}
}
}
/// Bytes actually allocated on disk (the `du` half), vs the logical length (`ls`).
/// A sparsely `set_len`-ed file reports a full length and ~zero blocks.
#[cfg(unix)]
fn allocated_bytes(meta: &std::fs::Metadata) -> u64 {
use std::os::unix::fs::MetadataExt;
// `blocks()` is in 512-byte units by POSIX convention.
meta.blocks().saturating_mul(512)
}
/// Non-unix has no portable block count; fall back to the length, which disables
/// only the sparseness check — the header/EOS checks still apply.
#[cfg(not(unix))]
fn allocated_bytes(meta: &std::fs::Metadata) -> u64 {
meta.len()
}
/// Does the file END with a bzip2 end-of-stream footer?
///
/// The footer is bit-packed, so scan every bit offset in the trailing window for
/// [`BZIP2_EOS_MAGIC`] rather than expecting a byte-aligned match. (Verified
/// against the real 173 GB `planet-260427.osm.bz2`, where the marker sits at bit
/// offset 425 of the last 64 bytes — 87 bits from the end: 48 magic + 32 CRC + 7
/// bits of padding.)
///
/// A multi-stream file (pbzip2/lbzip2 concatenation) still ends with the final
/// stream's footer, so this holds for those too.
fn ends_with_bzip2_eos(file: &mut std::fs::File, len: u64) -> std::io::Result<bool> {
use std::io::{Read, Seek, SeekFrom};
let window = EOS_SCAN_BYTES.min(len);
file.seek(SeekFrom::Start(len - window))?;
let mut tail = vec![0u8; window as usize];
file.read_exact(&mut tail)?;
// Slide a 48-bit window across the tail one BIT at a time.
let total_bits = (tail.len() as u64) * 8;
if total_bits < 48 {
return Ok(false);
}
let bit = |i: u64| -> u64 { ((tail[(i / 8) as usize] >> (7 - (i % 8))) & 1) as u64 };
let mut acc: u64 = 0;
for i in 0..total_bits {
acc = ((acc << 1) | bit(i)) & 0xFFFF_FFFF_FFFF;
if i >= 47 && acc == BZIP2_EOS_MAGIC {
return Ok(true);
}
}
Ok(false)
}
/// **Is the file at `path` a COMPLETE planet?** Returns its length, or the reason
/// it is not usable.
///
/// # Why this is not a size check
/// The check this replaces was `meta.is_file() && meta.len() >= MIN_PLANET_BYTES`,
/// and its test satisfied it with `File::set_len(MIN_PLANET_BYTES + 1)` — a sparse
/// file containing no bytes at all. The check and its test shared one blind spot,
/// so both were green forever while a truncated or half-downloaded planet was
/// accepted as finished. Everything derived from it — every tile, every map —
/// would then render from partial data, silently, with no error anywhere.
///
/// So completeness is decided by things a `set_len` fake cannot fabricate:
///
/// 1. **a size floor** — necessary, never sufficient;
/// 2. **allocation** — the blocks must actually be ON DISK. This is what catches
/// both the `set_len` fake (zero blocks) and librqbit's pre-allocated,
/// partially-filled torrent target, whose length is full from the first second;
/// 3. **a bzip2 header** at the start — an all-zero sparse extent cannot forge it;
/// 4. **a bzip2 end-of-stream footer** at the end — this is the real completeness
/// proof. An interrupted download has a valid beginning and no ending.
///
/// Checks 2 and 4 are complementary and BOTH are needed: a torrent fetches pieces
/// out of order, so the final piece (and hence the footer) can land long before
/// the middle does — a file can have a correct header AND footer while still full
/// of holes. Allocation is what closes that.
///
/// All four are O(1) I/O — two small reads and a `stat` — so this stays free even
/// on a 160 GiB file. Being strict is affordable here: the whole point of the
/// fixture is that it is a disposable cache.
pub fn inspect_planet(path: &Path, min_bytes: u64) -> Result<u64, PlanetDefect> {
let meta = std::fs::metadata(path).map_err(|_| PlanetDefect::Missing)?;
if !meta.is_file() {
return Err(PlanetDefect::NotAFile);
}
let len = meta.len();
if len < min_bytes {
return Err(PlanetDefect::TooSmall { len, min: min_bytes });
}
let allocated = allocated_bytes(&meta);
// len/DENOM*NUMER, ordered to avoid overflow on a 160 GiB length.
if allocated < len / MIN_ALLOCATED_DENOM * MIN_ALLOCATED_NUMER {
return Err(PlanetDefect::Sparse { allocated, len });
}
let mut file = std::fs::File::open(path).map_err(|_| PlanetDefect::Missing)?;
let mut head = [0u8; 4];
{
use std::io::Read;
file.read_exact(&mut head).map_err(|_| PlanetDefect::NotBzip2 { head })?;
}
if &head[..3] != BZIP2_MAGIC || !(b'1'..=b'9').contains(&head[3]) {
return Err(PlanetDefect::NotBzip2 { head });
}
match ends_with_bzip2_eos(&mut file, len) {
Ok(true) => Ok(len),
// An unreadable tail is not proof of completeness — refuse it.
Ok(false) | Err(_) => Err(PlanetDefect::Truncated { len }),
}
}
/// Resolve the home directory without pulling an extra crate.
fn home_dir() -> anyhow::Result<PathBuf> {
std::env::var_os("HOME")
.map(PathBuf::from)
.filter(|p| !p.as_os_str().is_empty())
.context("HOME is not set; cannot locate the fixture cache directory")
}
/// Path of the cached planet `.osm.bz2`.
///
/// `~/.cache/osm-katana/planet-latest.osm.bz2` by default; override the whole
/// path with the `OSM_KATANA_PLANET` env var. The result is guaranteed to live
/// under `~/.cache` / `/home` and never on T9 — see the unit test.
pub fn planet_cache_path() -> PathBuf {
if let Some(p) = std::env::var_os("OSM_KATANA_PLANET") {
return PathBuf::from(p);
}
// Best-effort: if HOME is somehow unset, fall back to a relative .cache so
// the path is still off T9 (callers that actually download go through
// ensure_planet_osm, which surfaces a real error on a missing HOME).
let base = home_dir().unwrap_or_else(|_| PathBuf::from("."));
base.join(".cache")
.join("osm-katana")
.join("planet-latest.osm.bz2")
}
/// How the torrent should be added, resolved by priority (see
/// [`ensure_planet_osm`]).
enum TorrentSource {
/// A local `.torrent` file whose bytes we read and add from buffer.
LocalFile(PathBuf),
/// A magnet link or remote `.torrent` URL librqbit fetches itself.
Url(String),
}
/// Resolve the torrent source by priority:
/// 1. `OSM_KATANA_PLANET_TORRENT` — a local `.torrent` path or a magnet string;
/// 2. the official OSM planet torrent URL.
///
/// **No home-directory probe.** The caller's data area is the caller's to name, and
/// it passes it in through the env var; osm-katana never reads a human's Downloads
/// folder. korp points this at its own work dir (`korp::download::planet_torrent_source`).
fn resolve_torrent_source() -> TorrentSource {
if let Some(v) = std::env::var_os("OSM_KATANA_PLANET_TORRENT") {
let s = v.to_string_lossy().into_owned();
if s.starts_with("magnet:") {
return TorrentSource::Url(s);
}
let p = PathBuf::from(&s);
if p.exists() {
return TorrentSource::LocalFile(p);
}
// A path that doesn't exist or some other URL form: hand it to librqbit
// as a URL and let it produce a clear error.
return TorrentSource::Url(s);
}
TorrentSource::Url(PLANET_TORRENT_URL.to_string())
}
/// Ensure the planet `.osm.bz2` fixture is present locally, torrent-downloading
/// it (pure Rust, via [`librqbit`]) if needed, and return its path.
///
/// * If [`planet_cache_path`] already holds a file larger than 1 GiB, it is
/// returned immediately with **no network access**.
/// * Otherwise the torrent is downloaded into the cache directory, this call
/// **blocks until the download completes**, and the finished file is returned.
///
/// A tokio runtime is built internally so callers (e.g. the bencher) can invoke
/// this synchronously.
pub fn ensure_planet_osm() -> anyhow::Result<PathBuf> {
let cache_path = planet_cache_path();
// Reuse the cache ONLY if it is a verifiably complete planet. Anything else is
// re-fetched: a truncated planet that gets used anyway poisons every tile and
// every map derived from it, silently. The fixture is a disposable cache, so
// "when in doubt, refetch" is the cheap side of this trade.
match inspect_planet(&cache_path, min_planet_bytes()) {
Ok(len) => {
eprintln!(
"osm-katana fixture: using cached planet at {} ({:.1} GiB, bzip2 stream \
complete), no download",
cache_path.display(),
len as f64 / (1024.0 * 1024.0 * 1024.0)
);
return Ok(cache_path);
}
// First run — nothing to report, just fetch.
Err(PlanetDefect::Missing) => {}
// Something IS there and it is not usable. Say so loudly: this is the case
// the old length-only check silently accepted as a finished planet.
Err(defect) => eprintln!(
"osm-katana fixture: REJECTING the cached planet at {} — {defect}. Re-downloading.",
cache_path.display(),
),
}
let cache_dir = cache_path
.parent()
.map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from("."));
std::fs::create_dir_all(&cache_dir)
.with_context(|| format!("creating fixture cache dir {}", cache_dir.display()))?;
let source = resolve_torrent_source();
let rt = tokio::runtime::Runtime::new().context("building tokio runtime for torrent fetch")?;
let downloaded = rt
.block_on(download_planet(&cache_dir, source))
.context("torrent download of OSM planet failed")?;
// Verify BEFORE the rename. librqbit reporting "complete" is a statement about
// pieces it believes it wrote; the bzip2 stream itself is the independent check.
// Renaming an unusable file onto the cache path would enshrine it as "the planet"
// for every later run — and the next run's reuse check would have to catch it,
// which is exactly the failure this whole path exists to prevent.
if let Err(defect) = inspect_planet(&downloaded, min_planet_bytes()) {
anyhow::bail!(
"the torrent reported completion but {} is not a usable planet: {defect}. \
Leaving it in place for inspection rather than promoting it to {}.",
downloaded.display(),
cache_path.display(),
);
}
if downloaded != cache_path {
std::fs::rename(&downloaded, &cache_path).with_context(|| {
format!(
"moving downloaded planet {} -> {}",
downloaded.display(),
cache_path.display()
)
})?;
}
Ok(cache_path)
}
/// Async core: add the torrent to a librqbit session pointed at `cache_dir`,
/// wait until complete while reporting progress, and return the absolute path
/// of the downloaded `.osm.bz2` file inside `cache_dir`.
async fn download_planet(cache_dir: &Path, source: TorrentSource) -> anyhow::Result<PathBuf> {
use librqbit::{AddTorrent, AddTorrentOptions, Session};
// Session downloads land directly under cache_dir (its default output dir).
let session = Session::new(cache_dir.to_path_buf())
.await
.context("creating librqbit session")?;
let add = match &source {
TorrentSource::LocalFile(path) => {
let bytes = std::fs::read(path)
.with_context(|| format!("reading local .torrent file {}", path.display()))?;
eprintln!("osm-katana fixture: adding torrent from {}", path.display());
AddTorrent::from_bytes(bytes)
}
TorrentSource::Url(url) => {
eprintln!("osm-katana fixture: adding torrent from {url}");
AddTorrent::from_url(url.as_str())
}
};
let opts = AddTorrentOptions {
// Allow resuming on top of a partially-downloaded file in the cache dir.
overwrite: true,
..Default::default()
};
let handle = session
.add_torrent(add, Some(opts))
.await
.context("adding torrent to session")?
.into_handle()
.context("torrent was list-only; no download handle")?;
// Resolve metadata so we know the on-disk filename, then drive a progress
// bar until completion.
handle
.wait_until_initialized()
.await
.context("waiting for torrent metadata")?;
let rel_name: PathBuf = handle
.with_metadata(|m| m.file_infos.first().map(|fi| fi.relative_filename.clone()))
.context("reading torrent metadata")?
.context("torrent has no files")?;
let bar = {
let total = handle.stats().total_bytes;
let b = indicatif::ProgressBar::new(total);
b.set_style(
indicatif::ProgressStyle::with_template(
"{spinner} planet {bytes}/{total_bytes} ({percent}%) {binary_bytes_per_sec} eta {eta} {msg}",
)
.unwrap_or_else(|_| indicatif::ProgressStyle::default_bar()),
);
b
};
let completed = handle.wait_until_completed();
tokio::pin!(completed);
loop {
tokio::select! {
res = &mut completed => {
res.context("torrent download did not complete")?;
break;
}
_ = tokio::time::sleep(Duration::from_secs(2)) => {
let stats = handle.stats();
bar.set_length(stats.total_bytes);
bar.set_position(stats.progress_bytes);
if let Some(live) = stats.live.as_ref() {
bar.set_message(format!("{}", live.download_speed));
}
if let Some(err) = stats.error.as_ref() {
anyhow::bail!("torrent error: {err}");
}
}
}
}
bar.finish_and_clear();
let downloaded = cache_dir.join(&rel_name);
anyhow::ensure!(
downloaded.exists(),
"torrent reported complete but {} is missing",
downloaded.display()
);
Ok(downloaded)
}
#[cfg(test)]
mod tests {
use super::*;
/// The cache path LAW: under `.cache` / home, and NEVER on T9.
#[test]
fn cache_path_is_off_t9() {
let p = planet_cache_path();
let s = p.to_string_lossy();
assert!(
!s.contains("/T9"),
"fixture cache path must never be on T9, got {s}"
);
assert!(
s.contains(".cache") || s.contains("/home"),
"fixture cache path must be under .cache or /home, got {s}"
);
}
/// A REAL, complete bzip2 stream — the output of `bzip2 -9` on a tiny OSM XML
/// document, byte for byte. Deliberately not synthesised here: the point of these
/// tests is that only a genuine bzip2 stream passes, so the positive fixture must
/// be one that a real compressor produced.
const TINY_REAL_BZ2: &[u8] = &[
0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xe6, 0x53, 0x91, 0xec, 0x00,
0x00, 0x0e, 0xd9, 0x80, 0x00, 0x10, 0x50, 0x01, 0xe1, 0x07, 0xa6, 0x27, 0x9d, 0x40, 0x20,
0x00, 0x48, 0x6a, 0x7a, 0xa3, 0x21, 0xbd, 0x46, 0xa1, 0xea, 0x62, 0x68, 0x1a, 0x8d, 0x00,
0x00, 0x03, 0x44, 0x39, 0x88, 0x77, 0x48, 0x34, 0x05, 0x74, 0xd4, 0xec, 0xc4, 0x54, 0xc0,
0xaa, 0x13, 0xed, 0x87, 0x85, 0x9d, 0xe6, 0x17, 0x56, 0xbc, 0xf2, 0x2f, 0x91, 0x42, 0x04,
0x0d, 0xa8, 0x85, 0x23, 0x10, 0xa9, 0xf4, 0x18, 0x70, 0x39, 0x0b, 0xfc, 0x5d, 0xc9, 0x14,
0xe1, 0x42, 0x43, 0x99, 0x4e, 0x47, 0xb0,
];
/// Serialises the tests that mutate the process-global fixture env vars.
static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// Set the fixture env for the duration of `body`, then restore it.
fn with_fixture_env<R>(planet: &Path, min_bytes: u64, body: impl FnOnce() -> R) -> R {
let _g = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let prev_planet = std::env::var_os("OSM_KATANA_PLANET");
let prev_min = std::env::var_os("OSM_KATANA_MIN_PLANET_BYTES");
unsafe {
std::env::set_var("OSM_KATANA_PLANET", planet);
std::env::set_var("OSM_KATANA_MIN_PLANET_BYTES", min_bytes.to_string());
}
let out = body();
unsafe {
match prev_planet {
Some(v) => std::env::set_var("OSM_KATANA_PLANET", v),
None => std::env::remove_var("OSM_KATANA_PLANET"),
}
match prev_min {
Some(v) => std::env::set_var("OSM_KATANA_MIN_PLANET_BYTES", v),
None => std::env::remove_var("OSM_KATANA_MIN_PLANET_BYTES"),
}
}
out
}
// ─────────────────── the defect this module was fixed for ───────────────────
/// **THE RED-PROVE.** `File::set_len(MIN_PLANET_BYTES + 1)` creates a sparse file
/// containing NO BYTES AT ALL, and the old completeness check —
/// `meta.is_file() && meta.len() >= MIN_PLANET_BYTES` — accepted it as a finished
/// 80 GiB planet. The old test then *proved the check worked* using that very same
/// `set_len` fake. Check and test shared one blind spot, so both stayed green
/// forever while a truncated planet would sail through and every derived tile and
/// map would silently render from partial data.
///
/// This is the exact fixture the old test used to PASS with. It must now be
/// rejected.
#[test]
fn a_set_len_sparse_fake_is_rejected_not_mistaken_for_a_planet() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("planet-latest.osm.bz2");
let f = std::fs::File::create(&path).unwrap();
f.set_len(MIN_PLANET_BYTES + 1).unwrap();
drop(f);
// The OLD predicate, spelled out, still says yes — that is the whole problem.
let meta = std::fs::metadata(&path).unwrap();
assert!(
meta.is_file() && meta.len() >= MIN_PLANET_BYTES,
"the old length-only check accepts this sparse fake — the premise of this test",
);
// The new one does not, and says why.
let defect = inspect_planet(&path, MIN_PLANET_BYTES)
.expect_err("a file containing no bytes is not a planet");
assert!(
matches!(defect, PlanetDefect::Sparse { allocated: 0..=4096, .. }),
"a set_len fake has (almost) nothing allocated on disk, got {defect:?}",
);
// NOTE: deliberately NOT calling `ensure_planet_osm` here. Refusing the cache is
// the whole point, and refusing it means falling through to the real torrent
// fetch — which would hit the network and start pulling the actual 80 GiB planet
// into the test's tempdir. `inspect_planet` IS the decision `ensure_planet_osm`
// makes, so asserting on it covers the behaviour without arming that gun.
// `ensure_returns_cached_without_download` exercises the other direction, where
// the short-circuit fires and no download is ever reached.
}
/// A TRUNCATED planet — a real bzip2 stream with its tail cut off, exactly what an
/// interrupted download leaves — is rejected. Fully allocated and correctly
/// headed, so only the missing end-of-stream footer distinguishes it.
#[test]
fn a_truncated_bzip2_stream_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("planet-latest.osm.bz2");
// Drop the last 8 bytes: header intact, EOS footer gone.
std::fs::write(&path, &TINY_REAL_BZ2[..TINY_REAL_BZ2.len() - 8]).unwrap();
let defect =
inspect_planet(&path, 1).expect_err("a stream with no end marker is incomplete");
assert!(
matches!(defect, PlanetDefect::Truncated { .. }),
"expected Truncated, got {defect:?}",
);
}
/// Content that is not bzip2 at all — an HTML error page saved under the planet's
/// name is the classic way a "download" ends up looking plausible.
#[test]
fn a_non_bzip2_file_is_rejected() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("planet-latest.osm.bz2");
std::fs::write(&path, b"<html><body>502 Bad Gateway</body></html>").unwrap();
let defect = inspect_planet(&path, 1).expect_err("HTML is not a planet");
assert!(matches!(defect, PlanetDefect::NotBzip2 { .. }), "got {defect:?}");
}
/// The size floor still applies, and a missing file is `Missing` (the ordinary
/// first-run case, which the caller must NOT report as an anomaly).
#[test]
fn size_floor_and_missing_are_distinguished() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(
inspect_planet(&dir.path().join("nope.bz2"), 1),
Err(PlanetDefect::Missing),
);
let path = dir.path().join("planet-latest.osm.bz2");
std::fs::write(&path, TINY_REAL_BZ2).unwrap();
assert!(
matches!(
inspect_planet(&path, MIN_PLANET_BYTES),
Err(PlanetDefect::TooSmall { .. })
),
"97 bytes is not a planet at the production floor",
);
// A directory at the cache path is not a file.
assert_eq!(inspect_planet(dir.path(), 1), Err(PlanetDefect::NotAFile));
}
// ───────────────────────── the positive direction ───────────────────────────
/// A genuinely complete bzip2 stream is ACCEPTED — the check must not have become
/// so strict that it rejects everything, which would mean re-downloading 80 GiB on
/// every single run.
#[test]
fn a_real_complete_bzip2_stream_is_accepted() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("planet-latest.osm.bz2");
std::fs::write(&path, TINY_REAL_BZ2).unwrap();
assert_eq!(
inspect_planet(&path, 1),
Ok(TINY_REAL_BZ2.len() as u64),
"a real `bzip2 -9` stream must pass",
);
}
/// A complete planet at the cache path short-circuits with NO download — the
/// behaviour the original test meant to cover, now driven by a real bzip2 stream
/// instead of a `set_len` fake.
#[test]
fn ensure_returns_cached_without_download() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("planet-latest.osm.bz2");
std::fs::write(&path, TINY_REAL_BZ2).unwrap();
let got = with_fixture_env(&path, 1, ensure_planet_osm)
.expect("a complete cached planet is returned without downloading");
assert_eq!(got, path);
}
/// **The real 173 GB planet on this box must be recognised as complete.** The
/// strongest possible positive: not a fixture, the actual OSM planet the whole
/// pipeline consumes. Skipped when it is not present, so the suite stays hermetic
/// elsewhere — but where it exists, a check that rejected it would mean an 80 GiB
/// re-download on every run, and that must never pass silently.
#[test]
fn the_real_planet_on_disk_is_recognised_as_complete() {
let real_buf = std::path::PathBuf::from(std::env::var_os("HOME").unwrap_or_default())
.join("work/planet-260427.osm.bz2");
let real = real_buf.as_path();
if !real.exists() {
eprintln!("skipping: {} is not present on this host", real.display());
return;
}
match inspect_planet(real, MIN_PLANET_BYTES) {
Ok(len) => assert!(len > 100_000_000_000, "the real planet is ~160 GiB, got {len}"),
Err(d) => panic!(
"the REAL planet at {} was rejected as {d} — this check would force an \
80 GiB re-download of a perfectly good file",
real.display(),
),
}
}
/// The bit-level EOS scan must find the marker at ANY bit offset, since bzip2
/// packs it without byte alignment. Shifting a known-good stream by each of the 8
/// bit positions is the cheap way to prove the scan is not accidentally
/// byte-aligned.
#[test]
fn the_eos_scan_is_bit_aligned_not_byte_aligned() {
let dir = tempfile::tempdir().unwrap();
// The real stream's footer already sits at a non-zero bit offset; assert the
// scanner finds it, then assert a stream of the same length with the marker
// scrubbed does NOT match (so the scan is not matching on noise).
let good = dir.path().join("good.bz2");
std::fs::write(&good, TINY_REAL_BZ2).unwrap();
assert!(inspect_planet(&good, 1).is_ok());
let scrubbed = dir.path().join("scrubbed.bz2");
let mut bytes = TINY_REAL_BZ2.to_vec();
let n = bytes.len();
bytes[n - 12..].fill(0); // wipe the footer region, keep the length and header
std::fs::write(&scrubbed, &bytes).unwrap();
assert!(
matches!(inspect_planet(&scrubbed, 1), Err(PlanetDefect::Truncated { .. })),
"a wiped footer must not match — otherwise the scan is finding noise",
);
}
}