znippy-common/src/common_config.rs
use crate::hostinfo;
use once_cell::sync::Lazy;
use std::cmp::min;
#[derive(Debug, Clone)] //
pub struct StrategicConfig {
pub max_core_allowed: usize,
pub max_core_in_flight: usize,
pub max_core_in_compress: usize,
pub max_mem_allowed: u64,
pub min_free_memory_ratio: f32,
pub file_split_block_size: u64,
pub max_chunks: u32,
pub compression_level: i32,
pub zstd_output_buffer_size: usize,
}
pub static CONFIG: Lazy<StrategicConfig> = Lazy::new(strategic_confi_large);
/// Fraction of the memory actually available to this process that the compress
/// slot pool may reserve. The pool is one of several allocations a compress run
/// makes (per-worker zstd/OpenZL scratch, the index batches, the OS page cache),
/// so it must not claim everything that is free.
const SLOT_POOL_MEM_FRACTION: f64 = 0.5;
/// Bytes the compress slot pool is allowed to reserve on this host **right now**.
///
/// The pool used to be a hardcoded `8 × 200 MiB = 1.6 GiB` reserved before a
/// single file was opened, independent of both the input size and the memory the
/// process actually has. On a backup pod with a modest limit that is an
/// OOM-kill. This is the memory half of the bound (`PoolPlan::plan` applies the
/// input half).
///
/// Order of precedence:
/// 1. `ZNIPPY_SLOT_POOL_BYTES` — an explicit ops override, accepting a plain
/// byte count or a `KiB`/`MiB`/`GiB` (also `KB`/`MB`/`GB`) suffix.
/// 2. The **cgroup** limit minus what the cgroup is already using — this is the
/// number that actually kills a container, and it is not the host's free RAM.
/// 3. The host's available memory.
///
/// A fraction ([`SLOT_POOL_MEM_FRACTION`]) of whichever bound applies is
/// returned. Never 0: `PoolPlan` floors the pool at one slice regardless, so a
/// pathological reading cannot deadlock the pipeline — it just makes it slow.
///
/// # Tier 2 was dead until 2026-08-04
///
/// This used `sysinfo::System::cgroup_limits()`, which is `limits_for_system()` —
/// it reads the cgroup-v2 **root** (`/sys/fs/cgroup/memory.max`,
/// `/sys/fs/cgroup/memory.current`). The root cgroup has no memory-controller
/// files of its own, so that call returns `None` on a standard cgroup-v2 host
/// *even when this process sits under a hard limit*. Tier 2 was documented,
/// green, and unreachable; every run fell through to tier 3, the host's free RAM
/// — precisely the number the doc comment says is the wrong one in a container.
/// [`hostinfo::cgroup_memory`] follows `/proc/self/cgroup` instead, so the tier
/// now fires. Measured on oden 2026-08-04: sysinfo `None`, hostinfo
/// `total_memory = 472446402560`.
pub fn slot_pool_budget_bytes() -> u64 {
if let Some(explicit) = std::env::var("ZNIPPY_SLOT_POOL_BYTES")
.ok()
.and_then(|raw| parse_byte_size(&raw))
{
log::info!("[slot_pool] budget {explicit} bytes (ZNIPPY_SLOT_POOL_BYTES)");
return explicit;
}
let host_available = hostinfo::mem_available_bytes().unwrap_or(0);
let headroom = match hostinfo::cgroup_memory() {
// In a container the cgroup ceiling — not the host's free RAM — is what
// the OOM killer measures against.
Some(cg) => cg.total_memory.saturating_sub(cg.rss).min(host_available.max(1)),
None => host_available,
};
let budget = (headroom as f64 * SLOT_POOL_MEM_FRACTION) as u64;
log::info!(
"[slot_pool] budget {} bytes ({:.0}% of {} bytes headroom)",
budget,
SLOT_POOL_MEM_FRACTION * 100.0,
headroom
);
budget
}
/// Parse `"268435456"`, `"256MiB"`, `"1 GiB"`, `"512MB"` → bytes.
/// Returns `None` for anything it cannot read, so a typo falls back to detection
/// rather than silently reserving something absurd.
pub fn parse_byte_size(raw: &str) -> Option<u64> {
let s = raw.trim();
let (digits, unit) = s.split_at(s.find(|c: char| !c.is_ascii_digit()).unwrap_or(s.len()));
let n: u64 = digits.parse().ok()?;
let mult = match unit.trim().to_ascii_lowercase().as_str() {
"" | "b" => 1u64,
"k" | "kb" | "kib" => 1024,
"m" | "mb" | "mib" => 1024 * 1024,
"g" | "gb" | "gib" => 1024 * 1024 * 1024,
_ => return None,
};
n.checked_mul(mult)
}
pub fn strategic_config(resource: f32) -> StrategicConfig {
let total_memory = hostinfo::mem_total_bytes().unwrap_or(0);
// SMT siblings share an execution unit, so physical cores is the honest
// width for a compress pipeline; logical count is the fallback when
// /proc/cpuinfo carries no topology (most non-x86).
let max_core_allowed =
hostinfo::physical_core_count().unwrap_or_else(hostinfo::logical_cpu_count);
let max_core_in_flight = ((max_core_allowed as f32) * 0.90).ceil() as usize;
let max_core_in_compress = max_core_allowed.saturating_sub(max_core_in_flight);
let min_free_memory_ratio = 1.0 - resource;
let compression_level = 19;
let max_mem_allowed = ((total_memory as f32) * (1.0 - min_free_memory_ratio)) as u64;
let file_split_block_size = 10 * 1024 * 1024;
let zstd_output_buffer_size = 1 * 1024 * 1024;
let max_chunks: u32 = (max_mem_allowed / file_split_block_size) as u32;
log::info!(
"[strategic_config] Detekterade {} kärnor och {} MiB minne",
max_core_allowed,
// `total_memory` is BYTES. Dividing by 1024 printed KiB under a "MiB"
// label — off by 1024 for as long as the line has existed.
total_memory / (1024 * 1024)
);
let sc = StrategicConfig {
max_core_allowed,
max_core_in_flight,
max_core_in_compress,
max_mem_allowed,
min_free_memory_ratio,
file_split_block_size,
compression_level,
max_chunks,
zstd_output_buffer_size,
};
log_strategic_conf(&sc);
sc
}
fn strategic_confi_large() -> StrategicConfig {
let mut sc = strategic_config(1.0);
// Cap at the legacy LARGE_SIZE (128 slots × 10 MB) — stored in archive metadata for compat.
sc.max_chunks = min(sc.max_chunks as u64, 128) as u32;
log::info!(
"[strategic_config large] max_core_in_flight={} max_core_in_compress={} (see log_strategic_conf: drives nothing, NOT a zstd thread count) max_chunks {} ",
sc.max_core_in_flight,
sc.max_core_in_compress,
sc.max_chunks
);
log_strategic_conf(&sc);
sc
}
fn log_strategic_conf(sc: &StrategicConfig) {
// The two labels used to read `(10%)` here and `(90%)` below — SWAPPED, and
// swapped for as long as the lines have existed. `max_core_in_flight` is
// `ceil(cores * 0.90)` (line 109) and `max_core_in_compress` is the
// remainder, so the in-flight width is the 90% and the compress reserve is
// the 10%. MEASURED on oden 2026-09-20 (32 physical cores): in_flight = 29,
// in_compress = 3, printed under labels claiming 10% and 90% respectively.
// An operator reading this log to work out why a run is not saturating the
// box was being told the pipeline runs 29 workers on "10% of the cores".
log::info!(
"[strategic_config] max_core_in_flight: {} (90% of {} cores)",
sc.max_core_in_flight,
sc.max_core_allowed
);
// NOTE, and it is a finding rather than a description: `max_core_in_compress`
// is READ BY NOTHING. `grep -rn max_core_in_compress` over this repo finds
// this log line, the struct field, its computation above, and
// `index.rs:105/124` which writes it into the archive metadata and reads it
// back out again — and no worker count, thread pool or zstd parameter
// anywhere. The "for zstd" in `strategic_confi_large`'s log line names a use
// that does not exist. It is left in place because it is PERSISTED in every
// archive written so far and `read_znippy_index` errors on its absence, so
// removing the field is a format change; but it is not a knob, and a reader
// should not go looking for what it tunes.
log::info!(
"[strategic_config] max_core_in_compress: {} (10% remainder; drives nothing — see note)",
sc.max_core_in_compress
);
log::info!(
"[strategic_config] min_free_memory_ratio: {:.0}%",
sc.min_free_memory_ratio * 100.0
);
log::info!(
"[strategic_config] compression_level: {}",
sc.compression_level
);
log::info!("[strategic_config] max_chunks: {}", sc.max_chunks);
log::info!(
"[strategic_config] zstd_output_buffer_size: {}",
sc.zstd_output_buffer_size
);
}