crates/monetize-server/tests/arrival_rate.rs
//! **"Can the estate take 10 users a day, and if that works, 1000 a day — and
//! does it spend money correctly while doing it?"**
//!
//! The owner's question, asked of the code rather than of the box. Everything
//! here is native and driven off a clock the test owns: no KVM, no wall-clock
//! sleep longer than the drive thread's own milliseconds, no network. The money
//! half of this estate is pure logic over a fake cloud, which is the only reason
//! a thousand arrivals a day can be measured at all.
//!
//! # The three limits under test, and where each one lives
//!
//! | limit | where | this estate |
//! |---|---|---|
//! | the shop's allowance | `monetize_server::oversell` — `[signups.<product>] max_sold_percent_of_capacity` | 600 % |
//! | the wallet | `monetize_server::grow::ceiling_remaining` — `[clouds.<n>] cap` | €25 (2 500 EUR-minor) |
//! | the iron | the data set the appliance serves from | 37 GiB |
//!
//! The first two are NOT the same wall and they do not move together. The
//! allowance is a percentage **of the iron**, so every growth raises it; the
//! wallet is a fixed number that only ever goes down. That is the whole shape
//! of the answer below.
//!
//! # What this file models, and the one number it invents
//!
//! A tenant "arrives" by going through the two gates a real customer goes
//! through, in the real order:
//!
//! 1. [`monetize_server::oversell::admit`] — the shop's capacity gate, the last
//! moment a refusal is free (after the webhook the card has been charged).
//! 2. `monetize::transaction::Transaction::run` — the money: reserve, ensure,
//! capture, push the entitlement. The fake cloud answers `OnHand`, which is
//! this estate's rule 1: *a purchase never buys cloud disk*; it is served
//! from the free room of the set.
//! 3. [`monetize_server::grow::batch::on_paid_order`] — the paid order queues
//! ONE evaluation for the batcher. It buys nothing itself.
//!
//! Then the batcher is ticked on the test's clock exactly as
//! `grow::batch::run` ticks it in `cli.rs` — every simulated second's worth of
//! decision, coalesced to a tick per arrival window and a tick per
//! [`TICK_EVERY_MS`] so that the 60 s watermark check and the 5 min hold retry
//! both get their turn.
//!
//! **The invented number is `writes_per_tenant_gib`** — how much of their cap a
//! tenant actually uses. Nothing in monetize can know it. It is a parameter of
//! the scenario, stated at every call site, and the two interesting values (0,
//! and a tenant who writes a tenth of what they bought) are both measured.
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use monetize::testing::{purchase, GrowScript, MockCloud, MockPay, MockProduct};
use monetize::transaction::{Outcome, Transaction};
use monetize_cloud::cap::{Ceiling, SpendCap};
use monetize_pay::Currency;
use monetize_product::{Growth, Product, Servable};
use monetize_server::config::{GrowSetConfig, GrowTrigger};
use monetize_server::grow::batch::{self, BatchConfig, GIB};
use monetize_server::grow::{ceiling_remaining, phase, GrowRuns, ProductGrow};
use monetize_server::registry::Registry;
use monetize_server::state::AppState;
// ── the estate, as the owner described it on 2026-09-19 ────────────────────
/// The appliance's data set: one member, 37 GiB. The figure the front was
/// measured at (37.8 GiB) rounded down to the whole GiB the cloud steps in.
const POOL_GIB: u64 = 37;
/// What one tenant buys — the `gunnar/10gb-1m` plan's cap.
const PLAN_GIB: u64 = 10;
/// `[signups.<product>] max_sold_percent_of_capacity` on this estate. 600 % of 37 GiB is
/// 222 GiB of promises against 37 GiB of disk.
const MAX_SOLD_PERCENT: u32 = 600;
/// `[clouds.upcloud] cap = { total_minor = 2500, currency = "Eur" }` — €25.
const CEILING_TOTAL_MINOR: u64 = 2_500;
/// UpCloud maxiops, as `monetize-cloud-impl`'s own price fixture has it:
/// 0.031 ¢/h per GB × 730 h/month = 22.63 ¢ → 23 EUR-minor per GiB-month.
const PER_GIB_MINOR: u64 = 23;
const DAY_MS: u64 = 86_400_000;
const T0: u64 = 1_900_000_000_000;
/// The batcher's own loop runs every second (`cli.rs`). A tick every five
/// simulated minutes is the coarsest cadence that still gives the 60 s
/// watermark check and the 5 min hold retry each their turn.
const TICK_EVERY_MS: u64 = 300_000;
const PHASES: [&str; 5] = ["draining", "unmounted", "growing", "mounting", "done"];
// ── the rig ────────────────────────────────────────────────────────────────
struct Estate {
state: Arc<AppState>,
clock: Arc<AtomicU64>,
product: Arc<MockProduct>,
cloud: Arc<MockCloud>,
pay: MockPay,
/// Bytes the tenants have actually written into the set.
written: Mutex<u64>,
/// The request ids of every batch that went IN FLIGHT — that is, of every
/// growth that was actually ordered. What is in here answers "who ordered
/// the disk": a purchase (`order:…`), a signup (`signup:…`), or the
/// watermark check's own trigger (`watermark:<ms>`).
ordered_by: Mutex<Vec<Vec<String>>>,
dir: PathBuf,
}
impl Estate {
/// The estate above, with the batcher's trigger chosen by the caller —
/// `Used` is the shipped default (`total − servable` from `Admin.Capacity`),
/// `Promised` counts the tenants' caps against the set.
fn up(name: &str, trigger: GrowTrigger) -> Estate {
let dir = std::env::temp_dir().join(format!("monetize-arrival-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let product = Arc::new(MockProduct::gunnar());
product.grows(PHASES.to_vec(), true);
let cloud = Arc::new(MockCloud::new());
cloud.grows(GrowScript { current_gib: POOL_GIB, per_gib_minor: PER_GIB_MINOR, ..GrowScript::default() });
let product_dyn: Arc<dyn Product> = product.clone();
let capped = SpendCap::new(cloud.clone(), Ceiling::total(CEILING_TOTAL_MINOR, Currency::Eur), dir.join("cap.json")).unwrap();
let registry = Registry::new().shared_product(product_dyn).cloud_capped(capped);
let clock = Arc::new(AtomicU64::new(T0));
let mut s = AppState::in_memory(registry);
let c = clock.clone();
s.clock = Box::new(move || c.load(Ordering::SeqCst));
s.signup_max_sold_percents = std::collections::BTreeMap::from([("gunnar".to_string(), MAX_SOLD_PERCENT)]);
let mut runs = GrowRuns::default();
runs.poll_interval = Duration::from_millis(1);
runs.unmount_ceiling = Duration::from_millis(500);
runs.appliance_ceiling = Duration::from_millis(500);
runs.batch = BatchConfig { trigger, ..BatchConfig::default() };
runs.add("gunnar", GrowSetConfig { cloud: "mockcloud".into(), server: "srv-1".into(), members: vec!["vol-1".into()] }, None, None).unwrap();
s.grow = runs;
let e = Estate {
state: Arc::new(s),
clock,
product,
cloud,
pay: MockPay::new(),
written: Mutex::new(0),
ordered_by: Mutex::new(Vec::new()),
dir,
};
e.refresh();
e
}
fn now(&self) -> u64 {
self.clock.load(Ordering::SeqCst)
}
fn at(&self, ms: u64) {
self.clock.store(ms, Ordering::SeqCst);
}
fn pg(&self) -> Arc<ProductGrow> {
self.state.grow.get("gunnar").unwrap()
}
/// The set's size right now, per the cloud that owns the volumes.
fn set_gib(&self) -> u64 {
self.cloud.grow.lock().unwrap().as_ref().unwrap().current_gib
}
/// What the ledger says has been committed to this provider — the number
/// `ceiling_remaining` subtracts from the €25.
fn spent_minor(&self) -> u64 {
self.state.ledger.total_provisioned_minor_by("mockcloud", Currency::Eur).unwrap()
}
/// What `grow::ceiling_remaining` has left of the €25 right now — the one
/// number the growth path consults before it spends.
fn remaining_minor(&self) -> u64 {
let cloud = self.state.registry.find_cloud("mockcloud").unwrap();
ceiling_remaining(&self.state, cloud).unwrap()
}
/// The appliance's capacity answer, re-derived from the set's real size and
/// what the tenants have written. Both the poller's cache and the plugin
/// itself, because the batcher reads the cache and the planner reads the
/// plugin.
fn refresh(&self) {
let total = self.set_gib() * GIB;
let used = (*self.written.lock().unwrap()).min(total);
let growth = Growth::AtRuntime("this appliance grows at runtime".into());
self.product.can_serve_growing(total - used, total, growth.clone());
let s = Servable::Measured { servable_bytes: total - used, total_bytes: total, growth, ceilings: Default::default() };
self.state.health.set_servable("gunnar", s, self.now());
}
/// `tenant` writes `bytes` into the set, and — when `polled` — the usage
/// poller reports it, which is what a working deployment does.
///
/// The flag is not decoration. `monetize::fleet::FleetTotal::capacity()` is
/// `used + servable`, where `used` is the USAGE STORE's sum: it throws away
/// the `total_bytes` the appliance measured and standing right beside it.
/// So a silent poller does not merely make the fleet page vague — it makes
/// the shop's own capacity figure SHRINK by exactly what the tenants have
/// written. See `a_silent_usage_poller_shrinks_the_shop_by_what_tenants_wrote`.
///
/// A tenant cannot write more than is free, whatever their cap says: the
/// bytes are capped at the set's free room, because that is what a full
/// disk does to a push and it is the very failure the oversell gate exists
/// to keep customers away from.
fn write(&self, tenant: &str, bytes: u64, polled: bool) -> u64 {
let free = (self.set_gib() * GIB).saturating_sub(*self.written.lock().unwrap());
let bytes = bytes.min(free);
*self.written.lock().unwrap() += bytes;
if polled && bytes > 0 {
let held = self.state.usage.get("gunnar", tenant).ok().flatten().and_then(|u| u.meters.get("pack_bytes").copied()).unwrap_or(0);
let u = monetize_product::Usage { meters: [("pack_bytes".to_string(), held + bytes)].into(), measured_at_unix_ms: self.now() };
self.state.usage.put("gunnar", tenant, &u).unwrap();
}
self.refresh();
bytes
}
fn tx(&self) -> Transaction<'_> {
Transaction {
pay: monetize_pay::Vendor::of(&self.pay),
cloud: &*self.cloud,
product: self.product.served(),
ledger: &self.state.ledger,
signer: self.state.signer_for(self.product.id()),
now_unix_ms: self.now(),
}
}
/// One tick, and if it put a growth in flight, let the drive park and tick
/// again so the batch is settled before the next arrival. The drive's own
/// waits are milliseconds; the WINDOW and the INTERVAL are the test's clock.
fn tick(&self) -> Vec<String> {
// `MockProduct`'s grow script answers one phase per `status()` call and
// then repeats the LAST one for ever, so a second growth would be told
// `done` before the cloud was ever asked. Re-arming between growths is
// a property of the double, not of the estate: nothing is in flight
// here, because `tick` never returns until the drive has parked.
self.product.grows(PHASES.to_vec(), true);
let mut lines = batch::tick(&self.state);
if let Some(f) = self.pg().batch.snapshot().in_flight {
self.ordered_by.lock().unwrap().push(f.request_ids.clone());
}
if self.pg().batch.snapshot().in_flight.is_some() {
let start = Instant::now();
while self.pg().is_running() || !self.pg().snapshot().is_terminal() {
assert!(start.elapsed() < Duration::from_secs(20), "the drive did not park: {:?}", self.pg().snapshot());
std::thread::sleep(Duration::from_millis(1));
}
self.refresh(); // the set is bigger now; the appliance says so
lines.extend(batch::tick(&self.state));
}
lines
}
fn tick_at(&self, ms: u64) -> Vec<String> {
self.at(ms);
self.tick()
}
}
impl Drop for Estate {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
// ── what one arrival comes to ──────────────────────────────────────────────
#[derive(Debug, Default)]
struct Report {
/// Customers who tried.
arrivals: u64,
/// Customers who were sold an account.
sold: u64,
sold_gib: u64,
/// SEK-minor captured from those sales.
revenue_minor: u64,
/// **Sales made while the promise was already past the LAST AFFORDABLE
/// DISK** — the caps promised exceeded the iron plus everything the wallet
/// could still buy. STINK S2's single number.
sold_past_the_wallet: u64,
/// The first arrival the SHOP refused, and why (1-based; 0 = never).
shop_refused_at: u64,
shop_refused_why: String,
refused: u64,
/// Growths that reached the cloud, and the sizes they resized to.
resizes: Vec<u64>,
/// EUR-minor committed to the provider.
spent_minor: u64,
/// The first arrival at which a growth was REFUSED OR CLAMPED by the spend
/// ceiling, and the sentence it was refused with (1-based; 0 = never).
wallet_bit_at: u64,
wallet_bit_why: String,
set_gib_end: u64,
promised_gib_end: u64,
/// The request ids of every growth that was actually ordered, in order.
ordered_by: Vec<Vec<String>>,
}
impl Report {
/// The `source:` half of every id that ordered a growth — `order`,
/// `signup` or `watermark`.
fn order_sources(&self) -> Vec<String> {
let mut out: Vec<String> = self.ordered_by.iter().flatten().map(|id| id.split(':').next().unwrap_or("?").to_owned()).collect();
out.sort();
out.dedup();
out
}
}
impl Report {
fn line(&self) -> String {
format!(
"{} arrivals: {} sold ({} GiB promised, {} SEK-minor taken), {} refused; \
set {POOL_GIB} → {} GiB in {} growths costing {} of {CEILING_TOTAL_MINOR} EUR-minor; \
wallet closed at customer {}, shop closed at customer {}",
self.arrivals,
self.sold,
self.sold_gib,
self.revenue_minor,
self.refused,
self.set_gib_end,
self.resizes.len(),
self.spent_minor,
self.wallet_bit_at,
self.shop_refused_at
)
}
}
/// One run of [`simulate`], spelled out so no call site is a row of bare
/// numbers.
#[derive(Clone, Copy)]
struct Scenario {
trigger: GrowTrigger,
per_day: u64,
days: u64,
/// How much of their 10 GiB cap each tenant actually writes. **The one
/// number this file invents** — nothing in monetize can know it.
writes_gib: u64,
/// Whether the usage poller reports what they wrote.
polled: bool,
}
/// **The simulation.** `per_day` tenants arrive evenly spaced over `days`, each
/// going through the shop's gate, the money, and the batcher, in that order.
fn simulate(name: &str, s: Scenario) -> Report {
let Scenario { trigger, per_day, days, writes_gib, polled } = s;
let e = Estate::up(name, trigger);
let gap = DAY_MS / per_day;
let mut r = Report::default();
let mut next_tick = T0;
for i in 0..per_day * days {
let at = T0 + i * gap;
// The batcher's own loop keeps running between arrivals.
while next_tick < at {
e.tick_at(next_tick);
next_tick += TICK_EVERY_MS;
}
e.at(at);
r.arrivals += 1;
let tenant = format!("t{i:05}");
// ── 1. the shop's gate, before a card is ever touched ──
let hold = match monetize_server::oversell::admit(&e.state, e.product.served(), PLAN_GIB * GIB) {
Ok(h) => h,
Err(why) => {
r.refused += 1;
if r.shop_refused_at == 0 {
r.shop_refused_at = r.arrivals;
r.shop_refused_why = why;
}
continue;
}
};
// ── 2. the money ──
let mut p = purchase(&tenant);
p.starts_unix_ms = at;
let out = e.tx().run(&p).expect("the mock vendor and the pool always answer");
let Outcome::Paid { reference, .. } = out else { panic!("{out:?}") };
drop(hold); // the caps are counted from the ledger from here on
r.sold += 1;
r.sold_gib += PLAN_GIB;
r.revenue_minor += p.price.minor;
// Was this sale already past the largest set the wallet can ever pay
// for? Asked AFTER the sale, of the promise it just made, against the
// iron as the product reports it now.
if monetize_server::oversell::past_the_wallet(
&e.state,
"gunnar",
batch::promised_bytes(&e.state, "gunnar").unwrap_or(0),
e.set_gib() * GIB,
)
.is_some()
{
r.sold_past_the_wallet += 1;
}
// ── 3. the paid order queues ONE evaluation; it buys nothing ──
batch::on_paid_order(&e.state, e.product.served(), &reference);
if writes_gib > 0 {
e.write(&tenant, writes_gib * GIB, polled);
}
// The window closes 10 s later.
let lines = e.tick_at(at + 10_000);
r.spent_minor = e.spent_minor();
if r.wallet_bit_at == 0 {
if let Some(l) = lines.iter().find(|l| l.contains("spend ceiling")) {
r.wallet_bit_at = r.arrivals;
r.wallet_bit_why = (*l).clone();
}
}
next_tick = at + 10_000 + TICK_EVERY_MS;
}
// Let the queue drain: a day's worth of ticks with nobody arriving.
let end = T0 + per_day * days * gap;
let mut t = end;
while t < end + DAY_MS {
let lines = e.tick_at(t);
if r.wallet_bit_at == 0 {
if let Some(l) = lines.iter().find(|l| l.contains("spend ceiling")) {
r.wallet_bit_at = r.arrivals;
r.wallet_bit_why = (*l).clone();
}
}
t += TICK_EVERY_MS;
}
r.resizes = e.cloud.grow_calls().into_iter().filter(|(v, _)| v == "grow").map(|(_, t)| t).collect();
r.spent_minor = e.spent_minor();
r.set_gib_end = e.set_gib();
r.promised_gib_end = batch::promised_bytes(&e.state, "gunnar").unwrap() / GIB;
r.ordered_by = e.ordered_by.lock().unwrap().clone();
eprintln!("ARRIVAL-RATE {name} [{:?}] {}", trigger, r.line());
eprintln!("ARRIVAL-RATE {name} ordered by: {:?}", r.ordered_by);
r
}
// ── A. the owner's question, at 10 a day and at 1 000 a day ────────────────
/// ★ **THE DEFAULT TRIGGER MAKES THE ARRIVAL RATE IRRELEVANT TO SPEND.**
///
/// `grow_trigger = "used"` is what ships, and "used" is `total − servable` from
/// `Admin.Capacity`: it moves when a tenant WRITES, never when a tenant BUYS.
/// So at ten a day and at a thousand a day, with tenants who have not yet
/// written a byte, the estate buys exactly the same amount of iron: **none**.
/// What stops it is the SHOP, at the same customer number both times, because
/// `max_sold_percent_of_capacity` is a percentage of a set that never grew.
///
/// The answer to "can it take 10 a day" is therefore not a rate at all. It is
/// a TOTAL: twenty-two, and the twenty-third is refused whether they are the
/// twenty-third of a slow week or of a busy half-hour.
#[test]
fn at_the_default_trigger_the_arrival_rate_changes_nothing_the_shop_stops_at_the_same_customer() {
let base = Scenario { trigger: GrowTrigger::Used, per_day: 10, days: 10, writes_gib: 0, polled: true };
let ten = simulate("used-10", base);
let thousand = simulate("used-1000", Scenario { per_day: 1_000, days: 1, ..base });
for r in [&ten, &thousand] {
// 600 % of 37 GiB is 222 GiB: twenty-two 10 GiB plans, and the
// twenty-third is refused BEFORE the card.
assert_eq!(r.sold, 22, "{}", r.line());
assert_eq!(r.sold_gib, 220);
assert_eq!(r.shop_refused_at, 23, "the 23rd customer is the first one refused: {}", r.line());
assert!(
r.shop_refused_why.contains("max_sold_percent_of_capacity=600") && r.shop_refused_why.contains("222.0 GiB"),
"the refusal names the knob and the allowance: {}",
r.shop_refused_why
);
// NOT ONE EURO. No growth was even evaluated as needing one.
assert!(r.resizes.is_empty(), "iron was bought for tenants who have written nothing: {:?}", r.resizes);
assert_eq!(r.spent_minor, 0);
assert_eq!(r.set_gib_end, POOL_GIB);
assert_eq!(r.wallet_bit_at, 0, "the wallet was never the thing that bit");
}
// The rate changed by 100×; every money-shaped number is identical.
assert_eq!((ten.sold, ten.sold_gib, ten.spent_minor, ten.revenue_minor), (thousand.sold, thousand.sold_gib, thousand.spent_minor, thousand.revenue_minor));
// 100× the arrivals is 100× the REFUSALS, and that is the only difference.
assert_eq!((ten.arrivals, ten.refused), (100, 78));
assert_eq!((thousand.arrivals, thousand.refused), (1_000, 978));
}
/// ★ **THE SHOP OUTRUNS THE WALLET** — the case the owner asked to have
/// measured rather than assumed.
///
/// Now the tenants USE what they bought (8 GiB of their 10), the set fills, the
/// batcher grows it — and every growth raises the shop's allowance, because the
/// allowance is 600 % **of the iron**. The wallet does not rise: €25 is €25.
/// So the two limits move apart, and the question is how far the shop runs
/// past the last disk the estate could afford.
///
/// Both rates are measured because the minimum interval between growths is an
/// HOUR: at ten a day no arrival ever waits for it, at a thousand a day nearly
/// every one does, and only the ≥ 90 % emergency path gets through.
#[test]
fn tenants_who_write_make_the_shop_outrun_the_wallet_at_both_rates() {
let base = Scenario { trigger: GrowTrigger::Used, per_day: 10, days: 10, writes_gib: 8, polled: true };
let ten = simulate("write-10", base);
let thousand = simulate("write-1000", Scenario { per_day: 1_000, days: 1, ..base });
for r in [&ten, &thousand] {
// ── WHAT WAS SPENT, AND WHICH LIMIT STOPPED IT ──
// €24.84 of the €25: the last growth was CLAMPED by the ceiling, not
// refused by the credits and not refused by a quota. The ceiling is the
// only limit that spoke, and it is the FIRST one that spoke.
assert_eq!(r.spent_minor, 2_484, "{}", r.line());
assert!(CEILING_TOTAL_MINOR - r.spent_minor < PER_GIB_MINOR, "there was room for another whole GiB: {}", r.line());
assert!(r.wallet_bit_at > 0 && r.wallet_bit_at <= 13, "the ceiling spoke at customer {}: {}", r.wallet_bit_at, r.line());
assert!(r.wallet_bit_why.contains("spend ceiling") && r.wallet_bit_why.contains(" EUR left") && !r.wallet_bit_why.contains("minor units"), "{}", r.wallet_bit_why);
assert_eq!(r.set_gib_end, 145, "€25 at {PER_GIB_MINOR} EUR-minor a GiB-month buys 108 GiB on top of the {POOL_GIB}: {}", r.line());
// ── WHAT WAS SOLD ──
assert_eq!((r.sold, r.sold_gib), (87, 870), "{}", r.line());
assert_eq!(r.shop_refused_at, 88, "{}", r.line());
assert!(r.shop_refused_why.contains("max_sold_percent_of_capacity=600"), "{}", r.shop_refused_why);
// ── THE ANSWER: THE SHOP KEEPS SELLING AFTER IT CANNOT BUY THE DISK ──
// The wallet closes at customer ~12 and the shop at customer 88, so
// about SEVENTY-FIVE accounts are sold against iron that will never be
// bought. The allowance rose with each growth; the €25 did not.
assert!(r.sold - r.wallet_bit_at >= 74, "only {} accounts were sold after the wallet closed: {}", r.sold - r.wallet_bit_at, r.line());
assert_eq!(r.promised_gib_end, 870);
assert_eq!(r.promised_gib_end * 100 / r.set_gib_end, 600, "600 % of the LAST set size, which is the allowance the growths bought");
// In THIS simulation the write lands in the same instant as the
// purchase, so the purchase's own batch is the one that sees the
// crossing. On the live front the two are hours apart and the
// watermark check is what orders the disk — see
// `growth_is_ordered_by_the_watermark_not_by_the_purchase`.
assert_eq!(r.order_sources(), vec!["order".to_string()], "{:?}", r.ordered_by);
}
// The rate changes how many drain/unmount cycles it takes — 5 at ten a day,
// 4 at a thousand, because the 1 h minimum interval coalesces more of them —
// and changes NOTHING about the money.
assert_ne!(ten.resizes.len(), thousand.resizes.len(), "the rate is supposed to change the number of growths");
assert_eq!((ten.sold, ten.spent_minor, ten.set_gib_end), (thousand.sold, thousand.spent_minor, thousand.set_gib_end));
eprintln!("MEASURED used 10/day : {}", ten.line());
eprintln!("MEASURED used 1000/day : {}", thousand.line());
}
/// ★ **THE SAME QUESTION WITH `grow_trigger = "promised"`** — the setting that
/// makes a SALE, not a write, move the fill. This is the arrival-rate question
/// in its purest form: the tenants write nothing at all, and the estate still
/// spends, because what it counts is what it has promised.
#[test]
fn at_the_promised_trigger_the_sale_itself_buys_iron_and_the_ceiling_is_what_stops_it() {
let base = Scenario { trigger: GrowTrigger::Promised, per_day: 10, days: 10, writes_gib: 0, polled: true };
let ten = simulate("promised-10", base);
let thousand = simulate("promised-1000", Scenario { per_day: 1_000, days: 1, ..base });
for r in [&ten, &thousand] {
// The estate comes to rest in EXACTLY the same place as when the
// tenants were the ones filling the disk — 87 accounts, 145 GiB of
// iron, €24.84 spent — which is the point: the two triggers differ in
// WHEN the estate spends, not in HOW MUCH it may.
assert_eq!((r.sold, r.sold_gib, r.set_gib_end, r.spent_minor), (87, 870, 145, 2_484), "{}", r.line());
assert_eq!(r.shop_refused_at, 88, "{}", r.line());
assert!(r.wallet_bit_at > 0 && r.wallet_bit_at <= 13, "{}", r.line());
assert!(r.sold - r.wallet_bit_at >= 74, "{}", r.line());
// The shop sold 870 GiB against 145 GiB of disk with not one byte
// written: the promises alone are six times the iron.
assert_eq!(r.promised_gib_end, 870);
}
eprintln!("MEASURED promised 10/day : {}", ten.line());
eprintln!("MEASURED promised 1000/day : {}", thousand.line());
}
// ── who orders the disk ────────────────────────────────────────────────────
/// ★ **THE GROWTH IS ORDERED BY THE WATERMARK, NOT BY THE PURCHASE** — the
/// sequence measured on the front on 2026-09-19.
///
/// A paid order queues `order:<reference>` and the window closes ten seconds
/// later, which is long before the customer has pushed anything. At the default
/// trigger the fill has not moved, so that batch is `satisfied` — *served from
/// the free room, no resize* — and the purchase is done with.
///
/// What actually buys the disk is the batcher's OWN trigger: `grow::batch::run`
/// ticks every second (`cli.rs`), every sixtieth of those ticks re-reads the
/// poller's fill while the queue is quiet, and when that reading crosses the
/// high-water mark it queues `watermark:<ms>`. The batch that grows therefore
/// carries a timestamp, never a purchase id — which is why looking for the
/// customer who "caused" a growth in the batch book finds nobody.
#[test]
fn growth_is_ordered_by_the_watermark_not_by_the_purchase() {
let e = Estate::up("watermark", GrowTrigger::Used);
// ── the sale. The box is empty; the customer has pushed nothing. ──
let at = T0 + 60_000;
e.at(at);
let out = e.tx().run(&purchase("alice")).expect("the purchase goes through");
let Outcome::Paid { reference, .. } = out else { panic!("{out:?}") };
batch::on_paid_order(&e.state, e.product.served(), &reference);
let order_id = format!("order:{reference}");
assert!(e.pg().batch.snapshot().requests.iter().any(|r| r.id == order_id), "the paid order queued an evaluation");
// Ten seconds later the window closes on a set nobody has filled.
e.tick_at(at + 10_000);
let book = e.pg().batch.snapshot();
let q = book.requests.iter().find(|r| r.id == order_id).expect("the request is still on the book");
assert_eq!(q.state, "satisfied", "{q:?}");
assert!(q.note.contains("< high-water 80%"), "the purchase's own batch bought nothing: {}", q.note);
assert!(e.cloud.grow_calls().is_empty(), "a purchase bought cloud disk");
// ── hours later, the customer actually uses the account ──
let later = at + 6 * 3_600_000;
e.at(later);
e.write("alice", 31 * GIB, true); // 31 of 37 GiB: 83.7 %
// The queue is quiet, so the tick's own watermark check is the only thing
// that can speak — and it does, a minute after the last one.
let lines = e.tick_at(later + 60_000);
assert!(lines.iter().any(|l| l.contains("watermark:")), "{lines:?}");
// ── STINK S6: THE LINE SAYS HOW OLD THE NUMBER IT DECIDED ON WAS ──────
// Three rates interact — this loop at 1 s, the watermark re-read at 60 s,
// the fill itself refreshed by the poller at 5 min — and their product,
// "the mark can be five minutes stale", was written down nowhere. Here the
// write landed exactly a minute before the check, and the line says so;
// on the live front the same sentence will read up to 300 s.
let queued = lines.iter().find(|l| l.contains("watermark:")).expect("the watermark queued a trigger");
assert!(queued.contains("measured 60 s ago"), "the queue line must say how old the reading was: {queued}");
let lines = e.tick_at(later + 70_000);
assert!(!e.cloud.grow_calls().is_empty(), "nothing grew: {lines:?}");
// **THE ASSERTION**: the batch that ordered the disk carries a watermark
// id and no purchase at all.
let ordered = e.ordered_by.lock().unwrap().clone();
assert_eq!(ordered.len(), 1, "exactly one growth: {ordered:?}");
assert_eq!(ordered[0].len(), 1, "one trigger in it: {ordered:?}");
assert!(ordered[0][0].starts_with("watermark:"), "the growth was ordered by {:?}", ordered[0][0]);
assert!(!ordered[0].contains(&order_id), "the purchase is in the batch that bought the disk: {ordered:?}");
}
// ── C. the gap, named ──────────────────────────────────────────────────────
/// ★ **KNOWN GAP: NOTHING IN THE GROWTH PATH ASKS WHAT HAS BEEN COLLECTED.**
///
/// This test does not fail and it is not meant to. It DOCUMENTS what is
/// missing, so that the day somebody adds a revenue-linked ceiling it has a red
/// half waiting for it.
///
/// `grow::ceiling_remaining` reads exactly two numbers: the operator's
/// `[clouds.<n>] cap`, and `Ledger::total_provisioned_minor_by` — what has been
/// SPENT. There is no third. The ledger has no "collected" fold at all: it can
/// say what a provider was ever paid and cannot say what the customers ever
/// paid US, so the growth path could not consult revenue even if it wanted to.
///
/// The proof is an equality, not a prose claim. Two estates are driven to the
/// same fill: one with eighty-seven paying customers behind it, one with **not
/// one krona ever collected** — no reserve, no capture, no attempt, an empty
/// ledger but for the growths themselves. They spend the same €24.84, grow to
/// the same 145 GiB, and `ceiling_remaining` answers the same number at every
/// step. Revenue is not an input.
///
/// **This is a statement of the gap, not a design.** What a revenue-linked
/// ceiling should be — a multiple of the monthly take, a fixed runway, a
/// per-product budget — is the owner's decision and is deliberately not
/// invented here.
#[test]
fn known_gap_the_growth_path_never_asks_what_was_collected() {
/// Fill the set to the high-water mark and let the batcher spend, over and
/// over, until the ceiling refuses. Answers (spent, set_gib, remaining).
fn spend_it_all(e: &Estate) -> (u64, u64, u64) {
let mut t = T0;
for _ in 0..40 {
// Whoever is on this box has filled it. WHO is not asked anywhere.
let total = e.set_gib() * GIB;
*e.written.lock().unwrap() = total * 9 / 10;
e.refresh();
t += 3_600_000 + 120_000; // past the 1 h minimum interval
e.tick_at(t);
e.tick_at(t + 60_000);
e.tick_at(t + 70_000);
}
(e.spent_minor(), e.set_gib(), e.remaining_minor())
}
// (a) an estate with eighty-seven paying customers behind it.
let paid = Estate::up("gap-paid", GrowTrigger::Used);
let mut revenue = 0u64;
for i in 0..87u64 {
paid.at(T0 + i * 1_000);
let mut p = purchase(&format!("t{i:05}"));
p.starts_unix_ms = T0 + i * 1_000;
assert!(matches!(paid.tx().run(&p).unwrap(), Outcome::Paid { .. }));
revenue += p.price.minor;
}
assert_eq!(revenue, 1_914_000, "87 × 220.00 SEK");
let paid_end = spend_it_all(&paid);
// (b) an estate that has never taken a single payment.
let free = Estate::up("gap-free", GrowTrigger::Used);
assert!(free.state.ledger.recent(1_000).unwrap().is_empty(), "the ledger must start with nothing in it");
let free_end = spend_it_all(&free);
// **THE GAP, AS AN EQUALITY.** €24.84 spent either way.
assert_eq!(paid_end, free_end, "revenue changed the spend; if this ever fails the gap has been closed");
assert_eq!(free_end.0, 2_484, "the estate with no revenue spent the whole ceiling");
assert_eq!(free_end.2, CEILING_TOTAL_MINOR - 2_484, "and `ceiling_remaining` counted only the spend");
// And the ledger of the estate that took no money holds growths and
// nothing else: there is no `Captured` row to compare a growth against,
// and no fold that would find one.
let kinds: Vec<String> = free.state.ledger.recent(1_000).unwrap().into_iter().map(|(_, row)| row.step.name().to_owned()).collect();
assert!(!kinds.is_empty() && kinds.iter().all(|k| k == "Grown"), "only growths: {kinds:?}");
}
// ── a bug the arrival runs found ───────────────────────────────────────────
/// ★ **A SILENT USAGE POLLER SHRINKS THE SHOP BY WHATEVER THE TENANTS WROTE.**
///
/// `monetize::fleet::FleetTotal::capacity()` is `used + servable`, where `used`
/// is the sum over the USAGE STORE. The appliance's own `total_bytes` is
/// carried on `Servable::Measured` right beside `servable_bytes` and is not
/// read. So capacity is reconstructed from two numbers instead of taken from
/// the one that is already the answer — and the reconstruction is only correct
/// while the usage store is complete and current.
///
/// It is not, twice over: `Admin.Usage` is polled on a timer, so it is always
/// at least one tick stale, and a product that cannot report per-tenant usage
/// reports none at all. In both cases capacity reads as SERVABLE alone, which
/// falls by exactly what the tenants have written — and the oversell gate is a
/// percentage of it.
///
/// Measured here: the same 37 GiB set, the same 600 %, the same tenants each
/// writing 8 GiB of their 10 GiB cap, the same €25 spent down to €24.84 and
/// the same 145 GiB of iron at the end of it. With the poller reporting, the
/// shop sells **87** accounts. With it silent, **it sells 15** and refuses the
/// sixteenth — on iron that is identical, for the same money.
///
/// The refusal is the safe direction to be wrong in, which is why this has
/// never been an outage. But it is a shop that closes itself on a figure that
/// is not true: the estate bought and paid for 145 GiB and is selling as
/// though it had bought a fifth of that, and the sentence it prints names an
/// allowance derived from a measurement that did not happen.
#[test]
fn a_silent_usage_poller_shrinks_the_shop_by_what_tenants_wrote() {
let base = Scenario { trigger: GrowTrigger::Used, per_day: 10, days: 10, writes_gib: 8, polled: true };
let silent = simulate("silent-poller", Scenario { polled: false, ..base });
let polled = simulate("polled-poller", base);
// THE SAME IRON, THE SAME BILL — five and a half times fewer customers.
assert_eq!((silent.set_gib_end, silent.spent_minor), (polled.set_gib_end, polled.spent_minor), "the estates must differ only in what was REPORTED");
assert_eq!((silent.sold, polled.sold), (15, 87), "silent {} / polled {}", silent.line(), polled.line());
assert_eq!(silent.shop_refused_at, 16, "{}", silent.line());
assert!(silent.shop_refused_why.contains("max_sold_percent_of_capacity=600"), "{}", silent.shop_refused_why);
// And the sums it refused on: 150 GiB promised against 145 GiB of disk is
// 103 % — the operator wrote 600 % down, and the gate closed the shop at
// one hundred and three, because the capacity it compared against was
// `servable` alone and the tenants had written nearly all of it.
assert_eq!(silent.promised_gib_end * 100 / silent.set_gib_end, 103, "{} GiB promised against {} GiB of disk", silent.promised_gib_end, silent.set_gib_end);
assert_eq!(polled.promised_gib_end * 100 / polled.set_gib_end, 600, "the same estate, reported, sells to the number that was written down");
// ── STINK S5: THE REFUSAL NOW SAYS WHICH THING TO GO AND FIX ──────────
// ── STINK S5: THE REFUSAL NOW SAYS WHICH THING TO GO AND FIX ──────────
// The figure is the honest one available, and it is a FLOOR: fifteen
// tenants hold a cap and no reading arrived for any of them. Before this
// the operator was handed an allowance, a percentage and a disk, with
// nothing to say the disk figure came out of readings that never came.
assert!(
silent.shop_refused_why.contains("MEASURED FROM AN INCOMPLETE FOLD") && silent.shop_refused_why.contains("no usage reading"),
"the refusal must name the fold it could not complete: {}",
silent.shop_refused_why
);
// **The COUNT is the signal, and it is measured, not assumed.** With the
// poller silent the fold can see NONE of the fifteen tenants it sold to.
// With it working the same sentence names 68 of 87 — the tenants who were
// sold an account after the set was full and so have written nothing to
// report. A tenant who holds nothing having no reading is ordinary; every
// tenant including the ones that filled the disk having none is a dead
// poller, and the difference is visible in the number.
assert!(silent.shop_refused_why.contains("15 tenant(s)"), "the silent estate cannot see ANY of its 15: {}", silent.shop_refused_why);
assert!(polled.shop_refused_why.contains("68 tenant(s)"), "the polled estate cannot see the 68 who wrote nothing: {}", polled.shop_refused_why);
assert_eq!(silent.sold, 15, "so the count above is EVERY tenant this estate sold");
assert!(polled.sold > 68, "and 68 is fewer than the {} the working estate sold: 19 of them have readings", polled.sold);
eprintln!("MEASURED poller silent : {}", silent.line());
eprintln!("MEASURED poller working: {}", polled.line());
}
// ── the money side, when the answers stop coming ───────────────────────────
//
// The vendor's own outages are already held by name and are not repeated here:
// `transaction::tests::refuse_twin_reserve_unavailable_writes_nothing` (an
// unreachable vendor writes not one row), `declined_at_reserve_locks_nothing_
// and_is_recorded` (a decline is an outcome and a row), `rerun_after_crash_
// between_capture_and_push_does_not_capture_twice`, `reconcile_finishes_
// orphans_from_vendor_status`, and `reconcile::tests::a_paid_session_unknown_
// to_monetize_becomes_a_pending_signup_and_is_settled_when_the_sink_answers`
// — the 2026-09-14 incident itself. What follows is what those do not reach.
/// ★ **KNOWN GAP: `can_absorb` CANNOT SAY "I COULD NOT BE ASKED".**
///
/// Every other verb a product answers is a `ProductError`, which has a
/// `Refused` arm and an `Unavailable` arm precisely so a caller can tell a NO
/// from a SILENCE. `Product::can_absorb` is `Result<Absorb, String>` — one
/// error channel, a bare sentence — so a full disk and an unreachable
/// `Admin.Capacity` arrive at the caller as the same value of the same type.
///
/// The consequences are not currently a customer-facing bug, and this test
/// says exactly why: both end as `Refusal::CannotAbsorb`, both are rendered to
/// the customer as one composed capacity sentence — *nothing has been charged,
/// and making room for it is ours to do* — which is the right sentence for
/// both. (It is no longer *"try again later"*: see
/// `monetize::transaction::words`.)
/// The difference survives only inside the product's own string, in the
/// operator's log, where nothing can branch on it.
///
/// That is the gap: an operator paging on "the appliance is unreachable" and an
/// operator paging on "the appliance is full" would do different things, and
/// the type hands them the same value. Named here; not closed here, because
/// widening that signature is a change to the plugin contract and the owner's
/// to make.
#[test]
fn known_gap_a_capacity_question_nobody_could_ask_is_the_same_type_as_a_full_disk() {
use monetize::order::Order;
use monetize::transaction::{words, Refusal};
use monetize_product::TenantId;
let order = |tenant: &str| Order { product: "gunnar".into(), tenant: TenantId(tenant.into()), caps: [("pack_bytes".to_string(), 10u64 * GIB)].into() };
// (a) the box is FULL — a real answer to a question that was asked.
let full = Estate::up("absorb-full", GrowTrigger::Used);
full.product.cannot_absorb("gunnar: the data set is full; an operator must extend the filesystem before another 10 GiB tenant");
let a = full.tx().quote(&order("alice"), &monetize::testing::terms(), T0, None).expect("a quote is answered").serve.expect_err("a full box refuses");
// (b) the box COULD NOT BE ASKED — no answer at all.
let down = Estate::up("absorb-down", GrowTrigger::Used);
down.product.cannot_absorb("gunnar: Admin.Capacity is unreachable: transport error: connection refused");
let b = down.tx().quote(&order("alice"), &monetize::testing::terms(), T0, None).expect("a quote is answered").serve.expect_err("an unreachable box also refuses");
// THE SAME VARIANT of the same refusal, from two facts about the world
// that are not the same fact.
assert!(matches!(a, Refusal::CannotAbsorb { .. }), "{a:?}");
assert!(matches!(b, Refusal::CannotAbsorb { .. }), "{b:?}");
assert_eq!(std::mem::discriminant(&a), std::mem::discriminant(&b));
// And the same sentence to the customer, which IS right — nothing has been
// charged either way and neither is theirs to fix.
assert_eq!(a.customer_words(), b.customer_words(), "one sentence for both, and no machine in either");
for said in [a.customer_words(), b.customer_words()] {
assert!(said.contains(words::NOTHING_CHARGED), "{said}");
assert!(!said.to_lowercase().contains("maintenance"), "neither of these is maintenance: {said}");
}
// The difference exists ONLY inside the product's own string. Nothing in
// monetize branches on it, and nothing can, because it is prose.
assert!(a.to_string().contains("the data set is full"), "{a}");
assert!(b.to_string().contains("unreachable"), "{b}");
// The ORDER that follows the quote refuses the same way for both, and the
// refusal is the last thing that happens: no vendor, no iron, no grant.
for e in [&full, &down] {
let err = e.tx().order(&order("alice"), &monetize::testing::terms(), T0, None).expect_err("the order is refused too");
assert!(matches!(err, monetize::transaction::TxError::Refused(Refusal::CannotAbsorb { .. })), "{err:?}");
}
// Neither took money and neither promised room.
for e in [&full, &down] {
assert!(e.state.ledger.recent(100).unwrap().is_empty());
assert_eq!(e.cloud.ensures(), 0);
assert!(e.product.pushed().is_empty());
}
}
/// ★ **THE SPEND CAP HOLDS UNDER CONCURRENCY — it did not until 2026-09-19.**
///
/// `SpendCap::ensure` takes its lock three times and holds it across none of
/// the work:
///
/// ```text
/// admit(resource) lock → read the totals → UNLOCK (nothing reserved)
/// inner.ensure(…) the provider call, no lock held
/// account(outcome) lock → add what was bought → unlock
/// ```
///
/// Nothing is reserved between the check and the accounting, so every call
/// that is inside `inner.ensure` when another one checks is invisible to it.
/// `grow_set` does reserve (`Spent::reserved`, and
/// `cap::tests::an_outage_keeps_the_reservation_across_a_reload_and_the_resume_
/// settles_it` holds it) — `ensure` does not.
///
/// Measured here, deterministically rather than by racing: four buyers are held
/// inside the provider call on a barrier, so all four have passed the check
/// before any of them is accounted for. A ceiling with room for TWO disks buys
/// FOUR — **twice the ceiling, and the refusal an operator wrote the number
/// for never happens.** Run sequentially, the same ceiling refuses the third,
/// which is the control.
///
/// This is a defect and not a gap: the guard is named, it is configured, and it
/// does not hold under concurrency. What it needs is for the admitted price to
/// be RESERVED under the same lock that checked it and released when the
/// provider answers — exactly what the growth path already does — and that is a
/// change to `monetize-cloud`, not to a test.
#[test]
fn concurrent_buyers_cannot_spend_past_the_spend_cap_because_admit_reserves() {
use monetize::testing::Answer;
use monetize_cloud::{CloudError, Ensured, Inventory, Price, Resource, ResourceId, ResourceStatus, ServerId, Zone};
/// [`MockCloud`], with every buyer held inside `ensure` until `n` of them
/// are there. The delay is the only thing this adds; it makes the window
/// that already exists wide enough to measure without making it a race.
/// **A gate that releases when every buyer has SETTLED**, which is not the
/// same as when `BUYERS` of them arrive.
///
/// It was a `Barrier::new(BUYERS)` while the defect was open, because all
/// four buyers reached the provider. With the cap holding its money, two of
/// them are refused at admission and NEVER reach the till — a barrier
/// waiting for four would then wait for ever, which is exactly what
/// happened when the fix first landed.
///
/// So a buyer either `arrive`s at the till or `gave_up` before it, and the
/// gate opens when those two account for everybody. Deterministic, no
/// sleeps, and it does not presume HOW MANY the ceiling will admit — which
/// is the number under test.
struct Gate {
/// `(at the till, still outstanding)`.
state: Mutex<(usize, usize)>,
cv: std::sync::Condvar,
}
impl Gate {
fn new(buyers: usize) -> Gate {
Gate { state: Mutex::new((0, buyers)), cv: std::sync::Condvar::new() }
}
fn arrive(&self) {
let mut g = self.state.lock().unwrap();
g.0 += 1;
self.cv.notify_all();
while g.0 < g.1 {
g = self.cv.wait(g).unwrap();
}
}
fn gave_up(&self) {
let mut g = self.state.lock().unwrap();
g.1 -= 1;
self.cv.notify_all();
}
}
struct HeldAtTheTill {
inner: Arc<MockCloud>,
gate: Arc<Gate>,
}
#[allow(dead_code)]
impl HeldAtTheTill {
pub fn provider(&self) -> &'static str {
self.inner.provider()
}
pub fn zones(&self) -> &[Zone] {
self.inner.zones()
}
pub fn price_resource(&self, r: &Resource) -> Result<Price, CloudError> {
monetize_cloud::Cloud::price(&self.inner, &monetize_cloud::cloud::Priced::Resource(r))
}
pub fn ensure(&self, reference: &str, r: &Resource) -> Result<Ensured, CloudError> {
self.gate.arrive();
self.inner.ensure(reference, r)
}
pub fn destroy_id(&self, id: &ResourceId) -> Result<(), CloudError> {
monetize_cloud::ask::destroy_one(&self.inner, id)
}
pub fn status(&self, id: &ResourceId) -> Result<ResourceStatus, CloudError> {
self.inner.status(id)
}
pub fn list_owned(&self) -> Result<Inventory, CloudError> {
monetize_cloud::ask::account(&self.inner)
}
}
#[allow(unused_variables, unreachable_code)]
impl monetize_cloud::cloud::Cloud for HeldAtTheTill {
fn manifest(&self) -> monetize_cloud::cloud::CloudManifest {
monetize_cloud::cloud::CloudManifest { provider: self.provider().to_string(), zones: self.zones().to_vec(), settings: Vec::new(), grow_refused: Some(monetize_cloud::grow::unsupported(self.provider()).to_string()) }
}
fn catalog(&self, zone: &monetize_cloud::Zone) -> Result<monetize_cloud::cloud::Catalog, monetize_cloud::CloudError> {
Err(monetize_cloud::lay::unsupported(self.provider()))
}
fn price(&self, what: &monetize_cloud::cloud::Priced<'_>) -> Result<monetize_cloud::Price, monetize_cloud::CloudError> {
match what {
monetize_cloud::cloud::Priced::Resource(r) => self.price_resource(r),
monetize_cloud::cloud::Priced::Plan(p) => monetize_cloud::price_estate_by_rows(p, |r| self.price_resource(r), None),
monetize_cloud::cloud::Priced::Grow(set) => Err(monetize_cloud::grow::unsupported(self.provider())),
}
}
fn lay(&self, plan: &monetize_cloud::lay::EstatePlan, journal: &monetize_cloud::grow::Journal) -> Result<monetize_cloud::lay::LaidEstate, monetize_cloud::CloudError> {
Err(monetize_cloud::lay::unsupported(self.provider()))
}
fn change(&self, change: &monetize_cloud::cloud::Change, journal: &monetize_cloud::grow::Journal) -> Result<monetize_cloud::cloud::Changed, monetize_cloud::CloudError> {
Ok(match change {
monetize_cloud::cloud::Change::Plan { server, plan, delta } => return Err(monetize_cloud::lay::unsupported_verb(self.provider(), "change_plan")),
monetize_cloud::cloud::Change::Grow { set, estate } => return Err(monetize_cloud::grow::unsupported(self.provider())),
monetize_cloud::cloud::Change::Settle { set } => return Err(monetize_cloud::grow::unsupported(self.provider())),
monetize_cloud::cloud::Change::ForgetMedia { laid } => monetize_cloud::cloud::Changed::Forgotten(Vec::new()),
monetize_cloud::cloud::Change::Ensure { reference, resource, room } => monetize_cloud::cloud::Changed::Ensured(self.ensure(reference, resource)?),
monetize_cloud::cloud::Change::Retire { reference, machine } => monetize_cloud::cloud::Changed::Retired(monetize_cloud::cloud::retire_by_rows(self, reference, machine)?),
monetize_cloud::cloud::Change::LabelAdopted { .. } => return Err(monetize_cloud::lay::unsupported_verb(self.provider(), "label_adopted")),
})
}
fn boot(&self, boot: &monetize_cloud::cloud::Boot, journal: &monetize_cloud::grow::Journal) -> Result<monetize_cloud::cloud::Booted, monetize_cloud::CloudError> {
Ok(match boot {
monetize_cloud::cloud::Boot::Keep(k) => return Err(monetize_cloud::lay::unsupported_verb(self.provider(), "keep_boot")),
monetize_cloud::cloud::Boot::Power { server, on } => return Err(monetize_cloud::lay::unsupported_verb(self.provider(), "set_power")),
monetize_cloud::cloud::Boot::Restore(r) => return Err(monetize_cloud::lay::unsupported_verb(self.provider(), "restore_from_backup")),
})
}
fn firewall(&self, laid: &monetize_cloud::lay::LaidEstate) -> Result<(), monetize_cloud::CloudError> {
Err(monetize_cloud::lay::unsupported_verb(self.provider(), "set_estate_firewall"))
}
fn destroy(&self, what: &monetize_cloud::cloud::Give<'_>) -> Result<monetize_cloud::Reclaimed, monetize_cloud::CloudError> {
match what {
monetize_cloud::cloud::Give::References(refs) => Ok(monetize_cloud::release_for_by_rows(self.list_owned()?, refs, |id| self.destroy_id(id))),
monetize_cloud::cloud::Give::Resource(id) => {
self.destroy_id(id)?;
Ok(monetize_cloud::Reclaimed { destroyed: vec![(*id).clone()], ..monetize_cloud::Reclaimed::default() })
}
}
}
fn inventory(&self, what: &monetize_cloud::cloud::Listing<'_>) -> Result<monetize_cloud::cloud::Listed, monetize_cloud::CloudError> {
Ok(match what {
monetize_cloud::cloud::Listing::Account => monetize_cloud::cloud::Listed::Account(self.list_owned()?),
monetize_cloud::cloud::Listing::Set(set) => return Err(monetize_cloud::grow::unsupported(self.provider())),
monetize_cloud::cloud::Listing::Status(id) => monetize_cloud::cloud::Listed::Status(self.status(id)?),
monetize_cloud::cloud::Listing::Under(refs) => monetize_cloud::cloud::Listed::Under(monetize_cloud::cloud::under(self.list_owned()?, refs)),
monetize_cloud::cloud::Listing::Room { delta, room } => monetize_cloud::cloud::Listed::Room(monetize_cloud::cloud::RoomLeft { serve: Ok(()), left_bytes: None, ceilings: Default::default() }),
monetize_cloud::cloud::Listing::Admits(_plan) => monetize_cloud::cloud::Listed::Admitted,
})
}
}
/// `MockCloud` prices every resource at 10.00 EUR a month.
const DISK_MINOR: u64 = 1_000;
const BUYERS: usize = 4;
let disk = Resource::Disk { server: ServerId("gunnar-se-sto1".into()), gib: 10 };
let dir = std::env::temp_dir().join(format!("monetize-cap-race-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
// ── the control: one at a time, the ceiling holds exactly ──
{
let inner = Arc::new(MockCloud::new());
inner.answer_all(Answer::Provisioned);
let gate = Arc::new(Gate::new(1));
let cap = SpendCap::new(HeldAtTheTill { inner: inner.clone(), gate }, Ceiling::total(2 * DISK_MINOR, Currency::Eur), dir.join("serial.json")).unwrap();
assert!(monetize_cloud::ask::ensure(&cap, "ref-1", &disk, None).is_ok());
assert!(monetize_cloud::ask::ensure(&cap, "ref-2", &disk, None).is_ok());
let refused = monetize_cloud::ask::ensure(&cap, "ref-3", &disk, None).expect_err("the third crosses the ceiling");
assert!(refused.to_string().contains("in total"), "{refused}");
assert_eq!(inner.live_count(), 2, "the third disk was bought anyway");
assert_eq!(cap.remaining(), 0);
}
// ── the same ceiling, four buyers inside the till at once ──
let inner = Arc::new(MockCloud::new());
inner.answer_all(Answer::Provisioned);
let gate = Arc::new(Gate::new(BUYERS));
let cap = SpendCap::new(
HeldAtTheTill { inner: inner.clone(), gate: gate.clone() },
Ceiling::total(2 * DISK_MINOR, Currency::Eur),
dir.join("race.json"),
)
.unwrap();
let refusals: Vec<String> = std::thread::scope(|s| {
let handles: Vec<_> = (0..BUYERS)
.map(|i| {
let (cap, disk, gate) = (&cap, &disk, gate.clone());
s.spawn(move || match monetize_cloud::ask::ensure(&cap, &format!("ref-{i}"), disk, None) {
Ok(_) => None,
Err(e) => {
// Refused at admission, so this one never reaches the
// till: the gate must stop waiting for it.
gate.gave_up();
Some(e.to_string())
}
})
})
.collect();
handles.into_iter().filter_map(|h| h.join().unwrap()).collect()
});
let bought = BUYERS - refusals.len();
// **THE MEASUREMENT.** The ceiling holds under concurrency: the money an
// admitted buyer is about to spend is RESERVED under the lock that admitted
// it, so the buyers behind it see that money and are refused on it.
//
// This asserted `bought == BUYERS` while the defect was open — four disks
// on a ceiling with room for two, twice the money committed, and the
// refusal an operator set that number for never happening. Its own note
// said: "the cap refused somebody, so the window has been closed — update
// this test, it is the good news." This is that update.
assert_eq!(bought, 2, "the ceiling has room for exactly two disks, concurrent or not");
assert_eq!(inner.live_count(), 2, "a disk was bought past the ceiling: {refusals:?}");
assert_eq!(cap.remaining(), 0);
assert_eq!(refusals.len(), 2, "two buyers should have been refused");
for why in &refusals {
assert!(why.contains("in total"), "the refusal names the ceiling it hit: {why}");
}
// The control above proves the same ceiling admits exactly two when they
// come one at a time. Concurrency now changes nothing, which is the point.
eprintln!(
"MEASURED spend cap: ceiling {} EUR-minor, {BUYERS} concurrent buyers, {bought} bought, {} EUR-minor committed",
2 * DISK_MINOR,
bought as u64 * DISK_MINOR
);
let _ = std::fs::remove_dir_all(&dir);
}
// ── the appliance stops answering ──────────────────────────────────────────
//
// `grow::tests` holds the appliance that accepts and then never PROGRESSES
// (`refuse_twin_a_start_while_open_is_refused_and_a_stuck_drain_is_a_named_
// timeout` — status keeps saying `draining` until `DRAIN_TIMEOUT`). What
// follows is the appliance that stops ANSWERING, which is a different failure
// and until now had no test at all: `MockProduct`'s `flush` and `status` could
// not be made to fail, so `phase::FLUSH_TIMEOUT` and `phase::APPLIANCE_TIMEOUT`
// were named in `grow.rs` and reached by nothing.
/// ★ **A FLUSH NOBODY COULD ANSWER PARKS; A FLUSH THAT WAS REFUSED FAILS.**
///
/// The same rule as everywhere else in this file, on the appliance's side of
/// the growth: `Unavailable` means the question did not get through, so the
/// growth waits and the next tick asks again; `Refused` is an answer, so it is
/// over. Either way **nothing is touched** — the flush is the first step, and
/// no volume has been detached and no money committed when it fails.
#[test]
fn a_flush_that_could_not_be_asked_parks_and_a_flush_that_was_refused_fails() {
use monetize_product::ProductError;
use monetize_server::grow::phase;
// (a) UNAVAILABLE — the appliance is unreachable.
let e = Estate::up("flush-unavailable", GrowTrigger::Used);
e.product.flush_fails(ProductError::Unavailable("transport error: connection refused".into()));
monetize_server::grow::start(&e.state, "gunnar", 64).expect("the growth is approved: the flush has not been tried yet");
wait_until_parked(&e);
let run = e.pg().snapshot();
assert_eq!(run.phase, phase::FLUSH_TIMEOUT, "{run:?}");
assert!(run.phase_detail.contains("connection refused"), "{}", run.phase_detail);
assert!(phase::is_timeout(&run.phase), "a parked timeout, so an operator's Resume can drive it on");
assert!(!phase::is_terminal(&run.phase), "a growth nobody could ask about is not a growth that is over");
assert!(e.cloud.grow_calls().is_empty(), "a volume was touched before the twin was known to be caught up");
assert_eq!(e.spent_minor(), 0, "money was committed for a growth that never began");
assert!(e.product.grow_script().unwrap().started.is_empty(), "the go-ahead was sent anyway");
// (b) REFUSED — the appliance answered, and the answer was no.
let r = Estate::up("flush-refused", GrowTrigger::Used);
r.product.flush_fails(ProductError::Refused("gunnar: replication is disabled on this deployment".into()));
monetize_server::grow::start(&r.state, "gunnar", 64).expect("approved");
wait_until_parked(&r);
let run = r.pg().snapshot();
assert_eq!(run.phase, phase::FAILED, "{run:?}");
assert!(run.last_error.contains("the flush was refused, nothing was touched"), "{}", run.last_error);
assert!(r.cloud.grow_calls().is_empty());
assert_eq!(r.spent_minor(), 0);
}
/// ★ **AN APPLIANCE THAT TAKES THE GO-AHEAD AND THEN STOPS ANSWERING IS A
/// NAMED TIMEOUT, AND THE VOLUMES ARE NOT LEFT DETACHED.**
///
/// The dangerous one. `Admin.GrowStart` succeeded, so the box has been told to
/// unmount, and then nothing comes back. The growth must not sit in the phase
/// for ever, must not guess that the box is done, and must above all not go on
/// to the cloud and detach volumes from an appliance whose state is unknown.
#[test]
fn an_appliance_that_accepts_the_go_ahead_and_goes_silent_is_a_named_timeout() {
use monetize_product::ProductError;
use monetize_server::grow::phase;
let e = Estate::up("appliance-silent", GrowTrigger::Used);
// `start` still answers — the box took the go-ahead — and every `status`
// after it does not.
e.product.status_answers(Some(ProductError::Unavailable("deadline exceeded after 30 s".into())));
monetize_server::grow::start(&e.state, "gunnar", 64).expect("approved");
wait_until_parked(&e);
let run = e.pg().snapshot();
assert!(phase::is_timeout(&run.phase), "the growth ended {} rather than a named timeout: {run:?}", run.phase);
assert!(run.last_error.contains("deadline exceeded"), "the appliance's own words are kept: {}", run.last_error);
assert!(!phase::is_terminal(&run.phase), "a silent box is not a finished growth");
// NOTHING WAS TOUCHED AND NOTHING WAS SPENT. The volumes are where the
// appliance left them, which is the only safe answer when its state is
// unknown.
assert!(e.cloud.grow_calls().is_empty(), "the cloud was driven against an appliance nobody could ask about");
assert_eq!(e.spent_minor(), 0);
assert!(e.state.ledger.recent(100).unwrap().is_empty(), "a Grown row for a growth that never reached the cloud");
// And when the box comes back, the operator's Resume drives it on rather
// than starting a second growth.
e.product.status_answers(None);
e.product.grows(PHASES.to_vec(), true);
monetize_server::grow::resume(&e.state, "gunnar", false).expect("a parked timeout resumes");
wait_until_parked(&e);
assert_eq!(e.pg().snapshot().phase, phase::DONE, "{:?}", e.pg().snapshot());
assert_eq!(e.cloud.grow_calls().len(), 1, "exactly one resize, after the box came back");
}
/// The drive runs on its own thread; this is the only wall-clock wait in the
/// file and it is bounded in milliseconds, never in the growth's own timeouts.
fn wait_until_parked(e: &Estate) {
let start = Instant::now();
while e.pg().is_running() || phase::is_open(&e.pg().snapshot().phase) {
assert!(start.elapsed() < Duration::from_secs(20), "the drive did not park: {:?}", e.pg().snapshot());
std::thread::sleep(Duration::from_millis(1));
}
}
/// ★ **A TWIN THAT STOPPED FOLLOWING BLOCKS EVERY GROWTH, FOR EVER, AND THE
/// SENTENCE IT PRINTS IS ABOUT REPLICATION LAG.**
///
/// This is the estate's real failure mode, not an invented one. MEASURED on
/// gunnar.rs 2026-09-19: the twin read `applied 0 of 19264` and was silent,
/// which matches `start_follower` calling `run()` exactly ONCE — the thread
/// exits for ever on any pre-loop failure and nothing retries it.
///
/// `flushing` is phase 2, before `twin-sizing` and before anything is touched,
/// and it refuses to start a growth unless the twin is caught up. A twin whose
/// follower is dead can never catch up. So **every** growth the estate will
/// ever order dies at the same step, and each one dies TERMINALLY (`failed`,
/// not a parked timeout), so nothing re-drives it and no operator is paged.
///
/// The cruelty is in the words: the estate is broken because a thread died,
/// and every message it produces is about replication lag. Nothing in the
/// refusal names the follower, so the operator reads "the twin is behind",
/// waits for it to catch up, and waits for ever.
///
/// Meanwhile the shop does not consult the twin at all — `max_sold_percent` is
/// a percentage of the IRON — so it keeps selling into a set that can no
/// longer grow. That is the pair this test pins: **growth stopped, selling did
/// not.**
#[test]
fn a_twin_that_stopped_following_blocks_every_growth_while_the_shop_keeps_selling() {
use monetize_product::ProductError;
use monetize_server::grow::phase;
let e = Estate::up("twin-stopped-following", GrowTrigger::Used);
// The appliance's own words for a follower that is not moving.
e.product
.flush_fails(ProductError::Refused("gunnar: the twin is not caught up: applied 0 of 19264".into()));
let began_at = e.set_gib();
// Three growths, at rising targets, exactly as a filling estate would ask
// for them over the following hours.
for (n, target) in [64u64, 96, 128].into_iter().enumerate() {
monetize_server::grow::start(&e.state, "gunnar", target)
.unwrap_or_else(|err| panic!("attempt {n} was not even approved: {err}"));
wait_until_parked(&e);
let run = e.pg().snapshot();
// It is OVER, not parked. A timeout would be re-driven by the next
// poller tick; `failed` is nobody coming back.
assert_eq!(run.phase, phase::FAILED, "attempt {n} ended {}: {run:?}", run.phase);
assert!(phase::is_terminal(&run.phase), "attempt {n} is not terminal, so something would retry it");
assert!(!phase::is_timeout(&run.phase), "attempt {n} is a timeout, which an operator Resume would drive on");
// The same sentence every time — the estate learns nothing between
// attempts, because there is nothing in the flush's answer to learn.
assert!(
run.last_error.contains("not caught up") || run.last_error.contains("applied 0 of 19264"),
"attempt {n} lost the appliance's own words: {}",
run.last_error
);
// ★ and it is about REPLICATION, never about the thread that died.
let said = run.last_error.to_ascii_lowercase();
assert!(
!said.contains("follower") && !said.contains("thread"),
"attempt {n} named the real cause, which would make this test obsolete and the estate better: {}",
run.last_error
);
}
// NOTHING was touched and NOTHING was spent, across all three. The flush is
// the first step, so that part of the design holds exactly as intended.
assert!(e.cloud.grow_calls().is_empty(), "a volume was touched behind a twin nobody proved was caught up");
assert_eq!(e.spent_minor(), 0, "money was committed for growths that never began");
assert_eq!(e.set_gib(), began_at, "the set grew while the flush was refusing every time");
assert!(e.product.grow_script().unwrap().started.is_empty(), "the go-ahead reached the appliance anyway");
// ★ AND THE SHOP NEVER ASKED. Its ceiling is `max_sold_percent` of the
// iron, and the iron is exactly what stopped growing, so the allowance is
// unchanged and the next customer is sold as if nothing were wrong.
let allowance_gib = began_at * u64::from(MAX_SOLD_PERCENT) / 100;
assert!(
allowance_gib >= PLAN_GIB,
"the shop can still sell {allowance_gib} GiB of promises against a set that can no longer grow past {began_at} GiB"
);
// When the follower is restarted, the very next growth goes through — which
// is the proof that the twin was the only thing wrong, and that the estate
// had no way to say so.
e.product.flush_ok();
monetize_server::grow::start(&e.state, "gunnar", 64).expect("approved once the twin follows again");
wait_until_parked(&e);
assert_eq!(e.pg().snapshot().phase, phase::DONE, "{:?}", e.pg().snapshot());
assert_eq!(e.cloud.grow_calls().len(), 1, "exactly one resize, after the twin came back");
}
// ── the two ceilings, and the question nobody asks ─────────────────────────
/// ★ **THE SHOP RUNS PAST THE LAST AFFORDABLE DISK, AND NOW THERE IS A NUMBER
/// FOR IT.** STINK S2, which asked for exactly this row: *customers sold past
/// the last affordable disk*.
///
/// The two ceilings are different units. `max_sold_percent_of_capacity` is a
/// percentage OF THE IRON, so every growth raises it; `[clouds.<n>] cap` is
/// €25 and only falls. Until `grow::affordable_bytes` there was nowhere in the
/// code that held both, so nothing could say how far apart they had walked.
///
/// The measurement, in three parts:
///
/// 1. **The last affordable disk is knowable BEFORE anything is sold.** At T0
/// the set is 37 GiB and the wallet can still buy 108 (€25 at 23 EUR-minor
/// a GiB-month), so 145 GiB is the largest set this estate can ever serve —
/// and 145 GiB is exactly what the measured run ends at, an hour and five
/// growths later. The number is not a prediction, it is the same
/// arithmetic the clamp spends by.
/// 2. **The shop promises 870 GiB against it**, because 600 % of the LAST set
/// size is what the allowance became.
/// 3. **Seventy-odd customers are sold after the promise passed the last
/// affordable disk** — and every one of them is now a RED
/// `shop-past-the-wallet` row with the gap in bytes, where before there was
/// nothing at all.
#[test]
fn the_shop_sells_past_the_last_affordable_disk_and_the_gap_is_a_number() {
// ── 1. before a single sale ──
let e = Estate::up("past-wallet", GrowTrigger::Used);
let affordable = monetize_server::grow::affordable_bytes(&e.state, "gunnar").expect("the wallet can be asked");
assert_eq!(affordable / GIB, 108, "€{} at {PER_GIB_MINOR} EUR-minor a GiB-month", CEILING_TOTAL_MINOR / 100);
assert_eq!(
(POOL_GIB * GIB + affordable) / GIB,
145,
"the LAST AFFORDABLE DISK, known before anything is sold — and the size the measured run actually ends at"
);
// Nothing is promised yet, so nothing is past it.
assert_eq!(monetize_server::oversell::past_the_wallet(&e.state, "gunnar", 0, POOL_GIB * GIB), None);
// One tenant's 10 GiB is inside it; 200 GiB of promises is not.
assert_eq!(monetize_server::oversell::past_the_wallet(&e.state, "gunnar", PLAN_GIB * GIB, POOL_GIB * GIB), None);
let gap = monetize_server::oversell::past_the_wallet(&e.state, "gunnar", 200 * GIB, POOL_GIB * GIB).expect("200 GiB is past 145");
assert_eq!((gap.over / GIB, gap.affordable / GIB, gap.capacity / GIB), (55, 108, 37), "{gap:?}");
drop(e);
// ── 2 and 3. the same estate the stink list measured ──
let r = simulate("past-wallet-run", Scenario { trigger: GrowTrigger::Used, per_day: 10, days: 10, writes_gib: 8, polled: true });
assert_eq!((r.sold, r.set_gib_end, r.spent_minor), (87, 145, 2_484), "{}", r.line());
assert_eq!(r.promised_gib_end, 870, "600 % of the last set size");
// THE NUMBER. Sales made when the promise was already past the largest set
// this estate can ever buy.
assert!(
r.sold_past_the_wallet >= 70,
"only {} of {} sales were past the last affordable disk: {}",
r.sold_past_the_wallet,
r.sold,
r.line()
);
assert!(r.sold_past_the_wallet < r.sold, "the first sales are not past it: {}", r.line());
eprintln!("MEASURED S2: {} of {} customers sold past the last affordable disk", r.sold_past_the_wallet, r.sold);
}
/// ★ **THE GROWTH PATH CAN NOW ASK WHAT WAS COLLECTED — AND STILL DOES NOT ACT
/// ON IT.** STINK S3, whose claim was that *"the ledger has no `collected` fold
/// at all, so the growth path could not consult revenue if it wanted to"*.
///
/// MEASURED against the code, that was half right. `Ledger::sales` has folded
/// every attempt into `granted | refunded | declined | open` all along; what
/// was missing was a per-product total and a caller. `Ledger::collected_minor`
/// is that total, over that same fold, and `grow::collected_minor` is the
/// growth path's reader for it.
///
/// So the gap is now exactly one sentence long, and it is a DESIGN decision
/// rather than a missing mechanism: two estates driven to the same fill, one
/// with 1 914 000 SEK-minor collected and one that has never taken a krona,
/// **read differently and spend identically**.
#[test]
fn the_growth_path_can_now_ask_what_was_collected_and_still_spends_the_same() {
let paid = Estate::up("collected-paid", GrowTrigger::Used);
let mut revenue = 0u64;
for i in 0..87u64 {
paid.at(T0 + i * 1_000);
let mut p = purchase(&format!("t{i:05}"));
p.starts_unix_ms = T0 + i * 1_000;
assert!(matches!(paid.tx().run(&p).unwrap(), Outcome::Paid { .. }));
revenue += p.price.minor;
}
let free = Estate::up("collected-free", GrowTrigger::Used);
// ── THE READING EXISTS, AND IT IS THE SALES FOLD'S OWN NUMBER ──
assert_eq!(monetize_server::grow::collected_minor(&paid.state, "gunnar", Currency::Sek), revenue);
assert_eq!(revenue, 1_914_000, "87 × 220.00 SEK");
assert_eq!(monetize_server::grow::collected_minor(&free.state, "gunnar", Currency::Sek), 0);
// A currency nobody sold in is 0, not the SEK figure: no invented FX.
assert_eq!(monetize_server::grow::collected_minor(&paid.state, "gunnar", Currency::Eur), 0);
// And a product nobody sold.
assert_eq!(monetize_server::grow::collected_minor(&paid.state, "not-a-product", Currency::Sek), 0);
// ── AND THE SPEND IS STILL BLIND TO IT ──
// Both estates filled to the high-water mark and left to the batcher.
for e in [&paid, &free] {
let mut t = T0;
for _ in 0..40 {
let total = e.set_gib() * GIB;
*e.written.lock().unwrap() = total * 9 / 10;
e.refresh();
t += 3_600_000 + 120_000;
e.tick_at(t);
e.tick_at(t + 60_000);
e.tick_at(t + 70_000);
}
}
assert_eq!(
(paid.spent_minor(), paid.set_gib()),
(free.spent_minor(), free.set_gib()),
"revenue changed the spend; if this ever fails, the owner has ruled and this test is the place to say so"
);
assert_eq!(free.spent_minor(), 2_484, "the estate that never took a krona spent the whole ceiling");
}