crates/monetize-cloud/src/cap/mod.rs
//! **`SpendCap` — a ceiling a cloud provider cannot spend past.**
//!
//! Written 2026-09-04 so a real provider account can be rehearsed against
//! without an unbounded bill.
//!
//! `Change::Ensure` is the only call in monetize that can spend
//! money at a provider, and the pool logic already stops it buying the SAME thing
//! twice (labels, then the pool, then buy — `upcloud`'s `ensure_disk`). What was
//! missing is a stop on buying too many DIFFERENT things: a loop that creates
//! fifty tenants against a real account buys fifty disks, each one correctly.
//!
//! This wraps any provider and refuses before it can:
//!
//! ```text
//! ensure(reference, resource)
//! ├─ price(resource) ask what it would cost
//! ├─ spent_this_month + price > ceiling ─▶ OverCap, naming all three numbers
//! └─ otherwise delegate
//! └─ Provisioned { cost } ─▶ add cost to the month's total (durable)
//! OnHand ─▶ add NOTHING; nothing was bought
//! ```
//!
//! # Two deliberate choices
//!
//! **The total is durable and keyed by calendar month.** In-memory would reset on
//! every restart, which on a service that is restarted by an installer is no cap
//! at all. It lives in one small JSON file beside the provider's own ledger.
//!
//! **The pre-check is conservative and can refuse a FREE call.** `ensure` may
//! answer `OnHand` — nothing bought — but nothing can know that until the
//! provider is asked, and asking is the thing being guarded. So the check uses
//! the resource's list price. Near the ceiling this refuses work that would have
//! cost nothing. That is the safe direction to be wrong in: a refusal is a
//! message, an overspend is an invoice.
//!
//! # What it is not
//!
//! It is not a budget for the whole business, and it does not know what the
//! provider is already charging for resources bought before it existed. It caps
//! what THIS monetize spends from the moment it is configured. `destroy` is never
//! capped — giving a resource back must always be allowed, and it is the one call
//! that makes the number go down at the provider.
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use monetize_pay::Currency;
use std::sync::Arc;
use crate::cloud::{
Boot, Booted, Catalog, Change, Changed, Cloud, CloudManifest, Give, Listed, Listing, Priced,
};
use crate::grow::Journal;
use crate::lay::{EstatePlan, LaidEstate};
use crate::{CloudError, Ensured, Price, Reclaimed, Resource, Room, Zone};
/// The running totals, as they are stored. A CACHE, not the truth — see
/// [`SpendCap::reconcile_total`].
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
struct Spent {
/// `YYYY-MM`. A different month starts from zero.
month: String,
/// Committed in `month`.
#[serde(alias = "minor")]
month_minor: u64,
/// Committed EVER, across every month. Never reset.
#[serde(default)]
total_minor: u64,
/// **Reserved, not yet committed**: the delta of a data-set growth that
/// has begun and not yet reported what it grew, keyed by the set
/// (`DataSet::resource_key`). Counted against both limits by `admit`, so
/// a second growth cannot spend the headroom the first is using; dropped
/// when the growth reports (T7: reserved before volume 1 is touched).
#[serde(default)]
reserved: std::collections::BTreeMap<String, u64>,
/// **Estates laid and committed, by reference**: `(month, cost)` — so an
/// estate given back in the month it was laid gives its month back
/// (`Give::References`; round 5.3.1, the UpCloud mockup's test2
/// row: a paid order's failed install was given back and laid again, and
/// the cap counted it twice and held the next order back).
#[serde(default)]
laid: std::collections::BTreeMap<String, (String, u64)>,
}
impl Spent {
fn reserved_total(&self) -> u64 {
self.reserved
.values()
.fold(0u64, |a, b| a.saturating_add(*b))
}
}
/// The ceiling itself. **At least one of the two limits must be set**; a
/// `Ceiling` with neither is refused by [`Ceiling::check`], because a guard that
/// limits nothing is worse than no guard — it reads like protection.
///
/// Both may be set, and then both are enforced. `0` means **refuse every
/// purchase** and is never read as "unlimited": an explicitly configured zero is
/// a deployment that may use only what is already on hand, which is exactly what
/// a rehearsal on a real account wants.
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct Ceiling {
/// Minor units per calendar MONTH, reset on the 1st.
pub monthly_minor: Option<u64>,
/// Minor units in TOTAL, ever, across every month — never reset.
///
/// This is the one a rehearsal wants. A monthly ceiling of 10 EUR permits
/// 10 EUR again next month and 120 EUR over a year, which is not what
/// "cap it at ten euros" means when the sentence is about a test budget.
pub total_minor: Option<u64>,
/// The currency both limits are denominated in. A price in any other
/// currency is refused rather than converted — there is no FX in this
/// codebase, and inventing one inside a spending guard is how a cap becomes
/// a fiction.
pub currency: Currency,
}
impl Ceiling {
/// A total-only ceiling: the rehearsal shape.
pub fn total(minor: u64, currency: Currency) -> Ceiling {
Ceiling {
monthly_minor: None,
total_minor: Some(minor),
currency,
}
}
/// A monthly-only ceiling.
pub fn monthly(minor: u64, currency: Currency) -> Ceiling {
Ceiling {
monthly_minor: Some(minor),
total_minor: None,
currency,
}
}
/// Refuse a ceiling that limits nothing. Called by [`SpendCap::new`], so a
/// cap that would never refuse cannot be constructed at all.
pub fn check(&self) -> Result<(), CloudError> {
if self.monthly_minor.is_none() && self.total_minor.is_none() {
return Err(CloudError::Refused(
"a spend ceiling with neither `monthly_minor` nor `total_minor` limits nothing, and a \
guard that never refuses is worse than none because it reads like protection. Set at \
least one; `total_minor` is the one a rehearsal wants."
.into(),
));
}
Ok(())
}
}
/// **The words an estate over the cloud's cap is refused with**, as a
/// [`CloudError::OverCap`] — the KIND is what the estate job reads as "wait in
/// the queue", never as a failure (the owner, 2026-10-08: the customer must
/// not know about a cap); the words are the operator's.
pub const ESTATE_OVER_CAP: &str = "the cloud's spend cap refuses";
/// The words any other purchase over the cap is refused with ([`SpendCap`]'s
/// `refuse`, a [`CloudError::OverCap`]): a plan change, a growth, a resource.
pub const PURCHASE_OVER_CAP: &str = "the spend cap refuses this purchase";
/// **What a spend that does not fit under the ceiling falls short by**
/// (`Cap::cap_short`, PURE): what it needs a month, the
/// ceiling it passes and what is already committed under it — the numbers an
/// operator's own order is refused with BEFORE anything is paid or changed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Short {
/// `this month` or `in total`.
pub which: &'static str,
pub needs: monetize_pay::Money,
pub cap: monetize_pay::Money,
pub committed: monetize_pay::Money,
}
impl Short {
/// The operator's words: `raise the cap first: this needs 204.22 EUR a
/// month, the cap is 200.00 EUR a month, 101.07 EUR is committed; nothing
/// was charged`.
pub fn words(&self) -> String {
format!(
"raise the cap first: {}; nothing was charged",
self.numbers()
)
}
/// The numbers alone: `this needs 204.22 EUR a month, the cap is 200.00
/// EUR a month, 101.07 EUR is committed`.
pub fn numbers(&self) -> String {
let cap = if self.which == "this month" {
format!("{} a month", self.cap.in_words())
} else {
format!("{} in total", self.cap.in_words())
};
format!(
"this needs {} a month, the cap is {cap}, {} is committed",
self.needs.in_words(),
self.committed.in_words()
)
}
}
/// **The spend cap's ledger**: the ceiling, the durable totals, the reservations. Held by
/// the [`SpendCap`] that guards a cloud's spends, and shared with the host that shows,
/// reconciles and asks it (the console, the estate queue, a lay's gate), so a cap is a
/// thing the host holds, never a question a cloud is asked.
pub struct Cap {
ceiling: Ceiling,
path: PathBuf,
spent: Mutex<Spent>,
month: Box<dyn Fn() -> String + Send + Sync>,
}
impl std::fmt::Debug for Cap {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Cap")
.field("ceiling", &self.ceiling)
.field("path", &self.path)
.finish_non_exhaustive()
}
}
/// **A cloud that cannot spend past its [`Cap`]**: every op that buys (a lay, a plan
/// change, a growth, an ensure) is admitted by the cap first and committed after; every
/// other op is the inner cloud's.
pub struct SpendCap<C> {
inner: C,
cap: Arc<Cap>,
}
impl<C: std::fmt::Debug> std::fmt::Debug for SpendCap<C> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SpendCap")
.field("inner", &self.inner)
.field("ceiling", &self.cap.ceiling)
.finish_non_exhaustive()
}
}
/// `YYYY-MM` from the system clock, UTC, by integer arithmetic on the epoch.
pub fn current_month() -> String {
let secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let (y, m) = year_month(secs);
format!("{y:04}-{m:02}")
}
/// Civil year and month from a Unix second count. Pure, so the boundary is
/// testable without waiting for one.
pub fn year_month(secs: u64) -> (u64, u64) {
let days = secs / 86_400;
// Days since 1970-01-01 -> civil date (Howard Hinnant's algorithm, integer only).
let z = days as i64 + 719_468;
let era = z.div_euclid(146_097);
let doe = z.rem_euclid(146_097);
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
let y = yoe + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = if m <= 2 { y + 1 } else { y };
(y as u64, m as u64)
}
impl Cap {
/// A ledger at `path` (its month's total and reservations), under `ceiling`.
pub fn new(ceiling: Ceiling, path: impl Into<PathBuf>) -> Result<Cap, CloudError> {
ceiling.check()?;
let path = path.into();
let spent = Self::load(&path)?;
Ok(Cap {
ceiling,
path,
spent: Mutex::new(spent),
month: Box::new(current_month),
})
}
fn load(path: &Path) -> Result<Spent, CloudError> {
match std::fs::read(path) {
Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| {
CloudError::Refused(format!(
"the spend cap's total at {} is unreadable: {e}",
path.display()
))
}),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Spent::default()),
Err(e) => Err(CloudError::Refused(format!(
"cannot read the spend cap's total at {}: {e}",
path.display()
))),
}
}
fn store(&self, spent: &Spent) -> Result<(), CloudError> {
if let Some(parent) = self.path.parent().filter(|p| !p.as_os_str().is_empty()) {
std::fs::create_dir_all(parent)
.map_err(|e| CloudError::Refused(format!("spend cap dir: {e}")))?;
}
let json = serde_json::to_vec_pretty(spent)
.map_err(|e| CloudError::Refused(format!("spend cap encode: {e}")))?;
std::fs::write(&self.path, json)
.map_err(|e| CloudError::Refused(format!("spend cap write: {e}")))
}
/// What has been spent in the CURRENT month. A stored total from an older
/// month reads as zero — and is not rewritten until something is bought, so
/// a read is never a side effect.
pub fn spent_this_month(&self) -> u64 {
let now = (self.month)();
let s = self.spent.lock().unwrap_or_else(|e| e.into_inner());
if s.month == now {
s.month_minor
} else {
0
}
}
/// What has been spent EVER, across every month.
pub fn spent_total(&self) -> u64 {
self.spent
.lock()
.unwrap_or_else(|e| e.into_inner())
.total_minor
}
/// **Correct the stored total from an authoritative source, upward only.**
///
/// The file this cap keeps is a CACHE. The truth is the caller's ledger,
/// which records a cost on every `Ensured { Provisioned { cost } }` step and
/// is append-only — so the total ever committed can always be recomputed
/// from it, and a lost or stale cache cannot quietly raise the budget.
/// `monetize-server` folds its ledger at start and calls this.
///
/// UPWARD ONLY, deliberately: `max` of the two. A cache that has somehow
/// recorded MORE than the ledger can see (a purchase whose ledger row failed
/// to write, the one ordering that can produce it) must not be talked down
/// by a smaller authoritative number, because under-counting spend is the
/// only direction that costs money. The monthly figure is left alone — it is
/// about a window the ledger fold does not know.
pub fn reconcile_total(&self, authoritative_total_minor: u64) -> Result<(), CloudError> {
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
if authoritative_total_minor <= spent.total_minor {
return Ok(());
}
spent.total_minor = authoritative_total_minor;
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)
}
pub fn ceiling(&self) -> Ceiling {
self.ceiling
}
/// What is left before EITHER limit refuses — the tighter of the two.
pub fn remaining(&self) -> u64 {
let reserved = self.reserved_minor();
let m = self
.ceiling
.monthly_minor
.map(|c| c.saturating_sub(self.spent_this_month().saturating_add(reserved)));
let t = self
.ceiling
.total_minor
.map(|c| c.saturating_sub(self.spent_total().saturating_add(reserved)));
match (m, t) {
(Some(a), Some(b)) => a.min(b),
(Some(a), None) => a,
(None, Some(b)) => b,
// Unreachable: `Ceiling::check` refuses this at construction.
(None, None) => 0,
}
}
/// **Admit AND reserve under ONE lock**, for a purchase that is about to
/// run.
///
/// MEASURED 2026-09-19: `ensure` was check-then-act. `admit` took the lock,
/// read, and RELEASED it; the provider call then ran with nothing held, and
/// `account` took the lock again afterwards. Between those two locks a
/// concurrent buyer saw the same headroom and was admitted on it. Four
/// buyers against a ceiling with room for two bought FOUR — 4000 EUR-minor
/// committed against 2000 — and the refusal an operator set that number for
/// never happened. Sequentially the same ceiling refuses the third.
///
/// `grow_set` already did this correctly (`reserve` before member one, T7);
/// `ensure` did not, and `ensure` is the one every PURCHASE goes through.
///
/// The reservation is durable and counts against both limits from the
/// instant it is taken, which is what makes the second caller's `admit`
/// see the first caller's money.
fn admit_and_reserve(&self, key: &str, asking: Price) -> Result<(), CloudError> {
let now = (self.month)();
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
let reserved = spent.reserved_total();
let this_month = if spent.month == now {
spent.month_minor
} else {
0
}
.saturating_add(reserved);
if let Some(ceiling) = self.ceiling.monthly_minor {
if this_month.saturating_add(asking.minor_per_month) > ceiling {
return Err(self.refuse("this month", asking, this_month, ceiling));
}
}
if let Some(ceiling) = self.ceiling.total_minor {
let total = spent.total_minor.saturating_add(reserved);
if total.saturating_add(asking.minor_per_month) > ceiling {
return Err(self.refuse("in total", asking, total, ceiling));
}
}
// Held before the lock is dropped: this is the whole fix.
spent
.reserved
.insert(key.to_owned(), asking.minor_per_month);
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)
}
/// **Admit `asking` and reserve it under `key` in ONE hold of the lock**
/// (wiring trace A27, 2026-10-08): an estate's lay and a plan change did
/// the check and the reservation under two holds, so an `ensure` or a
/// growth between them could pass the same headroom. `key`'s own earlier
/// reservation (a resumed lay's) is replaced, never counted against it.
/// `Ok(Err((which, committed, ceiling)))`: over that ceiling, nothing held.
#[allow(clippy::type_complexity)]
fn admit_reserving(
&self,
key: &str,
asking: Price,
) -> Result<Result<(), (&'static str, u64, u64)>, CloudError> {
let now = (self.month)();
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
spent.reserved.remove(key);
let reserved = spent.reserved_total();
let this_month = if spent.month == now {
spent.month_minor
} else {
0
}
.saturating_add(reserved);
let over = match (self.ceiling.monthly_minor, self.ceiling.total_minor) {
(Some(c), _) if this_month.saturating_add(asking.minor_per_month) > c => {
Some(("this month", this_month, c))
}
(_, Some(c))
if spent
.total_minor
.saturating_add(reserved)
.saturating_add(asking.minor_per_month)
> c =>
{
Some(("in total", spent.total_minor.saturating_add(reserved), c))
}
_ => None,
};
if over.is_none() {
spent
.reserved
.insert(key.to_owned(), asking.minor_per_month);
}
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)?;
Ok(match over {
Some(o) => Err(o),
None => Ok(()),
})
}
/// **Settle a purchase's reservation**: drop it and commit what the
/// provider actually charged. `OnHand` bought nothing and commits nothing —
/// the reservation simply goes.
fn settle_ensure(
&self,
key: &str,
now: String,
outcome: Ensured,
) -> Result<Ensured, CloudError> {
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
spent.reserved.remove(key);
if let Ensured::Provisioned { cost, .. } = &outcome {
if spent.month != now {
spent.month = now;
spent.month_minor = 0;
}
spent.month_minor = spent.month_minor.saturating_add(cost.minor_per_month);
spent.total_minor = spent.total_minor.saturating_add(cost.minor_per_month);
}
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)?;
Ok(outcome)
}
/// The ceiling check for an already-priced ask. What is RESERVED by a
/// growth in flight counts as committed on both limits: a reservation is
/// money the provider will bill the moment the resize lands.
fn admit_priced(&self, asking: Price) -> Result<(), CloudError> {
match self.over(asking, &[]) {
Some((which, committed, ceiling)) => {
Err(self.refuse(which, asking, committed, ceiling))
}
None => Ok(()),
}
}
/// **Which ceiling `asking` would pass**, if any: `(which, committed,
/// ceiling)`, the month's checked first. PURE. `mine`: reservations of
/// the asker's own that this asking replaces (a resumed lay's, a resumed
/// plan change's), not counted against it.
fn over(&self, asking: Price, mine: &[String]) -> Option<(&'static str, u64, u64)> {
let now = (self.month)();
let spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
let own: u64 = mine
.iter()
.filter_map(|k| spent.reserved.get(k).copied())
.sum();
let reserved = spent.reserved_total().saturating_sub(own);
let this_month = if spent.month == now {
spent.month_minor
} else {
0
}
.saturating_add(reserved);
if let Some(ceiling) = self.ceiling.monthly_minor {
if this_month.saturating_add(asking.minor_per_month) > ceiling {
return Some(("this month", this_month, ceiling));
}
}
if let Some(ceiling) = self.ceiling.total_minor {
let total = spent.total_minor.saturating_add(reserved);
if total.saturating_add(asking.minor_per_month) > ceiling {
return Some(("in total", total, ceiling));
}
}
None
}
/// **An estate over the cloud's cap, said in words** (2026-10-08, test2's
/// shop: the cap is the one thing that may stop a lay, and its refusal
/// must name the cap and the price): what the estate costs a month at
/// the provider, the cap, and what is already committed under it.
fn refuse_estate(
&self,
reference: &str,
which: &str,
price: Price,
committed: u64,
ceiling: u64,
) -> CloudError {
let words = |minor: u64| {
monetize_pay::Money {
minor,
currency: self.ceiling.currency,
}
.in_words()
};
let cap = if which == "this month" {
format!("{} a month", words(ceiling))
} else {
format!("{} in total", words(ceiling))
};
CloudError::OverCap(format!(
"{ESTATE_OVER_CAP} {reference}: the estate costs {} a month at the provider, the cap is {cap}, \
and {} is already committed under it; nothing was bought. The cap is the operator's to raise",
monetize_pay::Money { minor: price.minor_per_month, currency: price.currency }.in_words(),
words(committed)
))
}
/// **Settle a growth's reservation from what it reported**: commit the
/// outcome's cost (what actually grew — zero on the idempotent repeat,
/// the grown members' delta on a partial) and drop the reservation.
fn settle_grow(&self, key: &str, outcome: &crate::grow::GrowOutcome) -> Result<(), CloudError> {
let now = (self.month)();
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
spent.reserved.remove(key);
if let Some(cost) = outcome.cost() {
if cost.minor_per_month > 0 {
if spent.month != now {
spent.month = now;
spent.month_minor = 0;
}
spent.month_minor = spent.month_minor.saturating_add(cost.minor_per_month);
spent.total_minor = spent.total_minor.saturating_add(cost.minor_per_month);
}
}
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)
}
/// Turn a laid estate's reservation into spend: the month and the total
/// move by the estate's monthly price, once.
fn settle_lay(&self, key: &str, cost: Price) -> Result<(), CloudError> {
let now = (self.month)();
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
if spent.reserved.remove(key).is_some() && cost.minor_per_month > 0 {
if spent.month != now {
spent.month = now.clone();
spent.month_minor = 0;
}
spent.month_minor = spent.month_minor.saturating_add(cost.minor_per_month);
spent.total_minor = spent.total_minor.saturating_add(cost.minor_per_month);
if let Some(reference) = key.strip_prefix("lay:") {
spent.laid.retain(|_, (m, _)| *m == now);
spent
.laid
.insert(reference.to_string(), (now, cost.minor_per_month));
}
}
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)
}
/// Drop a reservation for a growth that refused before touching anything.
fn unreserve(&self, key: &str) -> Result<(), CloudError> {
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
if spent.reserved.remove(key).is_none() {
return Ok(());
}
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)
}
/// What growths in flight hold against the ceiling right now.
pub fn reserved_minor(&self) -> u64 {
self.spent
.lock()
.unwrap_or_else(|e| e.into_inner())
.reserved_total()
}
/// Move the cached totals for what `ensure` came back with.
fn account(&self, now: String, outcome: Ensured) -> Result<Ensured, CloudError> {
// Only a PURCHASE moves the total. `OnHand` bought nothing, so charging
// the month for it would make the cap tighten itself for free.
if let Ensured::Provisioned { cost, .. } = &outcome {
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
if spent.month != now {
// A new month zeroes the WINDOW and keeps the total. That
// asymmetry is the whole point of having both.
spent.month = now;
spent.month_minor = 0;
}
spent.month_minor = spent.month_minor.saturating_add(cost.minor_per_month);
spent.total_minor = spent.total_minor.saturating_add(cost.minor_per_month);
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)?;
}
Ok(outcome)
}
fn refuse(&self, which: &str, price: Price, committed: u64, ceiling: u64) -> CloudError {
// Every amount in the operator's words: currency and two decimals
// (live 2026-10-14 said "20422 Eur/month" for 204.22 EUR).
let words = |minor: u64| {
monetize_pay::Money {
minor,
currency: self.ceiling.currency,
}
.in_words()
};
let cap = if which == "this month" {
format!("{} a month", words(ceiling))
} else {
format!("{} in total", words(ceiling))
};
CloudError::OverCap(format!(
"{PURCHASE_OVER_CAP}: it would cost {} a month, the cap is {cap}, and {} is already committed {which}. \
Raise the cap, free pool space so the provider answers OnHand, or destroy what is no longer needed.",
monetize_pay::Money { minor: price.minor_per_month, currency: price.currency }.in_words(),
words(committed)
))
}
/// **Would laying `plan` (priced `asking` by its cloud) fit under the ceiling now?**
/// PURE: nothing is reserved. An order the cap cannot admit yet WAITS in its queue.
pub fn can_lay(&self, plan: &EstatePlan, asking: Price) -> Result<(), CloudError> {
if asking.currency != self.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this estate is priced in {:?}; there is no conversion here",
self.ceiling.currency, asking.currency
)));
}
let key = format!("lay:{}", crate::lay::spend_key(plan));
match self.over(asking, std::slice::from_ref(&key)) {
Some((which, committed, ceiling)) => {
Err(self.refuse_estate(&plan.reference, which, asking, committed, ceiling))
}
None => Ok(()),
}
}
/// **What `asking` more a month falls short by**, or `None` when it fits. PURE.
/// `mine`: reservation keys of the asker's own, not counted against it.
pub fn cap_short(&self, asking: Price, mine: &[String]) -> Result<Option<Short>, CloudError> {
if asking.currency != self.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this is priced in {:?}; there is no conversion here",
self.ceiling.currency, asking.currency
)));
}
let money = |minor: u64| monetize_pay::Money {
minor,
currency: self.ceiling.currency,
};
Ok(self
.over(asking, mine)
.map(|(which, committed, ceiling)| Short {
which,
needs: money(asking.minor_per_month),
cap: money(ceiling),
committed: money(committed),
}))
}
/// The month's committed spend credited back for what `references` laid (and every
/// relay of their boxes, `<reference>#<role>`) in this month: a give-back that went
/// through returns the month it laid.
fn credit_laid(&self, references: &[String]) -> Result<(), CloudError> {
let now = (self.month)();
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
let mut credited = false;
let keys: Vec<String> = spent
.laid
.keys()
.filter(|k| {
references.iter().any(|r| {
*k == r
|| k.strip_prefix(r.as_str())
.is_some_and(|rest| rest.starts_with('#'))
})
})
.cloned()
.collect();
for k in keys {
if let Some((month, cost)) = spent.laid.remove(&k) {
if month == now && spent.month == now {
spent.month_minor = spent.month_minor.saturating_sub(cost);
spent.total_minor = spent.total_minor.saturating_sub(cost);
credited = true;
}
}
}
if credited {
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)?;
}
Ok(())
}
/// One retired relay's lay credited back and its reservation dropped.
fn credit_retired(&self, reference: &str, role: &str) -> Result<(), CloudError> {
let key = crate::lay::relay_key(reference, role);
let now = (self.month)();
let mut spent = self.spent.lock().unwrap_or_else(|e| e.into_inner());
let mut moved = spent.reserved.remove(&format!("lay:{key}")).is_some();
if let Some((month, cost)) = spent.laid.remove(&key) {
moved = true;
if month == now && spent.month == now {
spent.month_minor = spent.month_minor.saturating_sub(cost);
spent.total_minor = spent.total_minor.saturating_sub(cost);
}
}
if moved {
let snapshot = spent.clone();
drop(spent);
self.store(&snapshot)?;
}
Ok(())
}
}
impl<C: Cloud> SpendCap<C> {
/// Wrap `inner`, keeping the month's total at `path`.
pub fn new(inner: C, ceiling: Ceiling, path: impl Into<PathBuf>) -> Result<Self, CloudError> {
Ok(SpendCap {
inner,
cap: Arc::new(Cap::new(ceiling, path)?),
})
}
/// A fixed month, so a test can cross the boundary without waiting a month.
/// Before the ledger is shared ([`SpendCap::cap`]).
pub fn with_month(mut self, month: impl Fn() -> String + Send + Sync + 'static) -> Self {
match Arc::get_mut(&mut self.cap) {
Some(c) => c.month = Box::new(month),
None => panic!("SpendCap::with_month after its ledger was shared"),
}
self
}
/// The ledger, for the host that holds this cloud.
pub fn cap(&self) -> Arc<Cap> {
Arc::clone(&self.cap)
}
pub fn inner(&self) -> &C {
&self.inner
}
pub fn spent_this_month(&self) -> u64 {
self.cap.spent_this_month()
}
pub fn spent_total(&self) -> u64 {
self.cap.spent_total()
}
pub fn reconcile_total(&self, authoritative_total_minor: u64) -> Result<(), CloudError> {
self.cap.reconcile_total(authoritative_total_minor)
}
pub fn ceiling(&self) -> Ceiling {
self.cap.ceiling()
}
pub fn remaining(&self) -> u64 {
self.cap.remaining()
}
pub fn reserved_minor(&self) -> u64 {
self.cap.reserved_minor()
}
/// **Would laying `plan` fit under the ceiling now?** Priced by the inner cloud. PURE.
pub fn can_lay(&self, plan: &EstatePlan) -> Result<(), CloudError> {
self.cap
.can_lay(plan, self.inner.price(&Priced::Plan(plan))?)
}
/// What `asking` more a month falls short by, or `None` when it fits. PURE.
pub fn cap_short(&self, asking: Price, mine: &[String]) -> Result<Option<Short>, CloudError> {
self.cap.cap_short(asking, mine)
}
/// **Would buying every resource in `delta` stay under the ceiling?** The
/// one check `ensure` runs before it buys and `can_serve` runs before an
/// operator confirms, so the quote and the purchase cannot disagree about
/// the money. Priced as a whole: a delta of two disks that each fit and
/// together do not is refused here, not after the first was bought.
fn admit(&self, delta: &[Resource]) -> Result<(), CloudError> {
let mut asking = Price {
minor_per_month: 0,
currency: self.cap.ceiling.currency,
};
for resource in delta {
let price = self.inner.price(&Priced::Resource(resource))?;
if price.currency != self.cap.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this resource is priced in {:?}; there is no conversion here, \
and guessing one inside a spending guard would make the cap a fiction",
self.cap.ceiling.currency, price.currency
)));
}
asking.minor_per_month = asking.minor_per_month.saturating_add(price.minor_per_month);
}
self.cap.admit_priced(asking)
}
/// The one body behind `Change::Ensure` and
/// `Change::Ensure` on a cap: `Some(Room::Buy)` and `None`
/// are both admitted against the ceiling (a room-less ask MAY buy — the
/// provider decides), and the inner call forwards exactly what was asked —
/// `None` stays room-less so a provider that refuses a guess (UpCloud, a
/// disk) gets to. A provider that refuses or fails releases the
/// reservation.
fn ensure_room(
&self,
reference: &str,
resource: &Resource,
room: Option<Room>,
) -> Result<Ensured, CloudError> {
let now = (self.cap.month)();
if let Some(room) = room.filter(|r| *r != Room::Buy) {
// Nothing is bought, so nothing is admitted and nothing is held.
let outcome = crate::ask::ensure(&self.inner, reference, resource, Some(room))?;
return self.cap.account(now, outcome);
}
// Priced OUTSIDE the lock: pricing may reach the provider, and holding
// the spend lock across a network call would serialise every buyer in
// the deployment behind the slowest quote.
let mut asking = Price {
minor_per_month: 0,
currency: self.cap.ceiling.currency,
};
let price = self.inner.price(&Priced::Resource(resource))?;
if price.currency != self.cap.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this resource is priced in {:?}; there is no conversion here, \
and guessing one inside a spending guard would make the cap a fiction",
self.cap.ceiling.currency, price.currency
)));
}
asking.minor_per_month = price.minor_per_month;
let key = format!("ensure:{reference}");
self.cap.admit_and_reserve(&key, asking)?;
let answer = match room {
Some(r) => crate::ask::ensure(&self.inner, reference, resource, Some(r)),
None => crate::ask::ensure(&self.inner, reference, resource, None),
};
match answer {
Ok(outcome) => self.cap.settle_ensure(&key, now, outcome),
Err(e) => {
// Best effort: the reservation is durable, so a failure to
// release it is reported by the ORIGINAL error rather than
// replaced by a bookkeeping one.
let _ = self.cap.unreserve(&key);
Err(e)
}
}
}
/// A growth admitted under the ceiling (T7), grown on the everyday lane or,
/// for a SOLD estate, the provider's estate lane.
fn grow_admitted(
&self,
set: &crate::grow::DataSet,
journal: &crate::grow::Journal,
estate: bool,
) -> Result<crate::grow::GrowOutcome, CloudError> {
let asking = self.inner.price(&Priced::Grow(set))?;
if asking.currency != self.cap.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this growth is priced in {:?}; there is no conversion here",
self.cap.ceiling.currency, asking.currency
)));
}
let key = set.resource_key();
// A reservation this set already holds (a resume in progress, a
// retry) is replaced, not added: it is the same growth — admitted and
// reserved in one hold of the lock (A27).
if let Err((which, committed, ceiling)) = self.cap.admit_reserving(&key, asking)? {
return Err(self.cap.refuse(which, asking, committed, ceiling));
}
match crate::ask::grow(&self.inner, set, estate, journal) {
Ok(outcome) => {
self.cap.settle_grow(&key, &outcome)?;
Ok(outcome)
}
Err(refused) if refused.is_refusal() => {
self.cap.unreserve(&key)?;
Err(refused)
}
Err(other) => Err(other),
}
}
}
impl<C: Cloud> Cloud for SpendCap<C> {
fn manifest(&self) -> CloudManifest {
self.inner.manifest()
}
fn catalog(&self, zone: &Zone) -> Result<Catalog, CloudError> {
self.inner.catalog(zone)
}
fn price(&self, what: &Priced<'_>) -> Result<Price, CloudError> {
self.inner.price(what)
}
/// **Admitted and reserved in one hold of the lock, committed once laid** (the
/// estate's whole monthly price); a refusal gives the reservation back.
fn lay(&self, plan: &EstatePlan, journal: &Journal) -> Result<LaidEstate, CloudError> {
let asking = self.inner.price(&Priced::Plan(plan))?;
if asking.currency != self.cap.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this estate is priced in {:?}; there is no conversion here",
self.cap.ceiling.currency, asking.currency
)));
}
let key = format!("lay:{}", crate::lay::spend_key(plan));
if let Err((which, committed, ceiling)) = self.cap.admit_reserving(&key, asking)? {
return Err(self
.cap
.refuse_estate(&plan.reference, which, asking, committed, ceiling));
}
match self.inner.lay(plan, journal) {
Ok(laid) => {
self.cap.settle_lay(&key, asking)?;
Ok(laid)
}
Err(refused) if refused.is_refusal() => {
self.cap.unreserve(&key)?;
Err(refused)
}
Err(other) => Err(other),
}
}
fn change(&self, change: &Change, journal: &Journal) -> Result<Changed, CloudError> {
match change {
// A plan change that costs more is admitted and reserved, committed once done.
Change::Plan { server, delta, .. } if delta.minor_per_month > 0 => {
if delta.currency != self.cap.ceiling.currency {
return Err(CloudError::Refused(format!(
"the spend cap is set in {:?} but this plan change is priced in {:?}; there is no conversion here",
self.cap.ceiling.currency, delta.currency
)));
}
let key = format!("plan:{}", server.id);
if let Err((which, committed, ceiling)) = self.cap.admit_reserving(&key, *delta)? {
return Err(self.cap.refuse(which, *delta, committed, ceiling));
}
match self.inner.change(change, journal) {
Ok(done) => {
self.cap.settle_lay(&key, *delta)?;
Ok(done)
}
Err(refused) if refused.is_refusal() => {
self.cap.unreserve(&key)?;
Err(refused)
}
Err(other) => Err(other),
}
}
// A growth: the delta reserved before the first member is touched (T7).
Change::Grow { set, estate } => self
.grow_admitted(set, journal, *estate)
.map(Changed::Grown),
// A settle commits what actually grew.
Change::Settle { set } => {
let out = self.inner.change(change, journal)?;
if let Changed::Grown(outcome) = &out {
self.cap.settle_grow(&set.resource_key(), outcome)?;
}
Ok(out)
}
Change::Ensure {
reference,
resource,
room,
} => self
.ensure_room(reference, resource, *room)
.map(Changed::Ensured),
// A retired relay gives its month back.
Change::Retire { reference, machine } => {
let out = self.inner.change(change, journal)?;
self.cap.credit_retired(reference, &machine.role)?;
Ok(out)
}
// Labels cost nothing: never capped (MAP-SEG 4).
Change::Plan { .. } | Change::ForgetMedia { .. } | Change::LabelAdopted { .. } => {
self.inner.change(change, journal)
}
}
}
fn boot(&self, boot: &Boot, journal: &Journal) -> Result<Booted, CloudError> {
self.inner.boot(boot, journal)
}
fn firewall(&self, laid: &LaidEstate) -> Result<(), CloudError> {
self.inner.firewall(laid)
}
/// Never capped. A whole give-back that went through returns the month its lays cost.
fn destroy(&self, what: &Give<'_>) -> Result<Reclaimed, CloudError> {
let out = self.inner.destroy(what)?;
if let Give::References(references) = what {
if out.failures.is_empty() {
self.cap.credit_laid(references)?;
}
}
Ok(out)
}
/// The room it serves within: the inner cloud's, and a purchase that BUYS must also
/// fit under the cap. Served from the box (`OnHand`) is never refused by a cap.
fn inventory(&self, what: &Listing<'_>) -> Result<Listed, CloudError> {
let answer = self.inner.inventory(what)?;
if let Listing::Admits(plan) = what {
self.can_lay(plan)?;
}
match (what, answer) {
(Listing::Room { delta, room }, Listed::Room(mut r))
if !matches!(room, Some(Room::OnHand)) =>
{
if r.serve.is_ok() {
r.serve = self.admit(delta).map_err(|e| e.to_string());
}
Ok(Listed::Room(r))
}
(_, other) => Ok(other),
}
}
}
/// **A cloud registered as one that cannot spend** (the box's own iron): it runs without
/// a spend ceiling on the strength of that promise, so the promise is enforced here — a
/// `Provisioned` answer is given back and refused, a lay or a growth that costs is refused.
pub struct CannotSpend<C> {
inner: C,
}
impl<C: std::fmt::Debug> std::fmt::Debug for CannotSpend<C> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CannotSpend")
.field("inner", &self.inner)
.finish()
}
}
impl<C: Cloud> CannotSpend<C> {
pub fn new(inner: C) -> Self {
CannotSpend { inner }
}
pub fn inner(&self) -> &C {
&self.inner
}
fn cannot(&self, what: &str, price: Price) -> CloudError {
CloudError::Refused(format!(
"{} is registered as a provider that cannot spend, and {what} would cost {} a month; nothing was touched",
self.inner.manifest().provider,
monetize_pay::Money { minor: price.minor_per_month, currency: price.currency }.in_words()
))
}
fn hold_to_the_promise(
&self,
reference: &str,
outcome: Ensured,
) -> Result<Ensured, CloudError> {
match outcome {
Ensured::Provisioned { id, cost } => {
// Give it back first, then refuse: a byte bought by a provider
// that promised not to buy must not outlive this call.
let returned = match crate::ask::destroy_one(&self.inner, &id) {
Ok(()) => "destroyed it",
Err(_) => "and could NOT destroy it — look at the provider's account",
};
Err(CloudError::Refused(format!(
"{} is registered as a provider that cannot spend, and it answered Provisioned ({}:{} at {} a month) for {reference}; \
{returned}. It runs without a spend ceiling on the strength of that promise, so the promise is enforced here",
self.inner.manifest().provider,
id.provider,
id.id,
monetize_pay::Money { minor: cost.minor_per_month, currency: cost.currency }.in_words()
)))
}
on_hand => Ok(on_hand),
}
}
}
impl<C: Cloud> Cloud for CannotSpend<C> {
fn manifest(&self) -> CloudManifest {
self.inner.manifest()
}
fn catalog(&self, zone: &Zone) -> Result<Catalog, CloudError> {
self.inner.catalog(zone)
}
fn price(&self, what: &Priced<'_>) -> Result<Price, CloudError> {
self.inner.price(what)
}
/// A lay buys: a cloud that cannot spend lays nothing.
fn lay(&self, plan: &EstatePlan, _journal: &Journal) -> Result<LaidEstate, CloudError> {
let _ = plan;
Err(crate::lay::unsupported(&self.inner.manifest().provider))
}
fn change(&self, change: &Change, journal: &Journal) -> Result<Changed, CloudError> {
match change {
Change::Ensure { reference, .. } => match self.inner.change(change, journal)? {
Changed::Ensured(outcome) => self
.hold_to_the_promise(reference, outcome)
.map(Changed::Ensured),
other => Err(crate::cloud::wrong_answer("ensure", &other)),
},
Change::Grow { set, .. } => {
let price = self.inner.price(&Priced::Grow(set))?;
if price.minor_per_month > 0 {
return Err(self.cannot("growing this set", price));
}
self.inner.change(change, journal)
}
Change::Plan { delta, .. } if delta.minor_per_month > 0 => {
Err(self.cannot("this plan change", *delta))
}
_ => self.inner.change(change, journal),
}
}
fn boot(&self, boot: &Boot, journal: &Journal) -> Result<Booted, CloudError> {
self.inner.boot(boot, journal)
}
fn firewall(&self, laid: &LaidEstate) -> Result<(), CloudError> {
self.inner.firewall(laid)
}
fn destroy(&self, what: &Give<'_>) -> Result<Reclaimed, CloudError> {
self.inner.destroy(what)
}
fn inventory(&self, what: &Listing<'_>) -> Result<Listed, CloudError> {
self.inner.inventory(what)
}
}
#[cfg(test)]
mod tests;