crates/monetize-server/src/estate_order/tests/prod_shape.rs
//! ★ **PROD-FIX 1 and 3 (prod ONE box, 2026-10-26, app c4457331)**: a box of prod's shape —
//! several sites, Gunnar's and Holger's, each with an estate this box laid whose main is
//! silent — read the way prod read it.
//!
//! * PROD-FIX 3: the box read UpCloud's storages several times a second. Every 5 s guardian
//! tick listed the whole account once per silent main (UpCloud: `GET /storage` and one
//! `GET /storage/<uuid>` per storage). The provider is asked about a silent main at most once
//! per [`monetize_guardian::PROVIDER_LOOK_EVERY_MS`], as a slow jera job behind the tick.
//! * PROD-FIX 1: the console's OVERVIEW loaded nothing for the owner. Its calls are
//! `Guardian.Cases` for the list, then per row `Guardian.Cases` again, `Accounts.List`,
//! `Accounts.Rows`, `Ledger.Sales`, `Products.List` and `ReconcileAdmin.ListPaidAtVendor`;
//! each `Guardian.Cases` asked the provider about every silent main, BLOCKING, seconds each.
//! The overview's calls now answer within a bound and ask no provider.
use std::sync::atomic::Ordering as AtomicOrdering;
use std::time::{Duration, Instant};
use monetize_proto as pb;
use tonic::metadata::MetadataValue;
use super::*;
use crate::testing::TestServer;
const TOKEN: &str = "prod-shape-token";
/// The sites of prod's ONE box, by product: Gunnar's and Holger's, each with an estate laid.
const SITES: [(&str, &str, &str); 4] = [
("gunnar-rs", "gunnar", "gunnar.rs"),
("holger-rs", "holger", "holger.rs"),
("test2-gunnar-rs", "gunnar", "test2.gunnar.rs"),
("test2-holger-rs", "holger", "test2.holger.rs"),
];
/// One laid estate per site (`acme0` on gunnar-rs, …), the boxes heard once at the lay and then
/// silent: what a box with Holger mains that do not report looks like to the guardian.
fn prod_shape(name: &str) -> Rig {
let dir = tmp(name);
let pay = HoldPay::default();
let cloud = Arc::new(MockCloud::new());
let kit = FakeKit::new("PASS");
// The installer's verdict names each box's fleet key: the guardian knows the boxes.
for (role, seed) in [("front", 3u8), ("main", 1), ("twin", 2)] {
let key = ed25519_dalek::SigningKey::from_bytes(&[seed; 32])
.verifying_key()
.to_bytes();
kit.appliance_keys
.lock()
.unwrap()
.insert(role.into(), monetize_embed::signing::base64_encode(&key));
}
let mut reg = Registry::new()
.sells("gunnar", pay.clone())
.cloud(cloud.clone())
.shared_product(shop_product())
.home("gunnar", "gunnar-rs");
for (label, product, site) in SITES {
reg = reg.estate(
label,
crate::config::EstateConfig {
product: product.into(),
vendor: "holdpay".into(),
site: site.into(),
owner: "ignalina".into(),
operator_site: true,
..Default::default()
},
);
}
let (state, clock, mails, health) = build_in(
reg,
&dir,
EstateOrderConfig {
vendor: String::new(),
apex: String::new(),
lifecycle: "gunnar".into(),
..cfg()
},
kit.clone(),
None,
None,
);
state
.fleet
.link("gunnar", &crate::testing::guardian::GUARDIAN);
state
.fleet
.link("holger", &crate::testing::guardian::GUARDIAN);
let r = Rig {
state,
pay,
cloud,
kit,
dir,
clock,
mails,
health,
};
for (i, (label, product, _)) in SITES.iter().enumerate() {
let company = format!("acme{i}");
start_on(
&r.state,
Some(label),
&company,
"buyer@mock.invalid",
"",
"alice",
&EstateSizes::default(),
)
.unwrap_or_else(|e| panic!("{label}: {e:?}"));
r.pay.authorize(&handle_of(&r, &company));
drive(&r.state, &company);
// Its boxes call home once (the install's quick test), and the money is captured.
drive(&r.state, &company);
let run = r.state.estates.get(&company).unwrap();
assert_eq!(run.phase, phase::CAPTURED, "{label}: {:#?}", run.history);
assert_eq!(
r.state
.fleet
.guardian()
.unwrap()
.store
.product_of(run.fleet_name())
.as_deref(),
Some(*product),
"{label}'s estate is its product's"
);
}
// Every box silent from here on (no report): past the guardian's bound.
r.clock.fetch_add(10 * 60_000, Ordering::SeqCst);
reset(&r);
r
}
fn reset(r: &Rig) {
r.cloud.inventories.store(0, AtomicOrdering::SeqCst);
r.cloud.storage_gets.store(0, AtomicOrdering::SeqCst);
r.cloud.server_gets.store(0, AtomicOrdering::SeqCst);
}
fn listings(r: &Rig) -> usize {
r.cloud.inventories.load(AtomicOrdering::SeqCst)
}
/// The silent mains this box laid: each is one the provider may be asked about.
fn silent_mains(r: &Rig) -> usize {
r.state
.estates
.all()
.iter()
.filter(|run| {
run.laid
.as_ref()
.is_some_and(|l| l.machine("main").is_some())
})
.count()
}
/// ★ **PROD-FIX 3 RED**: a minute of guardian ticks (every 5 s, as `fleet::run`) lists the
/// provider's account a bounded number of times — at most twice per silent main (its look, and
/// the one look of a case opened on it), never once per main per tick.
#[test]
fn a_minute_of_guardian_ticks_reads_the_providers_storages_a_bounded_number_of_times() {
let r = prod_shape("prod-shape-ticks");
let mains = silent_mains(&r);
assert_eq!(mains, 4, "four sites, an estate laid on each");
for _ in 0..12 {
r.clock.fetch_add(5_000, Ordering::SeqCst);
crate::fleet::tick(&r.state);
}
let (n, gets) = (
listings(&r),
r.cloud.storage_gets.load(AtomicOrdering::SeqCst),
);
assert!(n <= 2 * mains, "a minute of ticks listed the account {n} times ({gets} storage GETs at UpCloud's rate) for {mains} silent mains: a polling loop");
// The next minute asks nothing more: the looks are minutes apart.
reset(&r);
for _ in 0..12 {
r.clock.fetch_add(5_000, Ordering::SeqCst);
crate::fleet::tick(&r.state);
}
assert_eq!(
listings(&r),
0,
"a look is not asked again within its interval"
);
}
/// The overview's calls, as monetize-ui-server makes them (`api::overview`, then
/// `api::estate_summary` for every row at once), each with the console's bearer.
async fn the_overviews_calls(endpoint: String) -> usize {
fn authed<T>(m: T) -> tonic::Request<T> {
let mut req = tonic::Request::new(m);
req.metadata_mut().insert(
"authorization",
MetadataValue::try_from(format!("Bearer {TOKEN}")).unwrap(),
);
req
}
let channel = tonic::transport::Endpoint::from_shared(endpoint)
.unwrap()
.connect()
.await
.unwrap();
let mut guardian = pb::guardian_client::GuardianClient::new(channel.clone());
let estates = guardian
.cases(authed(pb::GuardianCasesRequest {
instance: String::new(),
}))
.await
.expect("the overview's list")
.into_inner()
.estates;
let rows: Vec<_> = estates
.iter()
.map(|e| {
let (channel, name, product) = (channel.clone(), e.instance.clone(), e.product.clone());
tokio::spawn(async move {
let mut guardian = pb::guardian_client::GuardianClient::new(channel.clone());
let mut accounts = pb::accounts_client::AccountsClient::new(channel.clone());
let mut ledger = pb::ledger_client::LedgerClient::new(channel.clone());
let mut products = pb::products_client::ProductsClient::new(channel.clone());
let mut reconcile = pb::reconcile_admin_client::ReconcileAdminClient::new(channel);
let fleet = guardian
.cases(authed(pb::GuardianCasesRequest {
instance: String::new(),
}))
.await
.expect("the row's estate")
.into_inner();
assert!(fleet.estates.iter().any(|e| e.instance == name));
if let Ok(listed) = accounts
.list(authed(pb::ListAccountsRequest {
product: String::new(),
limit: 0,
}))
.await
{
for a in listed
.into_inner()
.accounts
.iter()
.filter(|a| a.instances > 0)
{
let _ = accounts
.rows(authed(pb::AccountRowsRequest {
product: a.product.clone(),
tenant: a.tenant.clone(),
}))
.await;
}
}
// As the console reads them: a part that cannot be read is `null` on its row.
let _ = ledger
.sales(authed(pb::SalesRequest { product, limit: 0 }))
.await;
let _ = products.list(authed(pb::ListProductsRequest {})).await;
let _ = reconcile
.list_paid_at_vendor(authed(pb::ListPaidAtVendorRequest { limit: 0 }))
.await;
})
})
.collect();
for row in rows {
row.await.expect("a row's calls");
}
estates.len()
}
/// ★ **PROD-FIX 1 RED**: the overview of a box of prod's shape answers within a bound and asks
/// the provider nothing, while one listing of the account takes 250 ms (prod's took 1.6 s to
/// 11 s).
#[test]
fn the_overview_of_a_prod_shaped_box_answers_within_a_bound_and_asks_no_provider() {
let r = prod_shape("prod-shape-overview");
r.cloud.inventory_ms.store(250, AtomicOrdering::SeqCst);
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.enable_all()
.build()
.unwrap();
let srv = rt.block_on(TestServer::spawn(r.state.clone(), TOKEN));
let t0 = Instant::now();
let rows = rt.block_on(the_overviews_calls(srv.endpoint()));
let took = t0.elapsed();
rt.block_on(srv.stop());
assert_eq!(rows, SITES.len(), "every estate is a row");
assert_eq!(
listings(&r),
0,
"the overview's calls listed the provider's account {} times",
listings(&r)
);
assert!(
took < Duration::from_secs(1),
"the overview took {took:?}: it must answer within a bound"
);
}
/// ★ **PROD-FIX 3, the server's own loop**: `fleet::tick_on` asks the due looks as ONE slow
/// jera job behind the tick — the tick itself lists nothing — and a minute of ticks asks each
/// silent main once.
#[test]
fn the_servers_tick_asks_the_provider_looks_as_a_jera_job_behind_it() {
let r = prod_shape("prod-shape-look-job");
let mains = silent_mains(&r);
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.unwrap();
let state = r.state.clone();
let clock = r.clock.clone();
rt.block_on(async move {
for _ in 0..12 {
clock.fetch_add(5_000, Ordering::SeqCst);
let s = state.clone();
tokio::task::spawn_blocking(move || crate::fleet::tick_on(&s))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(50)).await;
}
for _ in 0..200 {
if state
.fleet
.guardian()
.unwrap()
.looks_due(&crate::fleet::ShopWorld { state: &state }, state.now())
.is_empty()
{
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
});
let g = r.state.fleet.guardian().unwrap();
for i in 0..SITES.len() {
let look = g
.look_of(&format!("acme{i}"), "main")
.expect("the job looked");
assert_eq!(look.word, "active", "the provider's word is kept");
}
let n = listings(&r);
assert!(n >= mains && n <= 2 * mains, "each silent main looked at once in the minute (and once more for a case opened on it): {n}");
}