Your browser sucks, JavaScript disabled! Gunnar's console needs JavaScript and WebAssembly. Git over SSH on port 2222 does not.

nordisk/monetize

crates/monetize-admin/tests/plan_changes.rs

//! ★ **Plan changes that take machines away, grow a standing machine's disk and add a twin, on
//! the ONE box's own program** (monetize v2.0 A4-FACTS; feature `trials`).
//!
//! The ONE box here is [`monetize_admin::linked()`] exactly, its state opened from the
//! `monetize.toml` its intake writes out of a feed, with the SIGNED guests: Stripe
//! (`vendor.stripe.gunnar.*`) against `fake_stripe_serve` as a PROCESS (its control door is the
//! buyer), UpCloud (`cloud.upcloud.*`) against `mock-upcloud` in process. The estate sold is the
//! flock ([`monetize_server::testing::flock`]): a parametric TEST product of Gunnar's shape (front,
//! main, a twin that is optional) with `n` nodes beside it, each with a data disk; its rules are
//! data in its facts. The installer is the one stand-in (real boxes do not boot here); the boxes
//! are [`FlockBox`]es that sign their Fleet calls with their own keys, pin them inside the install's
//! window and pull, verify and ack their signed orders.
//!
//! | case | the change | at UpCloud (the mock) | at Stripe (the fake) |
//! |---|---|---|---|
//! | the twin | `{twin: false}` → `{twin: true}` | the twin's server and disks laid beside front and main | the difference held, paid, captured |
//! | fewer | `{nodes: 2}` → `{nodes: 1}` | node-2's server and disks given back | nothing held: cheaper, the renewal bills one node less |
//! | disks | `{node_gib: 20}` → `{node_gib: 40}` | each node's data disk 40 GiB at its own address | the difference held, paid, captured |
//!
//! Inputs as the other trials (`MONETIZE_TRIAL_GUESTS`, `MONETIZE_FAKE_STRIPE`); with
//! `MONETIZE_TRIAL_OUT=<dir>` each case writes `plan-change-<case>.json` there, a row of the
//! trials' receipt. MOCK ONLY: test keys, loopback everything, `*@mock.invalid`.

#![cfg(feature = "trials")]

mod common;

use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use common::{b64 as b64_bytes, inputs, sign_into, FakeStripe, StandIn};
use ed25519_dalek::{Signer as _, SigningKey, VerifyingKey};
use mock_upcloud::estate::Estate;
use mock_upcloud::{Clock, Faults, Mock};
use monetize_embed::fleet::{
    call, verify_order, FleetCall, OrderAck, OrderRefusal, OrdersAfter, Report,
};
use monetize_proto as pb;
use monetize_server::config::Config;
use monetize_server::custody::Custody;
use monetize_server::estate_order::{
    self, change, phase, EstateOrderConfig, EstateOrders, EstateRun, FileDns,
};
use monetize_server::state::AppState;
use monetize_server::testing::flock;
use monetize_server::testing::TestServer;

const SK: &str = "sk_test_PLANchangeNEVERlive";
const WHSEC: &str = "whsec_test_PLANchange";
const BEARER: &str = "plan-change-bearer-of-sufficient-length";
const ZONE: &str = "se-sto1";
const SAAS: &str = "gunnar.rs";
const COMPANY: &str = "acme1";

/// One ONE box is built at a time: `linked()` reads its data root from the environment.
static BUILDING: Mutex<()> = Mutex::new(());

/// The case's verdict where `cargo xtask trials` gathers it (`plan-change-<case>.json`): a case
/// that panicked is a FAIL row.
struct Timed(&'static str, Instant);

impl Drop for Timed {
    fn drop(&mut self) {
        use std::io::Write as _;
        let ms = self.1.elapsed().as_millis();
        let _ = writeln!(std::io::stderr(), "PLAN-CHANGE {ms:>6} ms  {}", self.0);
        if let (Some(dir), Some(case)) = (
            std::env::var_os("MONETIZE_TRIAL_OUT"),
            std::thread::current()
                .name()
                .map(|n| n.rsplit("::").next().unwrap_or(n).to_owned()),
        ) {
            let dir = PathBuf::from(dir);
            let _ = std::fs::create_dir_all(&dir);
            let pass = !std::thread::panicking();
            let detail = if pass {
                self.0.to_string()
            } else {
                format!(
                    "{}: the case failed (its panic is in the test's output)",
                    self.0
                )
            };
            let v = serde_json::json!({"name": format!("plan-change-{case}"), "trial": format!("plan-change/{case}"), "site": "gunnar", "where": "linked", "pass": pass, "detail": detail, "secs": (ms as f64 / 100.0).round() / 10.0});
            let _ = std::fs::write(
                dir.join(format!("plan-change-{case}.json")),
                serde_json::to_vec_pretty(&v).unwrap(),
            );
        }
    }
}

// ─── the estate side: a flock box on the pull channel ──────────────────────────────────────

/// **One flock box**: its own fleet key (pinned on its first call inside the install's window),
/// its health words, and the orders it pulled — each through `verify_order` — and acked.
struct FlockBox {
    role: String,
    key: SigningKey,
    monetize: VerifyingKey,
    pinned: bool,
    last_seq: u64,
    acks: Vec<OrderAck>,
    /// The kinds it ran.
    ran: Vec<String>,
}

impl FlockBox {
    fn new(role: &str, monetize: [u8; 32]) -> FlockBox {
        let mut seed = [7u8; 32];
        for (i, b) in role.bytes().enumerate() {
            seed[i % 32] ^= b;
        }
        FlockBox {
            role: role.into(),
            key: SigningKey::from_bytes(&seed),
            monetize: VerifyingKey::from_bytes(&monetize).unwrap(),
            pinned: false,
            last_seq: 0,
            acks: Vec::new(),
            ran: Vec::new(),
        }
    }

    fn health(&self) -> BTreeMap<String, String> {
        let words: &[(&str, &str)] = match self.role.as_str() {
            "main" => &[("serving", "yes"), ("data", "ok"), ("twin", "paired")],
            "twin" => &[("serving", "no"), ("data", "ok"), ("primary", "live")],
            _ => &[("serving", "yes"), ("data", "ok")],
        };
        words
            .iter()
            .map(|(k, v)| (k.to_string(), v.to_string()))
            .collect()
    }

    fn signed(&self, which: &str, body: String, at: u64, pin: bool) -> pb::FleetSigned {
        let c = FleetCall {
            product: "gunnar".into(),
            instance: COMPANY.into(),
            role: self.role.clone(),
            call: which.into(),
            body,
            at_unix_ms: at,
        };
        let signature = self
            .key
            .sign(&monetize_embed::fleet::fleet_call_message(&c))
            .to_bytes()
            .to_vec();
        pb::FleetSigned {
            product: c.product,
            instance: c.instance,
            role: c.role,
            call: c.call,
            body: c.body,
            at_unix_ms: c.at_unix_ms,
            signature,
            box_key: if pin {
                self.key.verifying_key().to_bytes().to_vec()
            } else {
                Vec::new()
            },
        }
    }

    fn report_call(&mut self, at: u64) -> pb::FleetSigned {
        let r = Report {
            build: "flock-trial".into(),
            health: self.health(),
            acked: std::mem::take(&mut self.acks),
            twin_review: None,
            product: None,
            measured_unix_ms: at,
        };
        let pin = !self.pinned;
        self.signed(call::REPORT, serde_json::to_string(&r).unwrap(), at, pin)
    }

    fn orders_call(&self, at: u64) -> pb::FleetSigned {
        self.signed(
            call::ORDERS,
            serde_json::to_string(&OrdersAfter {
                after: self.last_seq,
                wait_ms: 0,
            })
            .unwrap(),
            at,
            false,
        )
    }

    /// **The box runs what it pulled**: an order that does not verify is dropped; the flock's
    /// drain empties the box (its data moved to the boxes that stay) and is acked ok.
    fn run(&mut self, mut orders: Vec<pb::FleetOrder>, now: u64) {
        orders.sort_by_key(|o| o.seq);
        for o in orders {
            let so = monetize_server::services::fleet::order_of(&o);
            let (ok, detail) = match verify_order(
                &so,
                &self.monetize,
                ("gunnar", COMPANY, &self.role),
                now,
                self.last_seq,
            ) {
                Err(OrderRefusal::Signature) | Err(OrderRefusal::Replayed { .. }) => continue,
                Err(e) => (false, e.to_string()),
                Ok(()) if o.kind == flock::DRAIN => (
                    true,
                    format!("{} emptied: its data is on the boxes that stay", self.role),
                ),
                Ok(()) => (true, format!("{} ran {}", self.role, o.kind)),
            };
            self.last_seq = self.last_seq.max(o.seq);
            if ok {
                self.ran.push(o.kind.clone());
            }
            self.acks.push(OrderAck {
                seq: o.seq,
                ok,
                detail,
                receipt_json: String::new(),
            });
        }
    }
}

// ─── the feed and the world ─────────────────────────────────────────────────────────────

/// **The feed**: the SIGNED Stripe guest for gunnar's account at the fake, the SIGNED UpCloud
/// guest at the mock, the SIGNED Gunnar guest pulled from gunnar.rs (the shop's product, whose
/// keys the flock's boxes are signed under), and the operator's fragment.
fn feed(dir: &std::path::Path, stripe: &str, upcloud: u16) -> BTreeMap<String, String> {
    use guest_host::verify::sign::dev;
    let inp = inputs();
    let (ca_key, ca_der) = dev::mint_ca("plan-change-plugin-root").unwrap();
    let signer = dev::new_ed25519_signer(&ca_key, &ca_der, "plan-change-signing").unwrap();
    let signed =
        |name: &str| sign_into(dir, &inp.guests.join(format!("{name}.wasm")), name, &signer);
    let mut f = BTreeMap::new();
    f.insert("plugin_root.plan-change".to_string(), b64_bytes(&ca_der));
    let (sw, sp) = signed("stripe");
    let (key, hook) = (
        "MONETIZE_STRIPE_GUNNAR_SECRET_KEY",
        "MONETIZE_STRIPE_GUNNAR_WEBHOOK_SECRET",
    );
    f.insert("vendor.stripe.gunnar.wasm".into(), b64_bytes(&sw));
    f.insert("vendor.stripe.gunnar.p7s".into(), b64_bytes(&sp));
    f.insert(
        "vendor.stripe.gunnar.toml".into(),
        format!(
            "enabled = true\ncountry = \"IE\"\ngrants = [\"http\", \"hmac\", \"state\"]\nhttp_hosts = [\"{stripe}\"]\nhmac_secrets = [\"{hook}\"]\nbase_url = \"{stripe}\"\nsuccess_url = \"https://gunnar.test/paid/{{reference}}\"\ncancel_url = \"https://gunnar.test/cancel/{{reference}}\"\nlivemode = false\ntax_code = \"txcd_10103001\"\nwebhook_secret = \"{hook}\"\n[[http_auth]]\nhost = \"{stripe}\"\nprefix = \"Bearer \"\nsecret = \"{key}\"\n"
        ),
    );
    f.insert(format!("secret.{key}"), SK.into());
    f.insert(format!("secret.{hook}"), WHSEC.into());
    let up = format!("http://127.0.0.1:{upcloud}");
    let (uw, upp) = signed("upcloud");
    f.insert("cloud.upcloud.wasm".into(), b64_bytes(&uw));
    f.insert("cloud.upcloud.p7s".into(), b64_bytes(&upp));
    f.insert(
        "cloud.upcloud.toml".into(),
        format!("enabled = true\ncap = {{ monthly_minor = 10000000, currency = \"Eur\" }}\ncountry = \"FI\"\nzone = \"{ZONE}\"\nallow_grow = true\ngrants = [\"http\", \"state\"]\nhttp_hosts = [\"{up}\"]\napi_base = \"{up}\"\n[[http_auth]]\nhost = \"{up}\"\nprefix = \"Bearer \"\nsecret = \"upcloud_token\"\n"),
    );
    f.insert(
        "secret.upcloud_token".into(),
        "ucat_mock_plan_change_not_a_real_token".into(),
    );
    let (gw, gp) = signed("gunnar");
    f.insert("plugin.gunnar.wasm".into(), b64_bytes(&gw));
    f.insert("plugin.gunnar.p7s".into(), b64_bytes(&gp));
    let main_key = SigningKey::from_bytes(&[21; 32]).verifying_key().to_bytes();
    f.insert("plugin.gunnar.toml".into(), format!("vendor = \"stripe/gunnar\"\ninstance = \"{SAAS}\"\ngrants = []\nfleet_keys = {{ main = \"{}\" }}\n", monetize_embed::signing::base64_encode(&main_key)));
    f.insert(
        "plugin.gunnar.prices".into(),
        "[SE]\ndisk_gib_month = 2200\n".into(),
    );
    f.insert("monetize.toml".into(), "\n[signups.gunnar.plans.\"gunnar/account\"]\namount_minor = 0\ncurrency = \"Sek\"\ncap_bytes = 0\nperiod_days = 0\n".into());
    f
}

/// Fields drop in order: the server, then the state (its UpCloud guest owns a blocking client),
/// then the runtime — all outside any async context.
struct World {
    server: Option<TestServer>,
    state: Arc<AppState>,
    fake: FakeStripe,
    mock: Arc<Mock>,
    channel: tonic::transport::Channel,
    boxes: BTreeMap<String, FlockBox>,
    rt: tokio::runtime::Runtime,
    dir: PathBuf,
}

impl World {
    fn new(name: &str) -> World {
        let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR"))
            .join(format!("plan-change-{name}-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(dir.join("guests")).unwrap();
        let rt = tokio::runtime::Builder::new_multi_thread()
            .worker_threads(4)
            .enable_all()
            .build()
            .unwrap();
        let fake = FakeStripe::start(&inputs().fake_stripe, SK, WHSEC);
        let mock = Mock::recording(Estate::new(Clock::new(1_000_000), Faults::none(), 11));
        let port = rt
            .block_on(mock_upcloud::serve(mock.clone(), "127.0.0.1:0"))
            .unwrap()
            .0;
        let root = dir.join("data");
        let state = {
            let _one = BUILDING.lock().unwrap_or_else(|e| e.into_inner());
            monetize_front::intake::prepare(&root).unwrap();
            let refused = monetize_front::intake::apply_pull(
                &root,
                &feed(&dir.join("guests"), &fake.base(), port),
            )
            .unwrap();
            assert!(
                refused.is_empty(),
                "the ONE box refused part of the feed: {refused:#?}"
            );
            let path = root.join(monetize_front::intake::CONFIG_REL);
            let cfg = Config::load(&path).unwrap_or_else(|e| {
                panic!(
                    "{}: {e}\n{}",
                    path.display(),
                    std::fs::read_to_string(&path).unwrap_or_default()
                )
            });
            std::env::set_var(monetize_admin::DATA_ENV, &root);
            let linked = monetize_admin::linked();
            let mut state = monetize_server::cli::open_state(&cfg, &linked)
                .unwrap_or_else(|e| panic!("the ONE box does not open: {e:#}"));
            // The flock is a TEST product's lifecycle, registered as a guest's would be; its
            // estates are sold on gunnar's account (its keys, its Stripe instance).
            flock::install();
            let offer = EstateOrderConfig {
                product: "gunnar".into(),
                lifecycle: flock::KIND.into(),
                cloud: "upcloud".into(),
                vendor: "stripe/gunnar".into(),
                apex: "gunnar.rs".into(),
                currency: "EUR".into(),
                zone: ZONE.into(),
                shop_door: "https://gunnar.monetize.test".into(),
                shape: monetize_server::testing::lifecycles::gunnar_shape(),
                resize_notice_secs: 0,
                ..EstateOrderConfig::default()
            };
            state.estates = EstateOrders::with(
                offer,
                dir.join("estate-orders"),
                Arc::new(StandIn),
                Arc::new(FileDns(dir.join("dns.json"))),
                Custody::open(dir.join("custody")).unwrap(),
            );
            Arc::new(state)
        };
        let server = rt.block_on(TestServer::spawn(state.clone(), BEARER));
        let channel = rt
            .block_on(
                tonic::transport::Channel::from_shared(server.endpoint())
                    .unwrap()
                    .connect(),
            )
            .unwrap();
        World {
            server: Some(server),
            state,
            fake,
            mock,
            channel,
            boxes: BTreeMap::new(),
            rt,
            dir,
        }
    }

    fn run(&self) -> EstateRun {
        self.state.estates.get(COMPANY).expect("the run")
    }

    fn laid_roles(&self) -> Vec<String> {
        self.run()
            .laid
            .map(|l| flock::roles_of(&l))
            .unwrap_or_default()
    }

    /// The buyer pays `id` at Stripe's page (the fake's control door).
    fn pay(&self, id: &str) {
        let (st, ev) = self.fake.control("POST", &format!("/pay/{id}"));
        assert_eq!(st, 200, "the fake Stripe's page refused to pay {id}: {ev}");
    }

    /// The vendor's own word on a hold, through the signed Stripe guest.
    fn at_stripe(&self, handle: &monetize_pay::ChargeHandle) -> monetize_pay::ChargeStatus {
        self.state
            .registry
            .find_vendor("stripe/gunnar")
            .expect("the Stripe guest")
            .charge(handle)
            .expect("the fake Stripe answers")
    }

    /// **Every box of the laid estate calls home**: a box whose pin window is open pins its key
    /// on its first call; each pulls and runs its orders, its acks riding the next report. A box
    /// of a role no longer laid is dropped once the guardian refuses it.
    fn boxes_call_home(&mut self) {
        let pk = self.state.signer_for("gunnar").public_key();
        let roles = self.laid_roles();
        let store = self.state.fleet.guardian().unwrap().store.clone();
        for role in &roles {
            if !self.boxes.contains_key(role)
                && (store.pin_window(COMPANY, role) > 0 || store.key_of(COMPANY, role).is_some())
                && store.product_of(COMPANY).is_some()
            {
                self.boxes.insert(role.clone(), FlockBox::new(role, pk));
            }
        }
        let now = self.state.now();
        let mut gone = Vec::new();
        for (role, b) in self.boxes.iter_mut() {
            let rep = b.report_call(now);
            match self
                .rt
                .block_on(pb::fleet_client::FleetClient::new(self.channel.clone()).report(rep))
            {
                Ok(_) => b.pinned = true,
                Err(e) if !roles.contains(role) => {
                    let _ = e;
                    gone.push(role.clone());
                    continue;
                }
                Err(e) => panic!("{role} was not heard: {e}"),
            }
            if let Ok(o) = self.rt.block_on(
                pb::fleet_client::FleetClient::new(self.channel.clone()).orders(b.orders_call(now)),
            ) {
                b.run(o.into_inner().orders, now);
            }
        }
        for r in gone {
            self.boxes.remove(&r);
        }
    }

    /// **A flock ordered at `sizes`, paid at Stripe and delivered**: the hold at the fake, laid at
    /// the mock through the signed UpCloud guest, installed by the stand-in, the boxes heard, the
    /// hold captured.
    fn delivered(&mut self, sizes: &estate_order::EstateSizes) {
        let run = estate_order::start_for(
            &self.state,
            COMPANY,
            "olle@buyer.mock.invalid",
            "",
            "olle",
            sizes,
        )
        .expect("the order starts");
        self.pay(&run.handle.as_ref().unwrap().id);
        let began = Instant::now();
        loop {
            estate_order::drive(&self.state, COMPANY);
            let r = self.run();
            if r.phase == phase::CAPTURED {
                break;
            }
            assert!(
                phase::is_open(&r.phase),
                "{COMPANY} left the open phases at {}: {}",
                r.phase,
                r.why
            );
            assert!(
                began.elapsed() < Duration::from_secs(150),
                "{COMPANY} stuck at {}: {}",
                r.phase,
                r.why
            );
            self.boxes_call_home();
            std::thread::sleep(Duration::from_millis(20));
        }
        self.boxes_call_home();
    }

    /// The change driven to its end, the boxes calling home between ticks.
    fn change_driven(&mut self) -> change::PlanChange {
        let began = Instant::now();
        loop {
            self.boxes_call_home();
            estate_order::drive(&self.state, COMPANY);
            let r = self.run();
            match r.change.as_ref() {
                None => return r.changes.last().cloned().expect("the change is kept"),
                Some(c) => assert!(
                    began.elapsed() < Duration::from_secs(150),
                    "the change is stuck at {}: {}",
                    c.status,
                    c.why
                ),
            }
            std::thread::sleep(Duration::from_millis(20));
        }
    }

    /// The mock's own record of a server (what the box is).
    fn server_at_mock(&self, uuid: &str) -> Option<mock_upcloud::estate::Server> {
        self.mock.estate.lock().unwrap().server(uuid).cloned()
    }
}

impl Drop for World {
    fn drop(&mut self) {
        if let Some(s) = self.server.take() {
            self.rt.block_on(s.stop());
        }
    }
}

// ─── the cases ───────────────────────────────────────────────────────────────────────────

/// ★ **A twin added to a Gunnar-shaped estate** (item 3): the flock ordered with no twin is laid
/// at UpCloud as a front and a main (the cloud's check reads the flock's shape: its twin is
/// optional); a plan change lays the twin there beside them, pairs it and lets it follow main,
/// the difference held, paid and captured at Stripe.
#[test]
fn a_twin_is_added_to_a_gunnar_shaped_estate_laid_at_upcloud_and_paid_at_stripe() {
    let _t = Timed(
        "twin: front+main laid at UpCloud, the twin added by a plan change, paid at Stripe",
        Instant::now(),
    );
    let mut w = World::new("twin");
    w.delivered(&flock::sizes(false, 0, 10));
    assert_eq!(
        w.laid_roles(),
        ["front", "main"],
        "a Gunnar-shaped estate with no twin, laid at the mock"
    );
    let before = w.run();
    let c = change::change(
        &w.state,
        COMPANY,
        &estate_order::EstateSizes {
            order: serde_json::json!({"twin": true}),
            ..Default::default()
        },
    )
    .unwrap_or_else(|e| panic!("the twin is added: {e:?}"));
    assert_eq!(
        c.adds.iter().map(|a| a.role.as_str()).collect::<Vec<_>>(),
        ["twin"]
    );
    let handle = c.handle.clone().expect("the difference is held at Stripe");
    w.pay(&handle.id);
    let done = w.change_driven();
    assert_eq!(
        done.status,
        change::status::DONE,
        "{done:#?}\n{:#?}",
        w.run().history
    );
    assert_eq!(w.laid_roles(), ["front", "main", "twin"]);
    let run = w.run();
    let twin = run.laid.as_ref().unwrap().machine("twin").cloned().unwrap();
    let at_mock = w
        .server_at_mock(&twin.server)
        .expect("the twin's server stands at UpCloud");
    assert_eq!(
        at_mock.plan, c.quote.twin_plan,
        "the twin follows main: the rule's plan"
    );
    let main_set: u64 = run
        .laid
        .as_ref()
        .unwrap()
        .machine("main")
        .unwrap()
        .disks
        .iter()
        .skip(1)
        .map(|d| d.gib)
        .sum();
    assert!(
        twin.disks.get(1).map(|d| d.gib).unwrap_or(0) >= main_set,
        "the twin's data disk holds main's whole set: {:?}",
        twin.disks
    );
    assert!(
        w.boxes.get("twin").is_some_and(|b| b.pinned),
        "the twin paired and called home on its own key"
    );
    assert!(
        matches!(
            w.at_stripe(&handle),
            monetize_pay::ChargeStatus::Captured { .. }
        ),
        "captured at Stripe"
    );
    assert_eq!(run.quote.twin_plan, c.quote.twin_plan);
    assert!(
        run.quote.monthly.minor > before.quote.monthly.minor,
        "the bill follows: the renewal prices the twin"
    );
}

/// ★ **A node taken away** (item 1): the flock's nodes 2 → 1 is cheaper, so nothing is held;
/// node-2 is sent its drain, acks it (emptied, its data on the node that stays), is given back
/// whole at UpCloud and read gone, the guardian forgets its key, no DNS record names it, and the
/// run's price is the new, lower one.
#[test]
fn a_node_taken_away_is_drained_and_given_back_at_upcloud_and_billed_no_more() {
    let _t = Timed(
        "fewer: node-2 drained, given back at UpCloud, forgotten, its DNS gone, billed less",
        Instant::now(),
    );
    let mut w = World::new("fewer");
    w.delivered(&flock::sizes(true, 2, 20));
    assert_eq!(w.laid_roles(), ["front", "main", "twin", "node", "node-2"]);
    let before = w.run();
    let node2 = before
        .laid
        .as_ref()
        .unwrap()
        .machine("node-2")
        .cloned()
        .unwrap();
    assert!(w.server_at_mock(&node2.server).is_some());
    let c = change::change(&w.state, COMPANY, &flock::sizes(true, 1, 20))
        .unwrap_or_else(|e| panic!("node-2 is taken away: {e:?}"));
    assert_eq!(
        c.removes
            .iter()
            .map(|x| x.role.as_str())
            .collect::<Vec<_>>(),
        ["node-2"]
    );
    assert!(
        c.handle.is_none() && c.amount.minor == 0,
        "cheaper: nothing is held at Stripe"
    );
    let done = w.change_driven();
    assert_eq!(
        done.status,
        change::status::DONE,
        "{done:#?}\n{:#?}",
        w.run().history
    );
    assert_eq!(w.laid_roles(), ["front", "main", "twin", "node"]);
    // node-2 ran its drain before it went (the box is gone from the world once refused).
    let run = w.run();
    assert!(
        run.history
            .iter()
            .any(|t| t.note.contains("node-2 emptied")),
        "{:#?}",
        run.history
    );
    let est = w.mock.estate.lock().unwrap();
    assert!(
        est.server(&node2.server)
            .is_none_or(|s| s.state == "deleted" || s.state == "maintenance"),
        "node-2's server is given back at UpCloud"
    );
    for d in &node2.disks {
        assert!(
            est.storage(&d.id).is_none(),
            "node-2's disk {} is given back at UpCloud",
            d.title
        );
    }
    drop(est);
    let g = w.state.fleet.guardian().unwrap();
    assert!(
        g.store.key_of(COMPANY, "node-2").is_none(),
        "the guardian forgets node-2's key"
    );
    let dns = monetize_server::estate_order::EstateDns::listed(&FileDns(w.dir.join("dns.json")))
        .expect("the recorded zone reads");
    assert!(
        dns.iter()
            .all(|r| r.value != node2.public_ipv4 && r.value != node2.utility_ipv4),
        "no record names node-2: {dns:?}"
    );
    assert!(
        run.quote.monthly.minor < before.quote.monthly.minor,
        "the bill follows: {} < {}",
        run.quote.monthly.minor,
        before.quote.monthly.minor
    );
    assert_eq!(run.quote.monthly, c.quote.monthly);
}

/// ★ **A standing machine's disk grows** (item 2): the flock's node_gib 20 → 40 grows node's and
/// node-2's data disks at UpCloud (each detached, resized and put back at its address on its
/// own server: what the box reads is 40 GiB), the difference held, paid and captured at Stripe;
/// a smaller number after is refused by name.
#[test]
fn a_standing_nodes_disk_grows_at_upcloud_paid_at_stripe_and_never_shrinks() {
    let _t = Timed("disks: node_gib 20 -> 40 grown at UpCloud at each disk's address, paid at Stripe, never smaller", Instant::now());
    let mut w = World::new("disks");
    w.delivered(&flock::sizes(true, 2, 20));
    let c = change::change(&w.state, COMPANY, &flock::sizes(true, 2, 40))
        .unwrap_or_else(|e| panic!("the standing disks grow: {e:?}"));
    assert_eq!(
        c.grows
            .iter()
            .map(|g| (g.role.as_str(), g.to_gib))
            .collect::<Vec<_>>(),
        [("node", 40), ("node-2", 40)]
    );
    let handle = c.handle.clone().expect("the difference is held at Stripe");
    w.pay(&handle.id);
    let done = w.change_driven();
    assert_eq!(
        done.status,
        change::status::DONE,
        "{done:#?}\n{:#?}",
        w.run().history
    );
    let run = w.run();
    for role in ["node", "node-2"] {
        let m = run.laid.as_ref().unwrap().machine(role).cloned().unwrap();
        assert_eq!(m.disks[1].gib, 40, "{role}: read back into the run");
        let s = w.server_at_mock(&m.server).expect("its server stands");
        let dev = s
            .devices
            .iter()
            .find(|d| d.storage == m.disks[1].id)
            .unwrap_or_else(|| {
                panic!(
                    "{role}'s data disk is attached to its own server again: {:?}",
                    s.devices
                )
            });
        assert_eq!(
            (dev.address.as_str(), dev.storage_size),
            (m.disks[1].address.as_str(), 40),
            "{role}: the box sees 40 GiB at its own address"
        );
    }
    assert!(
        matches!(
            w.at_stripe(&handle),
            monetize_pay::ChargeStatus::Captured { .. }
        ),
        "captured at Stripe"
    );
    assert_eq!(run.quote.monthly, c.quote.monthly, "the bill follows");
    let e = change::change(&w.state, COMPANY, &flock::sizes(true, 2, 30))
        .expect_err("data never shrinks");
    assert!(format!("{e:?}").contains("never shrinks"), "{e:?}");
}