crates/monetize-ui-server/tests/sites_unreachable.rs
//! ★ **MHUNT-1 (OX-MONHUNT 2026-10-28): the Sites page answers within the page bound with
//! unreachable sites present.** `GET /api/sites` asked `EstateOrder.Status` of every ready
//! site, one after another, for its first-admin door (FIRST-LINK-1); monetize answers that
//! status by asking the site's own public front (the shop's sale check, 20 s patience per
//! connect). One site whose front does not answer held the whole page for its patience, two
//! for twice that: the page loaded for ever, as the Overview did before PROD-4. Each door is
//! now asked at once and within the page's bound (`bounded::PAGE_WITHIN`): past it the site
//! is listed without its door (or with its last one), the read goes on, and a second look
//! while it is out asks nothing new.
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::body::Body;
use axum::Router;
use http::{Request, StatusCode};
use http_body_util::BodyExt;
use monetize_proto as pb;
use monetize_proto::estate_order_server::{EstateOrder, EstateOrderServer};
use monetize_ui_server::app::App;
use monetize_ui_server::grpc::Monetize;
use monetize_ui_server::serve;
use monetize_ui_server::sse::EventHub;
use serde_json::Value;
use tonic::{Request as TReq, Response, Status};
use tower::ServiceExt;
const OPERATOR: &str = "the-unreachable-sites-operator";
const ADMIN: &str = "break-glass-unreachable-sites";
/// What the page may wait, all told: the page bound and a little.
const PAGE_WITHIN: Duration = Duration::from_millis(2_500);
/// The ONE monetize: three ready sites, two of whose fronts never answer the door's ask.
#[derive(Default)]
struct Held {
statuses: AtomicUsize,
}
#[derive(Clone)]
struct Fake(Arc<Held>);
fn not_here<T>() -> Result<Response<T>, Status> {
Err(Status::unimplemented("not in this fake"))
}
fn ready(estate: &str) -> pb::EstateSite {
pb::EstateSite { estate: estate.into(), domain: format!("{estate}.example"), vendor: format!("stripe/{estate}"), first_admin: "ops".into(), state: "ready".into(), words: "It is up.".into(), why_not: "it is already ordered.".into(), ..Default::default() }
}
#[tonic::async_trait]
impl EstateOrder for Fake {
async fn product_order(&self, _: tonic::Request<pb::EstateProductOrderRequest>) -> Result<tonic::Response<pb::EstateProductOrderResponse>, tonic::Status> { Err(tonic::Status::unimplemented("product_order")) }
async fn product_order_answer(&self, _: tonic::Request<pb::EstateProductOrderAnswerRequest>) -> Result<tonic::Response<pb::EstateProductOrderAnswerResponse>, tonic::Status> { Err(tonic::Status::unimplemented("product_order_answer")) }
async fn product_order_kinds(&self, _: tonic::Request<pb::EstateProductOrderKindsRequest>) -> Result<tonic::Response<pb::EstateProductOrderKindsResponse>, tonic::Status> { Err(tonic::Status::unimplemented("product_order_kinds")) }
async fn list_sites(&self, _: TReq<pb::EstateSitesRequest>) -> Result<Response<pb::EstateSitesResponse>, Status> {
Ok(Response::new(pb::EstateSitesResponse { sites: vec![ready("dark-one-rs"), ready("lit-rs"), ready("dark-two-rs")], sizes: vec!["Mini".into()] }))
}
async fn status(&self, req: TReq<pb::EstateOrderStatusRequest>) -> Result<Response<pb::EstateOrderStatusResponse>, Status> {
self.0.statuses.fetch_add(1, Ordering::SeqCst);
let company = req.into_inner().company;
if company.starts_with("dark-") {
// The site's front does not answer: monetize waits out the sale check's patience.
tokio::time::sleep(Duration::from_secs(600)).await;
}
Ok(Response::new(pb::EstateOrderStatusResponse { company, state: "ready".into(), first_admin_door: "open".into(), first_admin_door_words: "the door is open".into(), ..Default::default() }))
}
async fn update_estate(&self, _: TReq<pb::EstateUpdateRequest>) -> Result<Response<pb::EstateUpdateResponse>, Status> {
not_here()
}
async fn update_status(&self, _: TReq<pb::EstateUpdateStatusRequest>) -> Result<Response<pb::EstateUpdateResponse>, Status> {
not_here()
}
async fn cancel_update(&self, _: TReq<pb::EstateUpdateStatusRequest>) -> Result<Response<pb::EstateUpdateResponse>, Status> {
not_here()
}
async fn set_live_update(&self, _: TReq<pb::EstateSetLiveUpdateRequest>) -> Result<Response<pb::EstateUpdateResponse>, Status> {
not_here()
}
async fn adopt_estate(&self, _: TReq<pb::EstateAdoptRequest>) -> Result<Response<pb::EstateAdoptResponse>, Status> {
not_here()
}
async fn list_code_red(&self, _: TReq<pb::CodeRedListRequest>) -> Result<Response<pb::CodeRedList>, Status> {
not_here()
}
async fn raise_code_red(&self, _: TReq<pb::CodeRed>) -> Result<Response<pb::CodeRedList>, Status> {
not_here()
}
async fn clear_code_red(&self, _: TReq<pb::CodeRedClearRequest>) -> Result<Response<pb::CodeRedList>, Status> {
not_here()
}
async fn narrate_log(&self, _: TReq<pb::EstateNarrateLogRequest>) -> Result<Response<pb::EstateNarrateLogResponse>, Status> {
not_here()
}
async fn install_log(&self, _: TReq<pb::EstateInstallLogRequest>) -> Result<Response<pb::EstateInstallLogResponse>, Status> {
not_here()
}
async fn ack_code_red(&self, _: TReq<pb::CodeRedAckRequest>) -> Result<Response<pb::CodeRedList>, Status> {
not_here()
}
async fn order_site(&self, _: TReq<pb::EstateOrderSiteRequest>) -> Result<Response<pb::EstateOrderSiteResponse>, Status> {
not_here()
}
async fn first_link(&self, _: TReq<pb::EstateOrderFirstLinkRequest>) -> Result<Response<pb::EstateOrderFirstLinkResponse>, Status> {
not_here()
}
async fn refeed(&self, _: TReq<pb::EstateOrderRefeedRequest>) -> Result<Response<pb::EstateOrderRefeedResponse>, Status> {
not_here()
}
async fn console(&self, _: TReq<pb::EstateOrderConsoleRequest>) -> Result<Response<pb::EstateOrderConsoleResponse>, Status> {
not_here()
}
async fn terminate(&self, _: TReq<pb::EstateOrderTerminateRequest>) -> Result<Response<pb::EstateOrderTerminateResponse>, Status> {
not_here()
}
async fn rebuild_main(&self, _: TReq<pb::EstateRebuildMainRequest>) -> Result<Response<pb::EstateRebuildMainResponse>, Status> {
not_here()
}
async fn start(&self, _: TReq<pb::EstateOrderStartRequest>) -> Result<Response<pb::EstateOrderStartResponse>, Status> {
not_here()
}
async fn renew(&self, _: TReq<pb::EstateOrderRenewRequest>) -> Result<Response<pb::EstateOrderRenewResponse>, Status> {
not_here()
}
async fn send_sign_in_again(&self, _: TReq<pb::EstateOrderSignInAgainRequest>) -> Result<Response<pb::EstateOrderSignInAgainResponse>, Status> {
not_here()
}
async fn change_plan(&self, _: TReq<pb::EstateOrderChangePlanRequest>) -> Result<Response<pb::EstateOrderChangePlanResponse>, Status> {
not_here()
}
async fn ladder(&self, _: TReq<pb::EstateOrderLadderRequest>) -> Result<Response<pb::EstateOrderLadderResponse>, Status> {
not_here()
}
async fn purchases(&self, _: TReq<pb::EstateOrderPurchasesRequest>) -> Result<Response<pb::EstateOrderPurchasesResponse>, Status> {
not_here()
}
async fn notice(&self, _: TReq<pb::EstateNoticeRequest>) -> Result<Response<pb::EstateNoticeResponse>, Status> {
not_here()
}
async fn sells(&self, _: TReq<pb::EstateOrderSellsRequest>) -> Result<Response<pb::EstateOrderSellsResponse>, Status> {
not_here()
}
async fn cancel(&self, _: TReq<pb::EstateOrderCancelRequest>) -> Result<Response<pb::EstateOrderCancelResponse>, Status> {
not_here()
}
async fn restore_front(&self, _: TReq<pb::EstateOrderRestoreFrontRequest>) -> Result<Response<pb::EstateOrderRestoreFrontResponse>, Status> {
not_here()
}
async fn settle(&self, _: TReq<pb::EstateOrderSettleRequest>) -> Result<Response<pb::EstateOrderSettleResponse>, Status> {
not_here()
}
}
fn operator_gate(req: TReq<()>) -> Result<TReq<()>, Status> {
match req.metadata().get("authorization").and_then(|v| v.to_str().ok()) {
Some(v) if v == format!("Bearer {OPERATOR}") => Ok(req),
_ => Err(Status::unauthenticated("bearer missing or wrong")),
}
}
async fn console() -> (Router, Arc<Held>) {
let held = Arc::new(Held::default());
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let at = l.local_addr().unwrap().to_string();
tokio::spawn(tonic::transport::Server::builder().add_service(EstateOrderServer::with_interceptor(Fake(held.clone()), operator_gate)).serve_with_incoming(tonic::transport::server::TcpIncoming::from(l)));
let operator = Monetize::connect_lazy(&at, OPERATOR).unwrap();
let app = Arc::new(App { webauthn: None, admin_token: Some(ADMIN.into()), events: EventHub::spawn(operator.clone(), Duration::from_secs(3600)), monetize: operator.clone(), operator: Some(operator), ui_dir: None });
(serve::router(app), held)
}
async fn get(router: &Router, path: &str) -> (StatusCode, Value, Duration) {
let t0 = Instant::now();
let req = Request::builder().uri(path).header("authorization", format!("Bearer {ADMIN}")).body(Body::empty()).unwrap();
let res = tokio::time::timeout(Duration::from_secs(60), router.clone().oneshot(req)).await.unwrap_or_else(|_| panic!("{path} did not answer within 60 s: an endless spinner")).unwrap();
let status = res.status();
let bytes = res.into_body().collect().await.unwrap().to_bytes();
let v = serde_json::from_slice(&bytes).unwrap_or_else(|_| Value::String(String::from_utf8_lossy(&bytes).into_owned()));
(status, v, t0.elapsed())
}
fn door_of<'a>(v: &'a Value, estate: &str) -> &'a str {
let site = v["sites"].as_array().and_then(|s| s.iter().find(|s| s["estate"] == estate)).unwrap_or_else(|| panic!("{estate} is not listed: {v}"));
site["first_admin_door"].as_str().unwrap_or_default()
}
/// ★ RED: two sites whose fronts never answer hold the Sites page no longer than its bound;
/// every site is listed, the one that answered with its door, the dark ones without one.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn the_sites_page_answers_within_its_bound_with_unreachable_sites_present() {
let (router, held) = console().await;
let (st, v, took) = get(&router, "/api/sites").await;
assert_eq!(st, StatusCode::OK, "{v}");
assert!(took < PAGE_WITHIN, "/api/sites took {took:?} with two unreachable sites");
assert_eq!(v["sites"].as_array().map(Vec::len), Some(3), "every site is listed: {v}");
assert_eq!(door_of(&v, "lit-rs"), "open", "the site that answered shows its door: {v}");
assert_eq!(door_of(&v, "dark-one-rs"), "", "an unanswered door is left empty (the button still stands)");
assert_eq!(door_of(&v, "dark-two-rs"), "");
// A second look while the dark reads are still out: within the bound again, and the dark
// sites are not asked a second time (one flight per site).
let before = held.statuses.load(Ordering::SeqCst);
let (st, _, took) = get(&router, "/api/sites").await;
assert_eq!(st, StatusCode::OK);
assert!(took < PAGE_WITHIN, "the second look took {took:?}");
let asked = held.statuses.load(Ordering::SeqCst) - before;
assert_eq!(asked, 0, "the second look asked {asked} statuses: the dark ones join their flights, and the lit door read a moment ago is not asked again (each ask probes the site with its invite)");
}