crates/upcloud-api/src/net.rs
//! **The transport boundary: what is retried and what never is.**
//!
//! Moved here from `private-gunnar-ops/xtask/src/remote/net.rs` (lane T14,
//! 2026-09-21) so the ONE wire implementation of [`crate::UpCloudApi`] carries
//! the same policy every client in that crate already had — and so that crate's
//! other HTTP clients (Cloudflare, the health probe) re-use this copy instead of
//! keeping a second one.
//!
//! MEASURED 2026-09-20, the live bring-up: `GET /1.3/price` on the account
//! answered "error sending request for url" ONCE, and a credential probe turned
//! that one dropped connection into "provider unavailable". The token was fine.
//! The link hiccuped.
//!
//! # The boundary, and it is not negotiable in the permissive direction
//!
//! * **TRANSPORT** — the question never reached the far end, or its answer never
//! came back whole: a refused connection, a reset, a timeout, a name that did
//! not resolve, a TLS handshake that fell over, an HTTP/1.1 framing error
//! mid-flight. Nobody composed that; it is the wire. RETRYABLE, with a bounded
//! budget and a backoff, and **every retry is printed** so a flaky link can
//! never be mistaken for a healthy one.
//! * **PROTOCOL** — the far end understood the question and composed an answer:
//! a status code, a refusal body, a URI this crate built wrong. That is DATA.
//! It is never retried and never softened. A 401 is a revoked credential, a
//! 403 is a scope, a 412 is UpCloud saying out_of_stock, a 404 is an absence —
//! and retrying any of them turns a clear refusal into a hang.
//!
//! Every client in this crate is built with `http_status_as_error(false)`, so a
//! status code never even arrives here as an `Err`: it comes back as an ordinary
//! response and is judged by the code that asked. That is not an accident, it is
//! the boundary made structural — the only thing this module can see is the wire.
use std::time::Duration;
/// Which side of the boundary an error fell on.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Side {
/// The wire. Bounded retry.
Transport,
/// A composed answer, or our own mistake. Final.
Protocol,
}
/// Classify a `ureq` error. The match is on the VARIANT, not on the text of the
/// message, because a message is a sentence someone can reword and a variant is
/// a decision the library made.
pub fn side(e: &ureq::Error) -> Side {
match e {
// The wire.
ureq::Error::Io(_)
| ureq::Error::Timeout(_)
| ureq::Error::HostNotFound
| ureq::Error::ConnectionFailed
| ureq::Error::Tls(_)
| ureq::Error::Protocol(_) => Side::Transport,
// An answer, or a request this crate built wrong. Both are final: the
// first because it is data, the second because trying it again will
// build the same broken request.
_ => Side::Protocol,
}
}
/// How long the wire is given to stop hiccuping.
#[derive(Debug, Clone, Copy)]
pub struct Budget {
/// Total attempts, the first one included. 1 = no retry at all.
pub attempts: u32,
/// The wait after the first failure; doubled each time, capped at `cap`.
pub first: Duration,
pub cap: Duration,
/// Retry only failures that prove the request NEVER LEFT THIS BOX.
///
/// For a GET or a DELETE this is off: they are idempotent, and asking twice
/// costs nothing. For a POST that CREATES something it is on, because a
/// connection reset after the bytes went out is ambiguous — the server may
/// have made the thing and lost the answer — and retrying that is how one
/// order becomes two servers on the invoice. A refused connection and a
/// name that did not resolve are not ambiguous: nothing was sent.
pub unsent_only: bool,
}
impl Budget {
/// What an API read gets: four attempts over ~7 s of waiting. Long enough
/// for a dropped connection and a re-dial, short enough that a cloud that
/// is genuinely down is named within the minute rather than leaned on.
pub const fn api() -> Budget {
Budget { attempts: 4, first: Duration::from_secs(1), cap: Duration::from_secs(4), unsent_only: false }
}
/// What a WRITE gets: the same budget, but only for failures that prove the
/// request never left this box. See `unsent_only`.
pub const fn write() -> Budget {
Budget { unsent_only: true, ..Budget::api() }
}
/// One attempt: the boundary is still declared, and nothing is retried.
pub const fn once() -> Budget {
Budget { attempts: 1, first: Duration::from_secs(0), cap: Duration::from_secs(0), unsent_only: false }
}
}
/// Did this failure happen BEFORE a single byte of the request went out?
///
/// A connection that was refused, a name that did not resolve, a connector that
/// gave up: nothing reached the far end, so asking again cannot duplicate
/// anything. A reset or a timeout mid-flight is NOT in this set — the request
/// may well have arrived and been acted on.
pub fn never_sent(e: &ureq::Error) -> bool {
match e {
ureq::Error::HostNotFound | ureq::Error::ConnectionFailed => true,
ureq::Error::Io(io) => matches!(
io.kind(),
std::io::ErrorKind::ConnectionRefused
| std::io::ErrorKind::NetworkUnreachable
| std::io::ErrorKind::HostUnreachable
| std::io::ErrorKind::AddrNotAvailable
),
_ => false,
}
}
/// Run `op`, retrying ONLY what `side` calls `Transport`, saying every retry out
/// loud. `what` names the call in the line that is printed and in the error.
pub fn call<T>(
what: &str,
budget: Budget,
op: impl FnMut() -> Result<T, ureq::Error>,
) -> Result<T, ureq::Error> {
call_with(what, budget, &mut |l| eprintln!("{l}"), &mut std::thread::sleep, op)
}
/// `call`, with the clock and the voice injected so the budget can be tested
/// without a test that sleeps.
pub fn call_with<T>(
what: &str,
budget: Budget,
say: &mut dyn FnMut(&str),
sleep: &mut dyn FnMut(Duration),
mut op: impl FnMut() -> Result<T, ureq::Error>,
) -> Result<T, ureq::Error> {
let mut wait = budget.first;
for attempt in 1..=budget.attempts.max(1) {
let e = match op() {
Ok(v) => {
if attempt > 1 {
say(&format!(" ok {what}: answered on attempt {attempt} of {}", budget.attempts));
}
return Ok(v);
}
Err(e) => e,
};
if side(&e) == Side::Protocol {
// A composed answer. It is the point of the call, not an obstacle.
return Err(e);
}
if budget.unsent_only && !never_sent(&e) {
say(&format!(
" RED {what}: the wire failed after the request went out ({e}) — NOT retried, \
because a write that may have landed must not be sent twice"
));
return Err(e);
}
if attempt == budget.attempts.max(1) {
say(&format!(
" RED {what}: the wire failed all {} attempts — {e}",
budget.attempts.max(1)
));
return Err(e);
}
say(&format!(
" … {what}: transport failure on attempt {attempt} of {} ({e}) — retrying in {} s",
budget.attempts,
wait.as_secs_f32()
));
sleep(wait);
wait = (wait * 2).min(budget.cap);
}
unreachable!("the loop returns on the last attempt")
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::RefCell;
fn io(kind: std::io::ErrorKind) -> ureq::Error {
ureq::Error::Io(std::io::Error::from(kind))
}
#[test]
fn the_wire_is_transport_and_an_answer_is_protocol() {
assert_eq!(side(&io(std::io::ErrorKind::ConnectionRefused)), Side::Transport);
assert_eq!(side(&io(std::io::ErrorKind::ConnectionReset)), Side::Transport);
assert_eq!(side(&ureq::Error::HostNotFound), Side::Transport);
assert_eq!(side(&ureq::Error::ConnectionFailed), Side::Transport);
assert_eq!(side(&ureq::Error::Tls("handshake")), Side::Transport);
// Composed by the far end, or by us. Never retried.
assert_eq!(side(&ureq::Error::StatusCode(401)), Side::Protocol);
assert_eq!(side(&ureq::Error::StatusCode(403)), Side::Protocol);
assert_eq!(side(&ureq::Error::StatusCode(412)), Side::Protocol);
assert_eq!(side(&ureq::Error::BadUri("nonsense".into())), Side::Protocol);
}
/// TODAY'S BUG: one dropped connection, and the second attempt would have
/// answered. Without the retry this call is a permanent refusal.
#[test]
fn one_hiccup_then_an_answer_is_an_answer_and_it_is_said_out_loud() {
let n = RefCell::new(0);
let said = RefCell::new(Vec::new());
let mut slept = Vec::new();
let got = call_with(
"GET /1.3/price",
Budget::api(),
&mut |l| said.borrow_mut().push(l.to_string()),
&mut |d| slept.push(d),
|| {
*n.borrow_mut() += 1;
if *n.borrow() == 1 { Err(io(std::io::ErrorKind::ConnectionReset)) } else { Ok(200u16) }
},
)
.expect("the second attempt answered");
assert_eq!(got, 200);
assert_eq!(*n.borrow(), 2);
assert_eq!(slept, vec![Duration::from_secs(1)], "one backoff, and it is the first");
let said = said.borrow();
assert!(said.iter().any(|l| l.contains("transport failure on attempt 1")), "{said:?}");
assert!(said.iter().any(|l| l.contains("answered on attempt 2")), "a retry is never silent: {said:?}");
}
/// The other direction: a real answer is NOT retried, ever. A revoked
/// credential must fail on the first call, not after four.
#[test]
fn a_composed_refusal_is_tried_exactly_once() {
for code in [401u16, 403, 404, 412] {
let n = RefCell::new(0);
let err = call_with(
"POST /1.3/server",
Budget::api(),
&mut |_| {},
&mut |_| panic!("a protocol answer must never sleep"),
|| {
*n.borrow_mut() += 1;
Err::<(), _>(ureq::Error::StatusCode(code))
},
)
.unwrap_err();
assert!(matches!(err, ureq::Error::StatusCode(c) if c == code));
assert_eq!(*n.borrow(), 1, "{code} was retried");
}
}
#[test]
fn a_wire_that_never_comes_back_is_named_red_after_the_whole_budget() {
let n = RefCell::new(0);
let said = RefCell::new(Vec::new());
let mut slept = Vec::new();
let err = call_with(
"GET /zones",
Budget::api(),
&mut |l| said.borrow_mut().push(l.to_string()),
&mut |d| slept.push(d),
|| {
*n.borrow_mut() += 1;
Err::<(), _>(ureq::Error::ConnectionFailed)
},
)
.unwrap_err();
assert!(matches!(err, ureq::Error::ConnectionFailed));
assert_eq!(*n.borrow(), 4);
// Doubling, capped: 1, 2, 4 — and no sleep after the last attempt.
assert_eq!(slept, vec![Duration::from_secs(1), Duration::from_secs(2), Duration::from_secs(4)]);
assert!(said.borrow().iter().any(|l| l.contains("failed all 4 attempts")), "{:?}", said.borrow());
}
/// A write is retried only when nothing can possibly have landed.
#[test]
fn a_write_is_never_sent_twice_after_the_bytes_went_out() {
// Refused: nothing left the box, so ask again.
let n = RefCell::new(0);
let _ = call_with(
"POST /1.3/server",
Budget::write(),
&mut |_| {},
&mut |_| {},
|| {
*n.borrow_mut() += 1;
Err::<(), _>(io(std::io::ErrorKind::ConnectionRefused))
},
);
assert_eq!(*n.borrow(), 4, "a refused connection created nothing");
// Reset mid-flight: the server may have made the thing. Once, and once only.
let n = RefCell::new(0);
let _ = call_with(
"POST /1.3/server",
Budget::write(),
&mut |_| {},
&mut |_| panic!("an ambiguous write must not sleep and try again"),
|| {
*n.borrow_mut() += 1;
Err::<(), _>(io(std::io::ErrorKind::ConnectionReset))
},
);
assert_eq!(*n.borrow(), 1);
// …and a GET of the same shape is still retried, because it is idempotent.
let n = RefCell::new(0);
let _ = call_with(
"GET /1.3/server",
Budget::api(),
&mut |_| {},
&mut |_| {},
|| {
*n.borrow_mut() += 1;
Err::<(), _>(io(std::io::ErrorKind::ConnectionReset))
},
);
assert_eq!(*n.borrow(), 4);
}
#[test]
fn a_budget_of_one_retries_nothing() {
let n = RefCell::new(0);
let _ = call_with(
"GET /account",
Budget::once(),
&mut |_| {},
&mut |_| panic!("no sleep with no retry"),
|| {
*n.borrow_mut() += 1;
Err::<(), _>(ureq::Error::ConnectionFailed)
},
);
assert_eq!(*n.borrow(), 1);
}
}