deploy/upcloud/tests/clone_probe.rs
//! **The measurement that decides the shape of the fix**: does a storage CLONE
//! also spend ~100 s in `syncing`, or is it cheap?
//!
//! The re-image uploads one ISO into TWO storages because the machine takes it
//! as two faces (see `src/machine.rs`). MEASURED on the live estate 2026-09-20:
//! the PUT of 43 485 184 bytes took **5 s** and the storage then sat in state
//! `syncing` for **100–130 s** before it turned `online`. The import object and
//! the storage keep two different clocks — the import reported `completed` at
//! 09:36:21Z while the poller still read `syncing` at 65 s and at 97 s.
//!
//! So the transfer was never the cost. If `POST /storage/{uuid}/clone` produces
//! a storage that ALSO goes through `syncing` for the same ~110 s, then cloning
//! saves the 5 s and nothing that matters, and the fix has to be somewhere else.
//! That is not a question to guess at, so this test asks the real API — with
//! ONE throwaway storage of its own, never the estate's.
//!
//! # THE ANSWER, MEASURED 2026-09-20 ON THE LIVE ACCOUNT
//!
//! One throwaway 1 GiB volume, 43 485 184 bytes of non-zero data imported into
//! it, then cloned. Both given back afterwards; the account was verified clean.
//!
//! ```text
//! PUT of 43 485 184 bytes 1 s
//! import -> online 114 s maintenance, syncing, maintenance, online
//! POST /storage/{uuid}/clone 728 ms (the call itself)
//! clone -> online 47 s maintenance, online <- NO `syncing`
//! ```
//!
//! **A clone is genuinely cheaper than an import: 47 s against 114 s, and it
//! never enters `syncing` at all.** The prior going in was that a clone would
//! sync like anything else and buy only the five seconds of transfer. That was
//! WRONG, and the only reason anyone knows is that it was measured.
//!
//! # AND IT IS STILL THE WRONG FIX, FOR A REASON ONLY BOTH NUMBERS SHOW
//!
//! A clone cannot start until its SOURCE is `online` — that is what a clone is.
//! So the clone road is serial BY CONSTRUCTION:
//!
//! ```text
//! clone road import 114 s -> clone 47 s = 161 s
//! what ships now import 114 s || import 114 s = ~115 s
//! ```
//!
//! Two imports run at the same time because they depend on nothing; an import
//! and a clone cannot, because the second depends on the first. **Building on
//! clone would have been a ~46 s REGRESSION against the fix already in the
//! tree**, and it would have looked like an optimisation the whole way in:
//! every step of it is faster than the step it replaces.
//!
//! So `clone_storage` stays as a door with a measurement attached, and nothing
//! is built on it. This file exists so the question is not re-opened from the
//! same plausible reasoning a third time.
//!
//! It is `#[ignore]`d and additionally gated on `$GUNNAR_UPCLOUD_LIVE_PROBE=1`,
//! because a test that spends money and talks to a live account must be asked
//! for twice.
//!
//! The title carries `gunnar-probe-` so that `upcloud-orphans` can see it: a
//! storage this test leaves behind after a panic must be RECOGNISABLE, not
//! invisible.
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use gunnar_upcloud::api::{Client, DEFAULT_ZONE};
use gunnar_upcloud::plan::upload_iso;
use gunnar_upcloud::upload::Iso;
/// What the live estate's medium weighs, to the byte — so the probe measures
/// the same shape of import and not a toy one.
const MEDIUM_BYTES: u64 = 43_485_184;
fn epoch() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
/// Poll a storage until it is `online`, and report how long that took and every
/// distinct state it passed through on the way.
fn time_to_online<A: upcloud_api::UpCloudApi>(
client: &Client<A>,
uuid: &str,
budget: Duration,
) -> Result<(Duration, Vec<String>), String> {
let started = Instant::now();
let mut seen: Vec<String> = Vec::new();
loop {
let s = client.storage_state(uuid)?;
if seen.last().map(|l| l != &s).unwrap_or(true) {
eprintln!(" t+{:>4} s state {s:?}", started.elapsed().as_secs());
seen.push(s.clone());
}
if s == "online" {
return Ok((started.elapsed(), seen));
}
if started.elapsed() > budget {
return Err(format!(
"{uuid} never reached online inside {} s — states seen: {seen:?}",
budget.as_secs()
));
}
std::thread::sleep(Duration::from_secs(2));
}
}
#[test]
#[ignore = "talks to the live UpCloud account and creates a storage; set GUNNAR_UPCLOUD_LIVE_PROBE=1"]
fn a_clone_is_measured_not_assumed() {
if std::env::var("GUNNAR_UPCLOUD_LIVE_PROBE").as_deref() != Ok("1") {
eprintln!("skipped: GUNNAR_UPCLOUD_LIVE_PROBE is not 1");
return;
}
let home = std::env::var("HOME").expect("HOME");
let creds = std::path::PathBuf::from(home).join("upload.txt");
// By KEY NAME, never by shape: `~/gunnar_secrets` holds seven `ucat_`
// strings across four UpCloud accounts and the one this estate means is the
// fifth. A probe that creates and deletes storages must be certain WHOSE
// account it is in. See `token`'s header.
let key = std::env::var("GUNNAR_UPCLOUD_TOKEN_KEY")
.unwrap_or_else(|_| gunnar_upcloud::token::DEFAULT_KEY.to_string());
let found = gunnar_upcloud::token::find_named(&creds, &key).expect("an UpCloud token");
let client = Client::new(live(found.token, 240));
// A file of exactly the medium's weight, written here so the probe carries
// no artefact of its own and depends on no build having run.
let dir = std::env::temp_dir().join(format!("gunnar-probe-{}", epoch()));
std::fs::create_dir_all(&dir).expect("temp dir");
let path = dir.join("medium.bin");
{
use std::io::Write;
let mut f = std::fs::File::create(&path).expect("create");
// Not zeroes: a thin-provisioned volume could sync a sparse image in no
// time at all and the measurement would flatter the clone.
let mut block = vec![0u8; 1 << 20];
let mut x: u32 = 0x9E37_79B9;
for b in block.iter_mut() {
x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
*b = (x >> 24) as u8;
}
let mut left = MEDIUM_BYTES;
while left > 0 {
let n = left.min(block.len() as u64) as usize;
f.write_all(&block[..n]).expect("write");
left -= n as u64;
}
f.flush().expect("flush");
}
let iso = Iso::read(&path).expect("hash the probe medium");
assert_eq!(iso.len, MEDIUM_BYTES);
let stamp = epoch();
let src_title = format!("gunnar-probe-{stamp}-source");
let clone_title = format!("gunnar-probe-{stamp}-clone");
eprintln!("\n── 1. POST /storage — {src_title:?}");
let src = client
.create_storage(&src_title, DEFAULT_ZONE)
.expect("create the throwaway source");
eprintln!(" {src}");
let outcome = (|| -> Result<(), String> {
eprintln!(
"── 2. POST /storage/{{uuid}}/import — direct_upload, then PUT {MEDIUM_BYTES} bytes"
);
let url = client.direct_upload_url(&src)?;
let put_started = Instant::now();
let reply = upload_iso(&client, &url, &iso)?;
let put_took = put_started.elapsed();
gunnar_upcloud::upload::verify_upload(&iso, &reply)?;
eprintln!(
" the PUT itself: {} s (digest matched)",
put_took.as_secs()
);
eprintln!("── 3. how long the IMPORTED storage takes to turn online");
let (import_wait, import_states) = time_to_online(&client, &src, Duration::from_secs(900))?;
eprintln!("── 4. POST /storage/{{uuid}}/clone — {clone_title:?}");
let clone_call = Instant::now();
let clone = client.clone_storage(&src, &clone_title, DEFAULT_ZONE)?;
eprintln!(
" {clone} — the call itself answered in {} ms",
clone_call.elapsed().as_millis()
);
eprintln!("── 5. how long the CLONE takes to turn online");
let clone_result = time_to_online(&client, &clone, Duration::from_secs(900));
eprintln!("\n╭─ THE ANSWER ─────────────────────────────────────────────");
eprintln!(
"│ PUT of {MEDIUM_BYTES} bytes {:>4} s",
put_took.as_secs()
);
eprintln!(
"│ import → online {:>4} s via {import_states:?}",
import_wait.as_secs()
);
match &clone_result {
Ok((d, states)) => eprintln!(
"│ clone → online {:>4} s via {states:?}",
d.as_secs()
),
Err(e) => eprintln!("│ clone → online NEVER {e}"),
}
eprintln!("╰──────────────────────────────────────────────────────────\n");
// Give the clone back first: UpCloud will not delete a storage that
// something still holds, and the source is the thing it was cut from.
if let Err(e) = client.delete_storage(&clone) {
eprintln!(" the clone {clone} could not be deleted: {e}");
}
clone_result.map(|_| ())
})();
if let Err(e) = client.delete_storage(&src) {
eprintln!(
" the source {src} ({src_title}) could not be deleted: {e} — DELETE IT BY HAND"
);
} else {
eprintln!(" gave back the throwaway source {src}");
}
let _ = std::fs::remove_dir_all(&dir);
outcome.expect("the probe");
}
/// **Read-only: did the probe above give its throwaway volumes back?**
///
/// The probe deletes both on the way out, including on the error path, but "it
/// deletes" is a claim about code and this is a claim about the ACCOUNT. A
/// 1 GiB volume left behind is billed, and a probe that quietly litters an
/// account it was lent is a probe nobody should run again.
///
/// It lists and never deletes: the estate's own storages are in this account
/// and nothing here may touch them.
#[test]
#[ignore = "reads the live UpCloud account; set GUNNAR_UPCLOUD_LIVE_PROBE=1"]
fn the_probe_left_nothing_behind() {
if std::env::var("GUNNAR_UPCLOUD_LIVE_PROBE").as_deref() != Ok("1") {
eprintln!("skipped: GUNNAR_UPCLOUD_LIVE_PROBE is not 1");
return;
}
let home = std::env::var("HOME").expect("HOME");
let creds = match std::env::var("GUNNAR_UPCLOUD_CREDS") {
Ok(p) if !p.is_empty() => std::path::PathBuf::from(p),
_ => std::path::PathBuf::from(home).join("upload.txt"),
};
// By KEY NAME, never by shape: `~/gunnar_secrets` holds seven `ucat_`
// strings across four UpCloud accounts and the one this estate means is the
// fifth. A probe that creates and deletes storages must be certain WHOSE
// account it is in. See `token`'s header.
let key = std::env::var("GUNNAR_UPCLOUD_TOKEN_KEY")
.unwrap_or_else(|_| gunnar_upcloud::token::DEFAULT_KEY.to_string());
let found = gunnar_upcloud::token::find_named(&creds, &key).expect("an UpCloud token");
let client = Client::new(live(found.token, 60));
let all = client.private_storages().expect("GET /storage/private");
let mine: Vec<&(String, String, String)> = all
.iter()
.filter(|(_, title, _)| title.starts_with("gunnar-probe-"))
.collect();
eprintln!("the account holds {} private storage(s)", all.len());
for (uuid, title, state) in &mine {
eprintln!(" LEFTOVER {uuid} {title} {state}");
}
assert!(
mine.is_empty(),
"{} probe storage(s) were left in the account and are billed — delete them: {:?}",
mine.len(),
mine.iter()
.map(|(u, t, _)| format!("{u} {t}"))
.collect::<Vec<_>>()
);
}
/// The account, through the ONE wire — and only in a shell that is not aimed at
/// a fake: `Endpoint::account` refuses by name if it is.
fn live(token: String, call_timeout_s: u64) -> Box<dyn upcloud_api::UpCloudApi + Send + Sync> {
let endpoint =
upcloud_api::Endpoint::account().expect("a live probe in a shell aimed at a fake");
upcloud_api::connect(
&endpoint,
upcloud_api::Credential::Token(token),
upcloud_api::Options {
call_timeout: Duration::from_secs(call_timeout_s),
upload_timeout: Duration::from_secs(call_timeout_s),
retry_transport: true,
},
)
}