crates/gunnar-server/tests/restore_credential.rs
//! **`Bootstrap.Restore` takes a credential, and the resume is the starter's
//! alone** (h-v1-6a, 2026-09-28).
//!
//! `Bootstrap.Restore` is served outside the gate (`gunnar-grpc/src/lib.rs`,
//! the one ungated service) on the utility NIC — `0.0.0.0` on the ISO variant —
//! as plaintext h2c. Its stream is the box's whole future: seq 1 is the KEK,
//! then the identity table (admin and SSH keys), the host key, the issuer CA
//! and every file. Until this lane the only thing standing between a stranger
//! and that stream was "is the box fresh?", and the RESUME branch of
//! `twin::restore_begin` skipped even that for anybody once `txlog-restoring`
//! existed — one broken stream and the box would take the rest of its history
//! from whoever asked next.
//!
//! What is proven here, over the wire, against the real plane:
//!
//! * a stream with no credential is refused before the box gives up a byte of
//! its own state (not restoring, head unchanged);
//! * a box that was never fed a key it may take a restore from refuses a
//! restore BY NAME, even one that carries a well-formed credential;
//! * a second caller — even one whose key the box would trust to START a
//! restore — cannot resume a restore somebody else started, and the one who
//! started it can.
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use gunnar_core::ssh_key::PrivateKey;
use gunnar_core::{NewRepo, RepoRef};
use gunnar_grpc::{EventBus, ServerConfig, ServiceContext};
use gunnar_proto::bootstrap_client::BootstrapClient;
use gunnar_proto::LogChunk;
use gunnar_server::stores::ServerStores;
use gunnar_server::twin;
use gunnar_server::CoreControlPlane;
fn free_addr() -> SocketAddr {
let l = std::net::TcpListener::bind("127.0.0.1:0").expect("bind");
l.local_addr().expect("addr")
}
fn wait<T>(what: &str, within: Duration, mut probe: impl FnMut() -> Option<T>) -> T {
let t0 = Instant::now();
loop {
if let Some(v) = probe() {
return v;
}
assert!(t0.elapsed() < within, "{what}: not within {within:?}");
std::thread::sleep(Duration::from_millis(25));
}
}
/// The fresh box's control plane, served the way `serve` serves it: the real
/// router, authentication REQUIRED (the default), Bootstrap outside the gate.
struct Served {
rt: tokio::runtime::Runtime,
addr: SocketAddr,
}
fn serve(stores: &Arc<ServerStores>) -> Served {
serve_with(stores, Vec::new())
}
/// [`serve`], with the keys the MEDIUM names (`--restore-pubkey`) — the kit's way in, where
/// no installer feeds the box.
fn serve_with(stores: &Arc<ServerStores>, restore_keys: Vec<String>) -> Served {
let plane: Arc<dyn gunnar_grpc::ControlPlane> =
Arc::new(CoreControlPlane::new(Arc::clone(stores)).with_restore_keys(restore_keys));
let ctx = ServiceContext::new(plane, Arc::new(EventBus::new(16)), ServerConfig::default());
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.expect("rt");
let addr = free_addr();
rt.spawn(async move {
let _ = gunnar_grpc::router(ctx, || 0).serve(addr).await;
});
wait("the box listens", Duration::from_secs(10), || {
std::net::TcpStream::connect(addr).ok()
});
Served { rt, addr }
}
impl Served {
/// Stream `chunks` into the box's `Bootstrap.Restore`, carrying `credential`
/// as `gunnar-authorization` when there is one.
fn restore(
&self,
chunks: Vec<LogChunk>,
credential: Option<String>,
) -> Result<gunnar_proto::RestoreResponse, tonic::Status> {
let url = format!("http://{}", self.addr);
self.rt.block_on(async move {
let channel = tonic::transport::Channel::from_shared(url)
.expect("url")
.connect_lazy();
let mut client = BootstrapClient::new(channel);
let mut request = tonic::Request::new(tokio_stream::iter(chunks));
if let Some(value) = credential {
request.metadata_mut().insert(
gunnar_grpc::CREDENTIAL_METADATA_KEY,
value.parse().expect("a legal header"),
);
}
client.restore(request).await.map(|r| r.into_inner())
})
}
}
/// A key, as the caller holds it, and its public line, as the installer feeds it.
fn key() -> (PrivateKey, String) {
let pem = gunnar_core::mint_ed25519().expect("mint");
let key = PrivateKey::from_openssh(std::str::from_utf8(&pem).unwrap()).expect("parse");
let line = key.public_key().to_openssh().expect("public line");
(key, line)
}
/// One fresh, single-use `gunnar-sshsig-v2` credential bound to `Bootstrap.Restore`.
fn restore_credential(key: &PrivateKey) -> String {
use std::sync::atomic::{AtomicU64, Ordering};
static N: AtomicU64 = AtomicU64::new(0);
let now = twin::now_ms();
let nonce = format!(
"restore-credential-test-{}-{}-{now}",
std::process::id(),
N.fetch_add(1, Ordering::SeqCst)
);
gunnar_grpc::sign_credential_for(key, now, &nonce, "Bootstrap.Restore")
.expect("sign")
.to_header_value()
}
/// **What the installer's door leaves on a box it fed** — `<data>/pairing/joined.toml`,
/// whose `feed` names the operator (`admin_pubkey`) and the estate's admin principals.
fn feed(root: &std::path::Path, admin: &str, monetize: &str) {
std::fs::create_dir_all(root.join("pairing")).unwrap();
let text = format!(
"estate = \"restore-credential-test\"\nrole = \"primary\"\npeers = []\n\n[me]\nname = \"main\"\noverlay = \"10.13.0.2\"\n\n\
[feed]\nadmin_pubkey = \"{admin}\"\n\"principal.monetize\" = \"machine admin {monetize}\"\n"
);
std::fs::write(root.join("pairing/joined.toml"), text).unwrap();
}
/// A primary with a log worth restoring: a handful of whole entries.
fn primary() -> (tempfile::TempDir, Arc<ServerStores>) {
let pd = tempfile::tempdir().unwrap();
let primary = Arc::new(ServerStores::open(pd.path()).unwrap());
for ns in ["a", "b", "c"] {
primary.catalog().create_namespace(ns).unwrap();
primary
.create_repo(NewRepo::new(RepoRef::new(ns, "app").unwrap()), false)
.unwrap();
}
assert!(
primary.txlog().first_seq() == 1 && primary.txlog().head_seq() >= 6,
"the primary's log is long enough to stop short in"
);
(pd, primary)
}
/// `from..=to` of `log` as the chunks a sender streams, every one announcing `head` and naming
/// the log it is (a network restore names its log, stinky row 7).
fn chunks(stores: &ServerStores, from: u64, to: u64, head: u64) -> Vec<LogChunk> {
stores
.txlog()
.range(from, to)
.unwrap()
.into_iter()
.map(|e| LogChunk {
seq: e.seq,
kind: u32::from(e.kind),
blob_len: e.blob.len() as u64,
meta: e.meta,
blob: e.blob,
last: true,
restore_head: head,
restore_bytes: 0,
log_id: "the-primarys-log".into(),
offset: None,
entry_crc: None,
snapshot_item: None,
snapshot_id: String::new(),
})
.collect()
}
/// **RED first**: one chunk — seq 1, the KEK — with NO credential, into a box
/// that is fresh. Before this lane the box took it: it gave up its own numbers,
/// marked itself restoring and applied a KEK a stranger chose.
#[test]
fn bootstrap_restore_without_a_credential_is_refused_on_a_fresh_box() {
let (_pd, primary) = primary();
let fd = tempfile::tempdir().unwrap();
let fresh = Arc::new(ServerStores::open(fd.path()).unwrap());
// Fed like a real box: there IS a key it would take a restore from — just
// not from a caller that presents none.
let (_admin, admin_line) = key();
let (_monetize, monetize_line) = key();
feed(fd.path(), &admin_line, &monetize_line);
let head_before = fresh.txlog().head_seq();
let served = serve(&fresh);
let refused = served
.restore(chunks(&primary, 1, 1, 1), None)
.expect_err("a restore with no credential must be refused");
assert!(
matches!(
refused.code(),
tonic::Code::Unauthenticated | tonic::Code::PermissionDenied
),
"refused as a credential failure, not as a precondition: {refused:?}"
);
assert!(
refused.message().contains("Bootstrap.Restore"),
"the refusal names the door: {refused:?}"
);
assert!(
!twin::is_restoring(fd.path()),
"a refused stranger leaves the box FRESH, not restoring"
);
assert_eq!(
fresh.txlog().head_seq(),
head_before,
"and its log exactly as it was: nothing was reset, nothing applied"
);
}
/// **Fail closed**: a box nobody fed a restore key refuses every restore, by a
/// refusal that says so, even one carrying a well-formed credential.
#[test]
fn a_box_never_fed_a_restore_key_refuses_restore_by_name() {
let (_pd, primary) = primary();
let fd = tempfile::tempdir().unwrap();
let fresh = Arc::new(ServerStores::open(fd.path()).unwrap());
let head_before = fresh.txlog().head_seq();
let served = serve(&fresh);
let (someone, _) = key();
let refused = served
.restore(
chunks(&primary, 1, 1, 1),
Some(restore_credential(&someone)),
)
.expect_err("an unfed box takes no restore");
assert_eq!(refused.code(), tonic::Code::PermissionDenied, "{refused:?}");
assert!(
refused.message().contains("[restore-no-key]"),
"a named refusal: {refused:?}"
);
assert!(!twin::is_restoring(fd.path()));
assert_eq!(fresh.txlog().head_seq(), head_before);
}
/// **The resume is the starter's.** A broken stream leaves the box restoring;
/// the next stream must come from the SAME key that started it.
#[test]
fn a_second_caller_cannot_resume_a_restore_it_did_not_start() {
let (_pd, primary) = primary();
let last = primary.txlog().head_seq();
let fd = tempfile::tempdir().unwrap();
let fresh = Arc::new(ServerStores::open(fd.path()).unwrap());
let (operator, operator_line) = key();
let (monetize, monetize_line) = key();
// BOTH keys are ones this box would take a restore FROM — the point is that
// being trusted to start one is not being the one who started this one.
feed(fd.path(), &operator_line, &monetize_line);
let (stranger, _) = key();
let served = serve(&fresh);
// The operator starts, and the stream ends short: the box stays restoring.
let short = served
.restore(
chunks(&primary, 1, 2, last),
Some(restore_credential(&operator)),
)
.expect_err("a short stream");
assert_eq!(short.code(), tonic::Code::FailedPrecondition, "{short:?}");
assert!(twin::is_restoring(fd.path()));
assert_eq!(fresh.txlog().head_seq(), 2);
// A stranger, and somebody with NO credential: refused, the head unmoved.
let refused = served
.restore(
chunks(&primary, 3, last, last),
Some(restore_credential(&stranger)),
)
.expect_err("a stranger");
assert!(
matches!(
refused.code(),
tonic::Code::Unauthenticated | tonic::Code::PermissionDenied
),
"{refused:?}"
);
let refused = served
.restore(chunks(&primary, 3, last, last), None)
.expect_err("no credential on the resume branch either");
assert!(
matches!(
refused.code(),
tonic::Code::Unauthenticated | tonic::Code::PermissionDenied
),
"{refused:?}"
);
assert_eq!(fresh.txlog().head_seq(), 2, "neither moved the head");
// A TRUSTED key that did not start it: refused by name.
let refused = served
.restore(
chunks(&primary, 3, last, last),
Some(restore_credential(&monetize)),
)
.expect_err("not the starter");
assert_eq!(refused.code(), tonic::Code::PermissionDenied, "{refused:?}");
assert!(
refused.message().contains("[restore-not-yours]"),
"a named refusal: {refused:?}"
);
assert_eq!(
fresh.txlog().head_seq(),
2,
"the head is where the starter left it"
);
assert!(twin::is_restoring(fd.path()));
// The starter resumes, and finishes.
let done = served
.restore(
chunks(&primary, 3, last, last),
Some(restore_credential(&operator)),
)
.expect("the starter resumes");
assert_eq!(done.head_seq, last);
assert!(done.restart_required);
assert!(!twin::is_restoring(fd.path()), "finished");
}
/// **The kit's door**: a box no installer feeds, started with `--restore-pubkey`, takes the
/// restore from that key — and still from no other.
#[test]
fn a_box_fed_only_by_the_medium_takes_the_restore_from_that_key() {
let (_pd, primary) = primary();
let last = primary.txlog().head_seq();
let fd = tempfile::tempdir().unwrap();
let fresh = Arc::new(ServerStores::open(fd.path()).unwrap());
let (operator, operator_line) = key();
let (stranger, _) = key();
let served = serve_with(&fresh, vec![operator_line]);
let refused = served
.restore(
chunks(&primary, 1, last, last),
Some(restore_credential(&stranger)),
)
.expect_err("not the medium's key");
assert_eq!(refused.code(), tonic::Code::Unauthenticated, "{refused:?}");
assert!(
refused.message().contains("[restore-unauthenticated]"),
"{refused:?}"
);
assert!(!twin::is_restoring(fd.path()));
let done = served
.restore(
chunks(&primary, 1, last, last),
Some(restore_credential(&operator)),
)
.expect("the medium's key restores");
assert_eq!(done.head_seq, last);
assert!(!twin::is_restoring(fd.path()));
}