crates/gunnar-server/tests/main_rebuild_auto.rs
//! **Main rebuild (owner 2026-10-12): the re-laid main fills itself** — the generic
//! primitives under the three triggers: the fed auto-fix (`repair_auto`), the fed repair
//! key (`secret.repair_key`), the awaiting-main kick, and the `data_head` health word.
#![cfg(all(feature = "replication", feature = "wg-client"))]
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use gunnar_core::ssh_key::PrivateKey;
use gunnar_core::{NewRepo, RepoRef};
use gunnar_grpc::{EventBus, ServerConfig, ServiceContext};
use gunnar_server::repair::{self, RepairConfig};
use gunnar_server::stores::ServerStores;
use gunnar_server::twin;
use gunnar_server::CoreControlPlane;
fn free_addr() -> SocketAddr {
let l = std::net::TcpListener::bind("127.0.0.1:0").expect("bind");
l.local_addr().expect("addr")
}
fn wait<T>(what: &str, within: Duration, mut probe: impl FnMut() -> Option<T>) -> T {
let t0 = Instant::now();
loop {
if let Some(v) = probe() {
return v;
}
assert!(t0.elapsed() < within, "{what}: not within {within:?}");
std::thread::sleep(Duration::from_millis(25));
}
}
/// A bare control plane for `stores` at `addr` (the old primary, the twin).
struct Served {
_rt: tokio::runtime::Runtime,
stop: Option<tokio::sync::oneshot::Sender<()>>,
addr: SocketAddr,
}
fn serve(stores: &Arc<ServerStores>, addr: SocketAddr) -> Served {
serve_as(
stores,
addr,
ServerConfig::default().insecure_allow_all_for_local_development(),
)
}
/// A box with REAL credential checks (no insecure mode): the twin the narrow repair key reads.
fn serve_secure(stores: &Arc<ServerStores>, addr: SocketAddr) -> Served {
serve_as(stores, addr, ServerConfig::default())
}
fn serve_as(stores: &Arc<ServerStores>, addr: SocketAddr, config: ServerConfig) -> Served {
let plane: Arc<dyn gunnar_grpc::ControlPlane> =
Arc::new(CoreControlPlane::new(Arc::clone(stores)));
let ctx = ServiceContext::new(plane, Arc::new(EventBus::new(16)), config);
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.expect("rt");
let (tx, rx) = tokio::sync::oneshot::channel::<()>();
rt.spawn(async move {
let _ = gunnar_grpc::router(ctx, || 0)
.serve_with_shutdown(addr, async {
let _ = rx.await;
})
.await;
});
wait("the box listens", Duration::from_secs(10), || {
std::net::TcpStream::connect(addr).ok()
});
Served {
_rt: rt,
stop: Some(tx),
addr,
}
}
impl Served {
fn kill(mut self) {
if let Some(tx) = self.stop.take() {
let _ = tx.send(());
}
let addr = self.addr;
wait("the box stops listening", Duration::from_secs(10), || {
std::net::TcpStream::connect(addr).is_err().then_some(())
});
}
}
fn write_repo(stores: &ServerStores, ns: &str, name: &str) {
if stores
.catalog()
.namespaces()
.unwrap()
.iter()
.all(|n| n.name != ns)
{
stores.catalog().create_namespace(ns).expect("namespace");
}
stores
.create_repo(NewRepo::new(RepoRef::new(ns, name).expect("legal")), false)
.expect("repo");
}
fn mint() -> (Vec<u8>, String) {
let pem = gunnar_core::mint_ed25519().expect("mint");
let line = PrivateKey::from_openssh(std::str::from_utf8(&pem).unwrap())
.expect("parse")
.public_key()
.to_openssh()
.expect("line");
(pem, line)
}
async fn status(at: SocketAddr) -> gunnar_proto::BootstrapStatusResponse {
let mut c = gunnar_proto::bootstrap_client::BootstrapClient::connect(format!("http://{at}"))
.await
.expect("connect");
c.status(gunnar_proto::BootstrapStatusRequest {})
.await
.expect("Bootstrap.Status")
.into_inner()
}
/// A re-laid main, started as `gunnar serve` starts it, with the repair wired in.
fn start_main(
rt: &tokio::runtime::Runtime,
stores: &Arc<ServerStores>,
restore_key: &str,
cfg: RepairConfig,
) -> (gunnar_server::Server, SocketAddr) {
let config = gunnar_server::ServeConfig {
no_grpc: false,
restore_pubkeys: vec![restore_key.to_owned()],
repair: Some(cfg),
..gunnar_server::ServeConfig::ephemeral()
};
let server = rt
.block_on(gunnar_server::Server::start(Arc::clone(stores), config))
.expect("the main starts");
let at = server.grpc_addr().expect("the control plane listens");
(server, at)
}
/// ★ **Main rebuild (2026-10-12): a re-laid main fills ITSELF from its twin** once the
/// estate's installer fed it auto-fix and a repair key — no `--repair-key`, no
/// `--repair-auto`, no twin key on the box (until now every such main waited in
/// `Hold::Awaiting` for a human: the restore lane that filled it was retired).
///
/// RED first: with the feed naming no key, the kick says why it cannot run; with auto-fix
/// not fed, it does nothing at all. GREEN: both fed → the main's log reaches the twin's head.
#[test]
fn a_relaid_main_fed_auto_fix_and_a_repair_key_fills_itself_from_its_twin() {
let (old_addr, twin_addr) = (free_addr(), free_addr());
// ── an estate: a primary with data, its twin following it ──
let pd = tempfile::tempdir().unwrap();
let primary = Arc::new(ServerStores::open(pd.path()).expect("primary"));
write_repo(&primary, "alice", "one");
write_repo(&primary, "bob", "two");
// The estate's NARROW repair key, fed to the primary as the installer feeds it: no grant,
// narrowed by its name; the twin learns it from the replicated identity table.
let (pem, key_line) = mint();
let (_, first_admin) = mint();
primary
.auth()
.bootstrap_admin(&first_admin)
.expect("the primary's admin");
let fed: std::collections::BTreeMap<String, String> = [(
format!("principal.{}", gunnar_grpc::REPAIR_PRINCIPAL),
format!("machine repair {key_line}"),
)]
.into_iter()
.collect();
gunnar_server::fed_principals::install(&primary, &fed).expect("the repair principal");
// The guard's R4 word: the main reports its own log head.
let words = repair::health(&primary, &old_addr.to_string());
assert_eq!(
words.get("data_head").map(String::as_str),
Some(primary.txlog().head_seq().to_string().as_str()),
"{words:?}"
);
let served_primary = serve(&primary, old_addr);
let td = tempfile::tempdir().unwrap();
let twin_stores = Arc::new(ServerStores::open(td.path()).expect("twin stores"));
twin::become_twin(&twin_stores, &old_addr.to_string()).expect("a fresh box becomes a twin");
twin::start_follower_every(Arc::clone(&twin_stores), Duration::from_millis(200));
wait(
"the twin levels with the primary",
Duration::from_secs(30),
|| {
let live = primary.txlog().head_seq();
twin::status(td.path())
.filter(|s| {
s.state == twin::State::Following
&& s.applied_seq == live
&& s.primary_head_seq == live
&& twin_stores.txlog().head_seq() == live
})
.map(|_| ())
},
);
assert!(
!repair::health(&twin_stores, "t").contains_key("data_head"),
"a twin reports its review, not a main's head"
);
// ── the main dies; the twin finds it gone ──
served_primary.kill();
drop(primary);
wait(
"the twin finds its primary gone",
Duration::from_secs(10),
|| {
twin::status(td.path())
.filter(|s| s.state == twin::State::Unreachable)
.map(|_| ())
},
);
let head = twin_stores.txlog().head_seq();
assert!(
twin_stores
.auth()
.principal(gunnar_grpc::REPAIR_PRINCIPAL)
.unwrap()
.is_some(),
"the twin holds the repair principal, replicated"
);
let served_twin = serve_secure(&twin_stores, twin_addr);
// ── the re-laid main: the medium names NO repair key and NO auto-fix ──
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.enable_all()
.build()
.expect("rt");
let md = tempfile::tempdir().unwrap();
let main = Arc::new(ServerStores::open(md.path()).expect("the re-laid main"));
let medium = RepairConfig {
auto: false,
key: None,
twin: None,
fleet: None,
};
let (server, at) = start_main(&rt, &main, &key_line, medium);
let twin_at = twin_addr.to_string();
assert!(repair::data_dead(&main));
// RED 1: no feed at all — auto-fix is off, nothing starts.
nordisk_pairing::feedbus::publish(Default::default());
assert_eq!(
repair::on_awaiting(&main, &twin_at),
None,
"auto-fix not fed: nothing starts"
);
// RED 2: auto-fix fed, no key fed — it says why it cannot run.
nordisk_pairing::feedbus::publish(
[(
gunnar_server::feed::FEED_REPAIR_AUTO.to_string(),
"true".to_string(),
)]
.into_iter()
.collect(),
);
let said = repair::on_awaiting(&main, &twin_at).expect("auto-fix is on: it says something");
assert!(
said.contains("cannot run") && said.contains(gunnar_server::feed::FEED_REPAIR_KEY),
"{said}"
);
assert_eq!(
rt.block_on(status(at)).log_head_seq,
main.txlog().head_seq(),
"nothing streamed"
);
// GREEN: auto-fix and the repair key both fed — the main fills itself.
let feed: nordisk_pairing::feedbus::Feed = [
(
gunnar_server::feed::FEED_REPAIR_AUTO.to_string(),
"true".to_string(),
),
(
gunnar_server::feed::FEED_REPAIR_KEY.to_string(),
String::from_utf8(pem).unwrap(),
),
]
.into_iter()
.collect();
let fed = gunnar_server::feed::Fed::from_feed(&feed);
assert!(fed.repair_auto && fed.repair_key.is_some(), "{fed:?}");
assert!(
!format!("{fed:?}").contains("PRIVATE KEY"),
"the fed key is withheld from Debug"
);
nordisk_pairing::feedbus::publish(feed);
// The kick from RED 2 failed before it started anything, so the next is due at once.
let said = repair::on_awaiting(&main, &twin_at).expect("the kick starts");
assert!(said.contains(&twin_at), "{said}");
assert_eq!(
repair::on_awaiting(&main, &twin_at),
None,
"one kick at a time"
);
wait(
"the main fills itself from the twin",
Duration::from_secs(60),
|| (rt.block_on(status(at)).log_head_seq == head).then_some(()),
);
wait("the restore settles", Duration::from_secs(30), || {
(!twin::is_restoring(md.path())).then_some(())
});
assert!(!repair::data_dead(&main), "the main has its data set back");
rt.block_on(server.shutdown()).expect("shutdown");
println!("ROW rebuild-auto-fed: fed repair_auto + secret.repair_key → {at} filled itself from {twin_addr} to {head}");
served_twin.kill();
twin::stop_follower();
}