holger-estate-front/src/autofill.rs
//! **A re-laid main is filled from its twin by the FRONT** (lane G11, 2026-10-21; Gunnar's
//! `gunnar-front` `autofill`, c7c9e11a, in Holger's boxes).
//!
//! A main this front re-admitted ([`crate::recharge`]) or the ONE box laid again starts DEAD and
//! HELD (holger-server's boot hold), reports `data=fresh`, and names the twin it awaits
//! (`RechargeStatus.awaiting_from`). Until now nothing filled it unless the ONE box's rebuild
//! plan ran or an operator pressed Recharge from twin. With the estate's fed opt-in
//! (`repair_auto = true`, nordisk-pairing [`autofill::FEED_REPAIR_AUTO`]) this front does that
//! press by itself:
//!
//! * **when** is the shared decision ([`autofill::due`], Gunnar's five cases): the main SAYS it
//! awaits its twin — fresh or restoring, and naming it. A main with data is never streamed into.
//! * **how** is the ONE recharge lane (Skidbladnir [`nordisk_twin::recharge::run`], the lane a
//! pulled `recharge_from_twin` order runs on the main): the four refusals, the main certified
//! against the twin, `Recharge`, the twin re-pointed at the new main. The console's bearer, over
//! the tunnel. If the ONE box's order fills the main first, the lane reads it recharged
//! (`AlreadyRecharged`) and the main's `Recharge` runs one at a time.
//!
//! Each look is a jera `front.autofill` job on the front's own ledger ([`JOBS_REL`]), every
//! [`autofill::LOOK_EVERY`] (the owner's rule: background work is jera jobs).
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use jera::jobs::{ChainQueue, ChainSpec, Priority};
use nordisk_pairing::autofill::{self, Awaits};
use nordisk_twin::certify::{Boxes, Issued};
use nordisk_twin::recharge::{
self as lane, Hearing, Lane, MainState, Names, Outcome, Proof, Pulled, TwinHolds,
};
use crate::certify::{proto, HolgerBoxes};
use crate::FrontVm;
/// The ledger kind of one look.
pub const KIND: &str = "front.autofill";
/// Where the looks are recorded.
pub const JOBS_REL: &str = "autofill/jobs";
/// The fresh main's principal on the twin's door (holger `twin_tls::RECHARGE_PRINCIPAL`).
pub const RECHARGE_PRINCIPAL: &str = "holger-recharge";
/// Holger's principals and slots (holger `recharge_lane::NAMES`).
pub const NAMES: Names<'static> = Names {
recharge_principal: RECHARGE_PRINCIPAL,
twin_principal: crate::certify::TWIN_PRINCIPAL,
recharge_purpose: "recharge",
twin_purpose: crate::certify::PURPOSE,
heard_within_ms: HEARD_WITHIN_MS,
};
/// A twin that heard its primary within this bound hears a LIVE box: three of holger's 60 s
/// follow polls (holger `recharge_lane::FENCE_HEARD_WITHIN_MS`).
pub const HEARD_WITHIN_MS: u64 = 3 * 60 * 1000;
/// **The front's proof at the fence gate** (Gunnar's autofill: `in_place`): the main it fills
/// is the RE-LAID main itself, so a twin that no longer hears its old primary is not a reason
/// to stop; a twin that still hears a primary with a head of its own is refused all the same
/// (another box holds the main's name).
pub fn proof() -> Proof {
Proof {
in_place: true,
..Default::default()
}
}
/// Pure: **the main's own status, as the shared decision reads it.**
pub fn awaits(s: &proto::RechargeStatusReply) -> Awaits<'_> {
Awaits {
fresh: s.fresh,
restoring: s.restoring,
recharge_from: &s.awaiting_from,
}
}
/// **The front's hands for the lane**: holger's boxes with the console's bearer, the twin's
/// install made a deliberate RE-POINT at `main` (step 7).
pub struct Filling<'a> {
pub boxes: &'a HolgerBoxes,
pub main: String,
}
impl Boxes for Filling<'_> {
fn become_twin(&self, reader_ctl: &str, source: &str) -> Result<String, String> {
self.boxes.become_twin(reader_ctl, source)
}
fn request(&self, reader_ctl: &str, purpose: &str) -> Result<String, String> {
self.boxes.request(reader_ctl, purpose)
}
fn issue(&self, source_ctl: &str, principal: &str, csr_pem: &str) -> Result<Issued, String> {
self.boxes.issue(source_ctl, principal, csr_pem)
}
fn install(&self, reader_ctl: &str, purpose: &str, issued: &Issued) -> Result<i64, String> {
// A re-point is a TWIN credential's; the main's recharge install never is one.
let repoint = (purpose == crate::certify::PURPOSE).then_some(self.main.as_str());
self.boxes.install_as(reader_ctl, purpose, issued, repoint)
}
}
impl Lane for Filling<'_> {
fn recharge_status(&self, main: &str) -> Result<MainState, String> {
let r = self.boxes.recharge_status(main)?;
Ok(MainState {
fresh: r.fresh,
restoring: r.restoring,
recharged: r.recharged,
held: r.held,
})
}
fn twin_status(&self, twin: &str) -> Result<TwinHolds, String> {
let r = self.boxes.twin_status(twin)?;
Ok(TwinHolds {
hearing: hearing_of(&r),
estate_files: r.estate_files,
artifacts: r.artifacts,
complete: r.complete,
why: r.why,
})
}
fn recharge(&self, main: &str) -> Result<Pulled, String> {
let r = self.boxes.recharge(main)?;
Ok(Pulled {
estate: r.estate,
artifacts: r.artifacts,
failed: r.failed,
})
}
}
/// Pure: **what the twin heard of its primary**, as its `TwinStatus` says it (holger
/// `recharge_lane::hearing_of`).
pub fn hearing_of(ts: &proto::TwinStatusReply) -> Hearing {
Hearing {
twin: ts.role == "twin",
primary: ts.primary.clone(),
last_answer_unix_ms: ts.primary_answer_unix_ms,
answer_ago_ms: ts.primary_answer_ago_ms,
primary_head: ts.primary_head,
state: ts.follow_state.clone(),
}
}
/// **One look, and the fill when it is due.** `Ok(None)`: nothing to do (and why, for the
/// ledger). Blocking: the queue's worker thread.
pub fn fill_once(
boxes: &HolgerBoxes,
main: &str,
twin: &str,
auto: Option<&str>,
) -> Result<Result<String, &'static str>, String> {
let status = boxes
.recharge_status(main)
.map_err(|e| format!("the main at {main}: RechargeStatus: {e}"))?;
if !autofill::due(auto, &awaits(&status)) {
return Ok(Err("the main does not await its twin"));
}
tracing::warn!(main = %main, twin = %twin, "auto-fix: the re-laid main awaits its twin — the front fills it (the console's bearer, the one recharge lane)");
let filling = Filling {
boxes,
main: main.to_string(),
};
match lane::run(&filling, &NAMES, main, Some(twin), false, &proof()) {
Ok(Outcome::Recharged { estate, artifacts, twin_follows }) => Ok(Ok(format!("filled from the twin at {twin}: {estate} ({} repositories); the twin follows it at {twin_follows}", artifacts.len()))),
Ok(Outcome::Resumable { failed }) => Err(format!("the fill from {twin} moved bytes but is not whole ({} failed): the main stays held and the next look resumes it", failed.len())),
Ok(Outcome::AlreadyRecharged) => Ok(Ok("was recharged meanwhile (the ONE box's order): nothing moved".into())),
Err(f) => Err(format!("the recharge was refused: {f}")),
}
}
/// What the front last said, so a refusal that repeats every look is said once (Gunnar's).
#[derive(Default)]
pub struct Watch {
said: Mutex<String>,
}
impl Watch {
fn say_once(&self, why: &str) {
let mut said = self.said.lock().unwrap_or_else(|p| p.into_inner());
if *said != why {
tracing::info!("auto-fix: {why}");
why.clone_into(&mut said);
}
}
}
/// **The job**: the estate's opt-in from the front's own feed, the main and the twin from the
/// verdict, the console's bearer; then [`fill_once`].
pub fn run(
w: &Watch,
vm: &FrontVm,
handle: &tokio::runtime::Handle,
spec: &ChainSpec,
) -> anyhow::Result<serde_json::Value> {
if spec.kind != KIND {
anyhow::bail!("no autofill job {:?}", spec.kind);
}
let Some(reply) = vm.latest() else {
return Ok(serde_json::json!({ "due": false, "why": "unfed" }));
};
let auto = reply.feed.get(autofill::FEED_REPAIR_AUTO).cloned();
if !autofill::auto_on(auto.as_deref()) {
return Ok(serde_json::json!({ "due": false, "why": "auto-fix is not fed" }));
}
let (Some(main), Some(twin)) = (
crate::primary_control(&reply),
crate::twin_addr(&reply).map(|ip| format!("{ip}:{}", crate::CONTROL_PORT)),
) else {
return Ok(
serde_json::json!({ "due": false, "why": "no main and twin in the verdict yet" }),
);
};
let bearers = match crate::console_key::bearers(&reply) {
Ok(Some(b)) => b,
Ok(None) => {
return Ok(
serde_json::json!({ "due": false, "why": "no box has offered its console key yet" }),
)
}
Err(e) => {
w.say_once(&e);
return Ok(serde_json::json!({ "due": false, "why": e }));
}
};
let boxes = HolgerBoxes {
handle: handle.clone(),
bearers: bearers.into(),
timeout: Duration::from_secs(15),
wg_only: crate::wg_only::wg_only(),
};
match fill_once(&boxes, &main, &twin, auto.as_deref()) {
Ok(Ok(words)) => {
tracing::warn!("auto-fix: the re-laid main {words}");
Ok(serde_json::json!({ "due": true, "words": words }))
}
Ok(Err(why)) => Ok(serde_json::json!({ "due": false, "why": why })),
Err(why) => {
w.say_once(&why);
Err(anyhow::anyhow!(why))
}
}
}
/// The look of `generation` (one per generation: a resubmit is a no-op).
pub fn spec(generation: i64) -> ChainSpec {
ChainSpec::new(
KIND,
format!("{KIND}-{generation}"),
KIND,
generation,
"holger-estate-front",
serde_json::json!({ "generation": generation }),
)
.with_target("a re-laid main that awaits its twin")
.with_priority(Priority::High)
}
/// **Start the watch** on the current runtime: the ledger at `<root>/`[`JOBS_REL`], its
/// ChainQueue's worker, and the ticker that queues one look per [`autofill::LOOK_EVERY`].
/// `false` when the ledger cannot open (said).
pub fn start(vm: Arc<FrontVm>, root: &Path) -> bool {
let dir = root.join(JOBS_REL);
let store = match jera::jobs::JobStore::open(&dir) {
Ok(s) => s,
Err(e) => {
tracing::error!(jobs = %dir.display(), "auto-fix: the front's job ledger cannot open ({e:#}): a re-laid main is not filled by the front");
return false;
}
};
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return false;
};
let q = ChainQueue::new(store.sink());
let w = Arc::new(Watch::default());
let h = handle.clone();
// A32-14, stays a task: the autofill lane's jera worker.
handle.spawn(Arc::clone(&q).run(move |spec| run(&w, &vm, &h, &spec)));
// A32-14, stays a task: the autofill lane's ticker (it submits the jera looks).
handle.spawn(async move {
let _store = store;
let mut generation = 0i64;
loop {
tokio::time::sleep(autofill::LOOK_EVERY).await;
generation += 1;
let _ = q.submit(spec(generation));
}
});
true
}
#[cfg(test)]
mod tests {
use super::*;
fn status(fresh: bool, restoring: bool, awaiting: &str) -> proto::RechargeStatusReply {
proto::RechargeStatusReply {
fresh,
restoring,
awaiting_from: awaiting.into(),
..Default::default()
}
}
/// ★ Row 15 (lane G11): the front fills only a main that SAYS it awaits its twin, and only
/// with the opt-in fed — Gunnar's five cases, over Holger's `RechargeStatus`.
#[test]
fn the_front_fills_only_an_awaiting_main_with_auto_fix_fed() {
let awaiting = status(true, false, "10.44.0.3:50051");
assert!(autofill::due(Some("true"), &awaits(&awaiting)));
assert!(
!autofill::due(None, &awaits(&awaiting))
&& !autofill::due(Some("yes"), &awaits(&awaiting)),
"auto-fix not fed"
);
assert!(
autofill::due(
Some("true"),
&awaits(&status(false, true, "10.44.0.3:50051"))
),
"a restore it began is resumed"
);
assert!(
!autofill::due(
Some("true"),
&awaits(&status(false, false, "10.44.0.3:50051"))
),
"a main with data is never streamed into"
);
assert!(
!autofill::due(Some("true"), &awaits(&status(true, false, ""))),
"a fresh main nobody re-admitted as a recharge is not filled"
);
}
/// The front's lane is holger's: the same principals and slots as the box's pulled order.
#[test]
fn the_front_runs_holgers_lane_names() {
assert_eq!(
(
NAMES.recharge_principal,
NAMES.twin_principal,
NAMES.recharge_purpose,
NAMES.twin_purpose
),
("holger-recharge", "holger-twin", "recharge", "twin")
);
assert!(
proof().in_place && !proof().primary_fenced,
"the front fills the re-laid main itself; it never claims a fence"
);
}
}