server/lib/src/clock.rs
//! **holger's wall clock on a box of the estate** — the time every expiry check
//! in this server is taken against, and how it follows the FRONT.
//!
//! The rules are not here. The sources, the quorum, the sanity window, the
//! signed-peer protocol (a nonce this box chose, answered with the front's
//! ed25519 SSHSIG over `(nonce, unix_ms)`), step-at-boot and slew-while-running
//! are edda `estate-clock`, the same library holger's front
//! (`holger-estate-front`) serves signed time with. This module decides only
//! what holger does with an answer:
//!
//! * **The box owns its clock** (PID 1 of its own medium): the answer moves
//! `CLOCK_REALTIME` (`estate_clock::fetch::sync`), so every reader of the
//! system clock — rustls' certificate dates, the twin credential's
//! `notAfter`, this module — is right at once, and nothing is owed.
//! * **It does not** (a container, a developer's host, or a kernel that refused
//! the step): the system clock is somebody else's, and it is NOT moved. The
//! verified offset is kept here instead ([`owed_ms`]) and [`now`] adds it, so
//! holger's own expiry checks still read the front's time.
//!
//! [`now`] is what an expiry check reads, never `SystemTime::now()`, and
//! [`is_trusted`] says whether a verified answer from the front stands behind it
//! — an expiry refused against an unsynchronised clock can say so.
//!
//! Where the configuration comes from: the medium's `korp.env.ESTATE_CLOCK` /
//! `ESTATE_CLOCK_KEY`, merged with the installer's feed ([`set_feed`], every
//! feed): the front offers `clock_peer` + `clock_key`, and a medium value that
//! DISAGREES with the door is refused by name (`pairing.feed_mismatch`), never
//! picked silently.
use crate::lock::LockOrRecover as _;
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
use std::sync::{Mutex, OnceLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
pub use estate_clock::config::{Config, CLOCK_ENV};
pub use estate_clock::offer::{fed_clock, feed_resyncs, last_outcome, outcome_is_right, FedRun};
pub use estate_clock::report::{Outcome, Report, SourceReport, Trigger};
/// The name this clock carries into evidence (an update bundle's verdict, a log).
pub const SOURCE_NAME: &str = "estate-clock";
/// Milliseconds the front's verified time is AHEAD of this machine's system
/// clock and the system clock was not moved (0 when it was, or when nothing
/// verified has answered).
static OWED_MS: AtomicI64 = AtomicI64::new(0);
/// Whether a verified answer from the front stands behind [`now`].
static TRUSTED: AtomicBool = AtomicBool::new(false);
/// **Now, by the estate's clock** — the system clock plus whatever verified
/// correction this process holds and could not apply to the machine.
pub fn now() -> SystemTime {
at(SystemTime::now(), owed_ms())
}
/// `system` corrected by `owed_ms`. Pure.
pub fn at(system: SystemTime, owed_ms: i64) -> SystemTime {
let d = Duration::from_millis(owed_ms.unsigned_abs());
if owed_ms >= 0 {
system.checked_add(d).unwrap_or(system)
} else {
system.checked_sub(d).unwrap_or(system)
}
}
/// [`now`] as whole seconds since the epoch (0 before it).
pub fn now_unix_secs() -> u64 {
now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
/// [`now`] as milliseconds since the epoch (negative before it).
pub fn now_unix_ms() -> i64 {
estate_clock::unix_ms(now())
}
/// The verified correction held in-process (see the module note).
pub fn owed_ms() -> i64 {
OWED_MS.load(Ordering::SeqCst)
}
/// Whether a verified answer from the front stands behind [`now`].
pub fn is_trusted() -> bool {
TRUSTED.load(Ordering::SeqCst)
}
/// **Take a sync's report into this process's view.** A vouching outcome on a
/// box that moved its clock owes nothing; a decided correction the machine's
/// clock did not take (`ApplyFailed`) is owed here; an outcome with no verified
/// answer (no quorum, a disagreement, misconfigured, off) changes nothing — the
/// last verified view stands, advancing with the system clock.
pub fn settle(report: &Report) {
match report.outcome {
Outcome::Stepped | Outcome::Slewed | Outcome::AlreadyRight => {
OWED_MS.store(0, Ordering::SeqCst);
TRUSTED.store(true, Ordering::SeqCst);
}
Outcome::ApplyFailed if !report.sources.is_empty() => {
OWED_MS.store(report.offset_ms, Ordering::SeqCst);
TRUSTED.store(true, Ordering::SeqCst);
}
_ => {}
}
}
/// Forget every verified view (a test's reset; a box never calls it).
pub fn reset() {
OWED_MS.store(0, Ordering::SeqCst);
TRUSTED.store(false, Ordering::SeqCst);
}
fn feed_slot() -> &'static Mutex<BTreeMap<String, String>> {
static FEED: OnceLock<Mutex<BTreeMap<String, String>>> = OnceLock::new();
FEED.get_or_init(|| Mutex::new(BTreeMap::new()))
}
/// **The installer fed this box** — kept, so every later sync reads the front's
/// offer (`offer.<vm>.clock_peer` / `clock_key`). Whether the feed's clock
/// values differ from the feed kept before (`estate_clock::offer::fed_clock_changed`).
pub fn set_feed(feed: &BTreeMap<String, String>) -> bool {
let mut slot = feed_slot().lock_or_recover();
let changed = estate_clock::offer::fed_clock_changed(&slot, feed);
*slot = feed.clone();
changed
}
/// **Whether a refusal says a certificate's DATES are wrong** — a TLS refusal
/// for an expired or not-yet-valid certificate (`estate_clock::trigger::is_tls_date_refusal`),
/// or nordisk-twin's own check of a credential against this box's clock
/// ("outside its validity window"). Between two boxes of one estate whose
/// certificates the front issued, that is a clock wrong on one side: the cue
/// for `estate_clock::trigger::skew_refused` (Gunnar f2e602dd: a twin whose
/// polls the primary refused for skew asks the front).
pub fn refused_for_dates(text: &str) -> bool {
estate_clock::trigger::is_tls_date_refusal(text) || text.contains(TWIN_VALIDITY_WORDS)
}
/// nordisk-twin's `TwinError::Expired` words.
const TWIN_VALIDITY_WORDS: &str = "outside its validity window";
/// **Refusals seen by this box** (a twin's follow cycle): when one of them is
/// [`refused_for_dates`], one sync is asked of the front
/// (`estate_clock::trigger::skew_refused`: rate-limited, carrying nothing from
/// the refusal). Whether a sync was started.
pub fn saw_refusals<'a>(texts: impl IntoIterator<Item = &'a str>) -> bool {
texts.into_iter().any(refused_for_dates) && estate_clock::trigger::skew_refused(0)
}
/// **The clock configuration in force now**: the medium's environment merged
/// with the last feed. Read on EVERY sync.
pub fn config_now() -> Result<estate_clock::config::Config, String> {
let feed = feed_slot().lock_or_recover().clone();
estate_clock::offer::box_config(&feed)
}
/// **One sync.** `owns_clock`: this process may move the machine's clock (PID 1
/// of its own medium). Never fails and never panics: every outcome is a
/// [`Report`], recorded (`estate_clock::report::last`) and settled into
/// [`now`].
#[cfg(feature = "clock-sync")]
pub fn sync(
config: &Result<estate_clock::config::Config, String>,
phase: Trigger,
owns_clock: bool,
) -> Report {
// Offline: `measure` refuses a configured source by name, before any dial.
let report = if owns_clock && !crate::egress::is_offline() {
estate_clock::fetch::sync(config, phase)
} else {
measure(config, phase)
};
settle(&report);
report
}
/// **One measuring sync** — the machine's clock is never moved, the verified
/// offset is owed in-process ([`now`]). What a process that does not own the
/// clock runs (the `bundle import` / `scrub` CLI), on every build that follows a
/// clock: with `clock-sync` it asks HTTPS sources too; with `clock-follow`
/// alone (a zero-egress build) it asks the signed peer and SNTP only, and an
/// `https://` source fails by name.
#[cfg(feature = "clock-follow")]
pub fn measure_once(
config: &Result<estate_clock::config::Config, String>,
phase: Trigger,
) -> Report {
let report = measure(config, phase);
settle(&report);
report
}
/// The last sync's one-line summary, for a refusal that has to say why no
/// verified time stands behind the clock.
pub fn last_summary() -> Option<String> {
estate_clock::report::last().map(|r| {
let mut line = r.summary();
for row in r.sources.iter().filter(|row| row.error.is_some()) {
line.push_str(&format!(
"; {} FAILED: {}",
row.source,
row.error.as_deref().unwrap_or_default()
));
}
line
})
}
/// The sync of a process that does not own the machine's clock: the same poll,
/// the same decision, and the system clock left alone — the offset is owed
/// in-process instead.
#[cfg(feature = "clock-follow")]
fn measure(config: &Result<estate_clock::config::Config, String>, phase: Trigger) -> Report {
use estate_clock::config::Config;
let report = match config {
Err(why) => Report::without_sources(phase, Outcome::Misconfigured, why.clone()),
Ok(Config::NotConfigured) => Report::without_sources(phase, Outcome::NotConfigured, format!("no clock source is configured ({} or the front's offer); holger's clock is the system clock, unverified", estate_clock::config::CLOCK_ENV)),
Ok(Config::Off) => Report::without_sources(phase, Outcome::Off, format!("{}=off: this box was told not to follow a clock", estate_clock::config::CLOCK_ENV)),
Ok(Config::Sources { .. }) if crate::egress::is_offline() => Report::without_sources(
phase,
Outcome::Misconfigured,
crate::egress::check("clock sync", "the configured clock sources").err().map(|e| e.to_string()).unwrap_or_default(),
),
Ok(Config::Sources { sources, peer_key }) => {
#[cfg(feature = "clock-sync")]
let polled = estate_clock::fetch::poll(sources, peer_key.as_deref());
#[cfg(not(feature = "clock-sync"))]
let polled = estate_clock::poll::poll(sources, peer_key.as_deref());
let action = estate_clock::adjust::choose(phase, &polled.verdict.decision);
let mut report = Report::from_poll(phase, &polled, action, Err(NOT_OURS.to_string()));
if report.outcome == Outcome::ApplyFailed {
report.detail = format!("holger's clock carries {:+} ms from {}; the machine's clock was not moved: {NOT_OURS}", report.offset_ms, report.sources.iter().filter(|s| s.error.is_none()).map(|s| s.source.as_str()).collect::<Vec<_>>().join(", "));
}
report
}
};
estate_clock::report::record(report.clone());
report
}
#[cfg(feature = "clock-follow")]
const NOT_OURS: &str =
"this process does not own the machine's clock (not PID 1 of its own medium)";
/// **Arm the running triggers, once per process** (`estate_clock::trigger::arm`):
/// a skew refusal and the 6-hour safety timer each start one [`sync`]. A fed
/// run arms them BEFORE its first pause (`estate_clock::offer::FedRun`, Gunnar
/// 1115a4e6); before G3 holger armed only the timer and never installed the
/// skew handler. Whether this call armed them.
#[cfg(feature = "clock-sync")]
pub fn arm_triggers(owns_clock: bool, log: fn(&Report)) -> bool {
estate_clock::trigger::arm(move |phase| log(&sync(&config_now(), phase, owns_clock)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_estate_clock_adds_what_is_owed_in_either_direction() {
let t = UNIX_EPOCH + Duration::from_secs(1_790_000_000);
assert_eq!(at(t, 7_200_000), t + Duration::from_secs(7_200));
assert_eq!(at(t, -7_200_000), t - Duration::from_secs(7_200));
assert_eq!(at(t, 0), t);
}
/// Gunnar f2e602dd, holger's shape: a twin's poll refused for certificate DATES (by
/// either side of the mutual TLS, or by nordisk-twin's own check against this clock)
/// asks the front; any other refusal does not.
#[test]
fn a_refusal_for_certificate_dates_is_told_from_any_other() {
for dates in [
"estate over tls: https://10.44.0.2:8443: invalid peer certificate: certificate expired: verification time 1790000000 (UNIX), but certificate is not valid after 1789990000 (10000 seconds ago)",
"transport error: received fatal alert: CertificateExpired",
"[holger-twin-3] the certificate is outside its validity window: notAfter 1 is not after now 2",
] {
assert!(refused_for_dates(dates), "{dates}");
}
for other in [
"invalid peer certificate: UnknownIssuer",
"connection refused",
"the primary is not answering",
] {
assert!(!refused_for_dates(other), "{other}");
}
}
/// A feed says whether its clock changed, so a repeat of the same clock is told from a
/// new one (`estate_clock::offer::feed_resyncs`).
#[test]
fn a_feed_says_whether_its_clock_changed() {
let feed = |peer: &str| {
BTreeMap::from([
("offer.vm1.clock_peer".to_string(), peer.to_string()),
(
"offer.vm1.clock_key".to_string(),
"ssh-ed25519 AAAAk".to_string(),
),
])
};
set_feed(&BTreeMap::new());
assert!(
set_feed(&feed("peer://10.44.0.1:50123")),
"the first clock is a change"
);
assert!(
!set_feed(&feed("peer://10.44.0.1:50123")),
"the same clock re-fed is no change"
);
assert!(
set_feed(&feed("peer://10.44.0.9:50123")),
"a new peer is a change"
);
set_feed(&BTreeMap::new());
}
}