crates/estate-clock/src/adjust.rs
//! **Moving the clock**: when to step, when to slew, and the two syscalls.
//!
//! # The rule
//!
//! | phase | offset | action |
//! |---|---|---|
//! | any | `< MIN_CORRECTION_MS` (2 s) | leave it — inside the source's own rounding |
//! | boot | any larger | **step** |
//! | running | `<= MAX_SLEW_MS` (30 s) | **slew** |
//! | running | larger | **step**, reported as a live step |
//!
//! **At boot a step is free.** Nothing has been issued, stamped or remembered
//! yet: the listeners are not bound (`gunnar-server` runs the boot sync before
//! any of them), so there is no timestamp for a jump to make inconsistent.
//!
//! **On a running server a step is not free**, and the reason is concrete.
//! `gunnar_grpc::auth::ReplayGuard` remembers a credential against the server's
//! clock at the moment it accepted it, and forgets entries older than
//! `now - MAX_CLOCK_SKEW_MS`. Step that clock **forward** by `d` and every entry
//! is forgotten `d` early, while the credential's own `issued_at` is still inside
//! the window — a replay gap of up to `d`. Step it **backward** and recent
//! `issued_at` values look future-dated, and anything ordered by server time
//! goes out of order. A slew (`adjtimex(ADJ_OFFSET_SINGLESHOT)`) moves the clock
//! at most 500 ppm — half a millisecond per second — so neither effect exists:
//! time never jumps and never runs backwards.
//!
//! **Why 30 s is the slew ceiling.** At 500 ppm a 30 s correction completes in
//! 60 000 s, under 17 hours, and the clock is inside the 300 s credential
//! window for the whole of it. The drift this is actually for is far smaller: a
//! KVM guest on kvmclock drifts in the tens of ppm, well under a second per
//! hour, so an hourly re-sync slews milliseconds. An offset above 30 s on a
//! running server is not drift — it is a host migration, a suspend, or a
//! hypervisor setting the RTC — and a slew would leave the clock wrong for days.
//!
//! **Why above it we step rather than refuse.** A clock outside the credential
//! window refuses *every* genuine credential and every Stripe webhook: a total
//! outage until the next re-image, which can be months. A step's replay gap is
//! bounded by `d` and, independently, by the five-minute window itself (a
//! credential older than five minutes by the *corrected* clock is refused
//! whatever the guard remembers). Of the two harms, the bounded one is taken,
//! and the report says `stepped (live)` so it is never silent.
//!
//! # The front's interval
//!
//! [`PERIODIC_INTERVAL`]: one hour, the FRONT's default (`ESTATE_CLOCK_EVERY`).
//! The appliance does not use it: it re-measures only on a skew refusal and a
//! 6-hour safety timer (`crate::trigger`). Against a 300 s window and tens of ppm of
//! drift that is three orders of magnitude of margin; more often would only add
//! exchanges to a peer and log lines to a report.
use std::time::Duration;
use crate::decide::{Decision, MIN_CORRECTION_MS};
use crate::report::Trigger;
/// The largest offset slewed on a running server. See the module note.
pub const MAX_SLEW_MS: i64 = 30_000;
/// How often a running server re-measures. See the module note.
pub const PERIODIC_INTERVAL: Duration = Duration::from_secs(3_600);
/// What to do about a decision.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Action {
/// Nothing: no decision to act on, or the clock is already right.
Leave,
/// Slew by this many milliseconds.
Slew(i64),
/// Step by this many milliseconds.
Step(i64),
}
/// The rule in the table above. Pure.
pub fn choose(phase: Trigger, decision: &Decision) -> Action {
let Decision::Correct { offset_ms, .. } = decision else {
return Action::Leave;
};
let offset_ms = *offset_ms;
if offset_ms.abs() < MIN_CORRECTION_MS {
return Action::Leave;
}
match phase {
Trigger::Boot => Action::Step(offset_ms),
Trigger::SkewRefusal | Trigger::Timer if offset_ms.abs() <= MAX_SLEW_MS => {
Action::Slew(offset_ms)
}
Trigger::SkewRefusal | Trigger::Timer => Action::Step(offset_ms),
}
}
/// `(tv_sec, tv_nsec)` for an epoch millisecond, Euclidean so a negative
/// millisecond still yields the non-negative nanosecond field the kernel demands.
pub fn timespec_parts(target_ms: i64) -> (i64, i64) {
(
target_ms.div_euclid(1_000),
target_ms.rem_euclid(1_000) * 1_000_000,
)
}
/// Carry out an action. Needs `CAP_SYS_TIME`; `EPERM` comes back as an error
/// for the caller to report, never a panic.
#[cfg(feature = "adjust")]
pub fn apply(action: Action) -> std::io::Result<()> {
match action {
Action::Leave => Ok(()),
Action::Step(offset_ms) => step(offset_ms),
Action::Slew(offset_ms) => slew(offset_ms),
}
}
/// Step `CLOCK_REALTIME` by `offset_ms`, relative to the clock as it reads *now*
/// (the samples are seconds old, and those were real seconds).
/// `rustix::time::clock_settime`: a raw syscall written in Rust.
#[cfg(feature = "adjust")]
fn step(offset_ms: i64) -> std::io::Result<()> {
use rustix::time::{clock_settime, ClockId, Timespec};
let (tv_sec, tv_nsec) = timespec_parts(crate::now_unix_ms() + offset_ms);
clock_settime(
ClockId::Realtime,
Timespec {
tv_sec,
tv_nsec: tv_nsec as _,
},
)
.map_err(|errno| std::io::Error::from_raw_os_error(errno.raw_os_error()))
}
/// Slew by `offset_ms`: `adjtimex(ADJ_OFFSET_SINGLESHOT)`, the kernel's
/// `adjtime(3)`. A new call REPLACES any slew still in progress rather than
/// adding to it, which is exactly right for a re-measurement: the new offset is
/// the whole remaining truth.
///
/// FFI, and the one place in the crate: rustix 1.1.4 has no `adjtimex`, and no
/// crate in this graph wraps it. See the manifest note on why `libc` costs no
/// package.
#[cfg(feature = "adjust")]
fn slew(offset_ms: i64) -> std::io::Result<()> {
// SAFETY: `timex` is plain old data; all-zero is a valid value for every
// field. `adjtimex` reads `modes` and `offset` and writes the struct back.
let mut tx: libc::timex = unsafe { std::mem::zeroed() };
tx.modes = libc::ADJ_OFFSET_SINGLESHOT as _;
tx.offset = (offset_ms * 1_000) as _;
// SAFETY: `tx` is a live, exclusively borrowed `timex` for the whole call.
let rc = unsafe { libc::adjtimex(&mut tx) };
if rc == -1 {
Err(std::io::Error::last_os_error())
} else {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn correct(offset_ms: i64) -> Decision {
Decision::Correct {
offset_ms,
agreed: vec!["peer://10.13.8.99:50123".into()],
}
}
#[test]
fn the_boot_offset_is_stepped() {
assert_eq!(
choose(Trigger::Boot, &correct(-7_200_000)),
Action::Step(-7_200_000)
);
assert_eq!(choose(Trigger::Boot, &correct(5_000)), Action::Step(5_000));
}
#[test]
fn a_small_offset_on_a_running_server_is_slewed_in_either_direction() {
assert_eq!(choose(Trigger::Timer, &correct(2_500)), Action::Slew(2_500));
assert_eq!(
choose(Trigger::Timer, &correct(-2_500)),
Action::Slew(-2_500)
);
assert_eq!(
choose(Trigger::Timer, &correct(MAX_SLEW_MS)),
Action::Slew(MAX_SLEW_MS)
);
}
#[test]
fn a_large_offset_on_a_running_server_is_stepped_not_left_for_days() {
assert_eq!(
choose(Trigger::Timer, &correct(MAX_SLEW_MS + 1)),
Action::Step(MAX_SLEW_MS + 1)
);
assert_eq!(
choose(Trigger::Timer, &correct(-7_200_000)),
Action::Step(-7_200_000)
);
}
#[test]
fn a_failed_or_unanimous_poll_moves_nothing() {
for phase in [Trigger::Boot, Trigger::Timer] {
assert_eq!(
choose(phase, &Decision::NoQuorum { had: 0, needed: 1 }),
Action::Leave
);
assert_eq!(
choose(
phase,
&Decision::Disagree {
spread_ms: 1,
offsets: vec![]
}
),
Action::Leave
);
assert_eq!(
choose(
phase,
&Decision::AlreadyRight {
offset_ms: 900,
agreed: vec![]
}
),
Action::Leave
);
}
}
#[test]
fn the_slew_ceiling_completes_inside_a_day_at_the_kernels_rate() {
// 500 ppm is the kernel's adjtime rate; the ceiling is chosen so a
// maximal slew finishes the same day and never leaves the window.
let seconds_to_complete = MAX_SLEW_MS as f64 / 1_000.0 / 500e-6;
assert!(seconds_to_complete < 86_400.0);
const { assert!(MAX_SLEW_MS < 300_000) };
}
#[test]
fn a_negative_epoch_millisecond_still_yields_a_legal_timespec() {
for ms in [-1_i64, -999, -1_000, -1_001, 0, 1, 999, 1_000] {
let (sec, nsec) = timespec_parts(ms);
assert!((0..1_000_000_000).contains(&nsec), "tv_nsec for {ms} ms");
assert_eq!(sec * 1_000 + nsec / 1_000_000, ms);
}
}
}