crates/monetize-admin/src/offsite.rs
//! **The ONE box's off-box copy** (BK-0, backup audit 2026-10-11): what the box
//! must not lose leaves it, sealed, on a schedule of its own.
//!
//! The ONE box (monetize.ignalina.dk) has ONE disk. On it: the ledger (the
//! money record), the signing root every product key is derived from and every
//! estate trusts, and the custody store (every sold estate's seeds and CA keys,
//! sealed under `custody/kek`). The UpCloud daily backup rule (the kit's
//! terraform) keeps the disk's history inside UpCloud; THIS keeps a copy off
//! UpCloud:
//!
//! 1. monetize exports (`Ledger.Export` on loopback, the hourly export kept
//! when it cannot answer) into `monetize/backup`;
//! 2. [`pack`]: the export (`export/…`), everything under `monetize/data`
//! but the stores the export already carries (`data/…`: `signing.key`,
//! `custody/kek`, `custody/*.sealed`, the journals), the identities
//! (`identity/…`) — one deterministic bundle (`ledger_backup::encode`);
//! 3. [`seal`]: age, to the X25519 recipients the kit fed
//! ([`FEED_RECIPIENTS`] — the off-site key whose private half is only in the
//! vault). NOTHING leaves the box in plaintext;
//! 4. PUT to the S3 store ([`R2`], SigV4, path style) under the fed prefix
//! (`one-monetize/<box>/`), then HEAD: shipped only when the store holds
//! every byte;
//! 5. [`STATUS_REL`] says when, what and why not.
//!
//! The way back: `monetize-admin offsite open --identity <file> --archive
//! <file> --into <empty dir>` ([`open_archive`]) decrypts, imports the export
//! into scratch stores under the bundled signing key ([`verify`]), and only
//! then writes the files out. A key, a secret, a credential is never said.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::Duration;
use monetize_front::{intake, ledger_backup as lb};
use monetize_server::backup as mb;
/// The X25519 age recipients the copy is sealed to (whitespace/comma separated).
pub const FEED_RECIPIENTS: &str = "offsite_recipients";
/// The S3 endpoint, `https://<account>.r2.cloudflarestorage.com`.
pub const FEED_ENDPOINT: &str = "offsite_r2_endpoint";
/// The store's access key id (a secret: never said).
pub const FEED_ACCESS_KEY: &str = "offsite_r2_access_key";
/// The store's secret key (never said).
pub const FEED_SECRET_KEY: &str = "offsite_r2_secret_key";
/// The bucket ([`DEFAULT_BUCKET`] when not fed).
pub const FEED_BUCKET: &str = "offsite_bucket";
/// The prefix the objects are put under (`one-monetize/<box>`).
pub const FEED_PREFIX: &str = "offsite_prefix";
/// How often, in minutes ([`DEFAULT_EVERY_MINUTES`] when not fed).
pub const FEED_EVERY: &str = "offsite_every_minutes";
/// Every feed key of the off-box copy.
pub const FEED_KEYS: [&str; 7] = [
FEED_RECIPIENTS,
FEED_ENDPOINT,
FEED_ACCESS_KEY,
FEED_SECRET_KEY,
FEED_BUCKET,
FEED_PREFIX,
FEED_EVERY,
];
/// The keys without which nothing is shipped.
pub const FEED_REQUIRED: [&str; 5] = [
FEED_RECIPIENTS,
FEED_ENDPOINT,
FEED_ACCESS_KEY,
FEED_SECRET_KEY,
FEED_PREFIX,
];
/// The shared bucket (`yvra-backups`), the one-monetize objects under their own prefix.
pub const DEFAULT_BUCKET: &str = "yvra-backups";
pub const DEFAULT_EVERY_MINUTES: u64 = 360;
pub const MIN_EVERY_MINUTES: u64 = 15;
/// The status of the last attempt, under the data root (0600).
pub const STATUS_REL: &str = "offsite/status.toml";
/// Object names: `<prefix>one-monetize-<unix, 11 digits>-<digest, 32 hex>.mfb.age`.
pub const OBJECT_PREFIX: &str = "one-monetize-";
pub const OBJECT_SUFFIX: &str = ".mfb.age";
/// An unchanged state is shipped again after a day all the same.
pub const RESEND_AFTER_SECS: u64 = 24 * 3600;
/// The bundle the box holds in memory, at most.
pub const MAX_PLAIN_BYTES: usize = 1 << 30;
/// The bundle's format entry.
pub const FORMAT_ENTRY: &str = "OFFSITE";
pub const FORMAT: &[u8] = b"one-monetize-offsite-1\n";
const EXPORT: &str = "export/";
const DATA: &str = "data/";
const IDENTITY: &str = "identity/";
/// The stores the export carries: packed as the export, never as raw files.
const EXPORTED_STORES: [&str; 4] = [
"ledger.redb",
"usage.redb",
"signups.redb",
"reconcile.redb",
];
/// A value that is never said (`Debug` shows `<secret>`).
#[derive(Clone, PartialEq, Eq)]
pub struct Secret(String);
impl Secret {
pub fn new(s: impl Into<String>) -> Secret {
Secret(s.into())
}
pub fn expose(&self) -> &str {
&self.0
}
}
impl std::fmt::Debug for Secret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("<secret>")
}
}
/// **The off-box copy's settings**, out of the feed ([`Config::of`]).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Config {
pub recipients: Vec<String>,
/// `https://<host>`, no path.
pub endpoint: String,
pub access_key: Secret,
pub secret_key: Secret,
pub bucket: String,
/// Ends in `/`, never starts with one.
pub prefix: String,
pub every: Duration,
}
impl Config {
/// Pure: the settings the feed gives, else the refusal — by key NAME, never a value.
pub fn of(feed: &BTreeMap<String, String>) -> Result<Config, String> {
let get = |k: &str| feed.get(k).map(|v| v.trim()).filter(|v| !v.is_empty());
if FEED_KEYS.iter().all(|k| get(k).is_none()) {
return Err(format!(
"[offsite-unconfigured] the kit fed no off-box copy ({}): nothing of this box leaves UpCloud — the kit's solo.toml [backup] and its R2 credentials feed it",
FEED_REQUIRED.join(", ")
));
}
let missing: Vec<&str> = FEED_REQUIRED
.iter()
.copied()
.filter(|k| get(k).is_none())
.collect();
if !missing.is_empty() {
return Err(format!(
"[offsite-config] the feed lacks {}: nothing is shipped until the kit feeds it",
missing.join(", ")
));
}
let need = |k: &str| get(k).unwrap_or_default();
let recipients = parse_recipients(need(FEED_RECIPIENTS))?;
let endpoint = endpoint_of(need(FEED_ENDPOINT))?;
let prefix = prefix_of(need(FEED_PREFIX))?;
let bucket = get(FEED_BUCKET).unwrap_or(DEFAULT_BUCKET);
if bucket.len() < 3
|| bucket.len() > 63
|| !bucket
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-' || b == b'.')
{
return Err(format!("[offsite-bucket] {FEED_BUCKET} `{bucket}` is not a bucket name (3–63 of a-z, 0-9, `-`, `.`)"));
}
let every = match get(FEED_EVERY) {
None => DEFAULT_EVERY_MINUTES,
Some(v) => v
.parse::<u64>()
.ok()
.filter(|m| *m >= MIN_EVERY_MINUTES)
.ok_or_else(|| format!("[offsite-every] {FEED_EVERY} `{v}` is not whole minutes of at least {MIN_EVERY_MINUTES}"))?,
};
Ok(Config {
recipients,
endpoint,
access_key: Secret::new(need(FEED_ACCESS_KEY)),
secret_key: Secret::new(need(FEED_SECRET_KEY)),
bucket: bucket.to_string(),
prefix,
every: Duration::from_secs(every * 60),
})
}
}
fn endpoint_of(v: &str) -> Result<String, String> {
let v = v.trim_end_matches('/');
let ok = v
.strip_prefix("https://")
.is_some_and(|h| !h.is_empty() && !h.contains(['/', '@', '?', '#', ' ']));
if ok {
Ok(v.to_string())
} else {
Err(format!("[offsite-endpoint] {FEED_ENDPOINT} must be `https://<host>` — no other scheme, no path, user or query"))
}
}
fn prefix_of(v: &str) -> Result<String, String> {
let p = v.trim_matches('/');
let ok = !p.is_empty()
&& !p.contains("..")
&& !p.contains("//")
&& p.bytes()
.all(|b| b.is_ascii_alphanumeric() || b"._/-".contains(&b));
if ok {
Ok(format!("{p}/"))
} else {
Err(format!("[offsite-prefix] {FEED_PREFIX} `{v}` is not a prefix (A-Z a-z 0-9 . _ - /, no `..`, no empty part)"))
}
}
/// Pure: the X25519 recipients in `text`, refused by position when one is not.
pub fn parse_recipients(text: &str) -> Result<Vec<String>, String> {
let mut out = Vec::new();
let words = text
.lines()
.filter(|l| !l.trim_start().starts_with('#'))
.flat_map(|l| l.split(|c: char| c.is_whitespace() || c == ','))
.filter(|w| !w.is_empty());
for (i, w) in words.enumerate() {
if w.parse::<age::x25519::Recipient>().is_err() {
let head: String = w.chars().take(12).collect();
return Err(format!(
"[offsite-recipient] recipient #{} (`{head}…`) is not an X25519 age recipient: a plugin recipient (age1yubikey…, age1tpm…) cannot be sealed to on the box — feed the off-site key's public half (age1…)",
i + 1
));
}
out.push(w.to_string());
}
if out.is_empty() {
return Err("[offsite-recipient] no recipient: the copy would be sealed to nobody".into());
}
Ok(out)
}
// ── the bundle ───────────────────────────────────────────────────────────────
fn read(p: &Path) -> std::io::Result<Vec<u8>> {
std::fs::read(p)
}
/// **What leaves the box**, read from `root` (module docs, step 2). The export's
/// manifest and the signing key are required.
pub fn pack(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> {
let mut e = BTreeMap::new();
e.insert(FORMAT_ENTRY.to_string(), FORMAT.to_vec());
let dir = root.join(lb::BACKUP_REL);
for f in lb::FILES {
match read(&dir.join(f)) {
Ok(b) => {
e.insert(format!("{EXPORT}{f}"), b);
}
Err(x) if f == mb::MANIFEST => {
return Err(format!(
"[offsite-export] {}: {x} — monetize wrote no export to copy",
dir.join(f).display()
))
}
Err(_) => {}
}
}
let data = root.join(intake::DATA_REL);
if !data.join("signing.key").is_file() {
return Err(format!("[offsite-signing] {}: the signing root is not there — a copy without it restores a ledger no estate trusts", data.join("signing.key").display()));
}
walk(&data, "", &mut e)?;
let ids = root.join(intake::MONETIZE_REL);
if let Ok(rd) = std::fs::read_dir(&ids) {
let mut names: Vec<(String, PathBuf)> = rd
.flatten()
.filter_map(|x| {
let n = x.file_name().to_string_lossy().to_string();
(n.starts_with("identity-")
&& !n.ends_with(".pub")
&& !n.contains(".staging")
&& x.path().is_file())
.then(|| (n, x.path()))
})
.collect();
names.sort();
for (n, p) in names {
e.insert(
format!("{IDENTITY}{n}"),
read(&p).map_err(|x| format!("[offsite-pack] {}: {x}", p.display()))?,
);
}
}
let total: usize = e.iter().map(|(k, v)| k.len() + v.len()).sum();
if total > MAX_PLAIN_BYTES {
return Err(format!(
"[offsite-pack] {total} bytes: over the {MAX_PLAIN_BYTES} this copy holds in memory"
));
}
Ok(e)
}
/// Every file under `dir` into `out` as `data/<rel>`: not the stores the export
/// carries, not a staged half-write, not a link.
fn walk(dir: &Path, rel: &str, out: &mut BTreeMap<String, Vec<u8>>) -> Result<(), String> {
let mut items: Vec<_> = std::fs::read_dir(dir)
.map_err(|x| format!("[offsite-pack] {}: {x}", dir.display()))?
.flatten()
.collect();
items.sort_by_key(|x| x.file_name());
for x in items {
let name = x.file_name().to_string_lossy().to_string();
if name.contains(".staging") || (rel.is_empty() && EXPORTED_STORES.contains(&name.as_str()))
{
continue;
}
let r = if rel.is_empty() {
name
} else {
format!("{rel}/{}", name)
};
let p = x.path();
let Ok(m) = std::fs::symlink_metadata(&p) else {
continue;
};
if m.is_dir() {
walk(&p, &r, out)?;
} else if m.is_file() {
out.insert(
format!("{DATA}{r}"),
read(&p).map_err(|e| format!("[offsite-pack] {}: {e}", p.display()))?,
);
}
}
Ok(())
}
/// **Seal** `plain` to `recipients` (age, X25519).
pub fn seal(recipients: &[String], plain: &[u8]) -> Result<Vec<u8>, String> {
use std::io::Write as _;
let rs = recipients
.iter()
.map(|r| {
r.parse::<age::x25519::Recipient>().map_err(|_| {
"[offsite-recipient] a recipient is not an X25519 age recipient".to_string()
})
})
.collect::<Result<Vec<_>, _>>()?;
let enc = age::Encryptor::with_recipients(rs.iter().map(|r| r as &dyn age::Recipient))
.map_err(|e| format!("[offsite-seal] {e}"))?;
let mut out = Vec::with_capacity(plain.len() + 1024);
let mut w = enc
.wrap_output(&mut out)
.map_err(|e| format!("[offsite-seal] {e}"))?;
w.write_all(plain)
.map_err(|e| format!("[offsite-seal] {e}"))?;
w.finish().map_err(|e| format!("[offsite-seal] {e}"))?;
Ok(out)
}
/// **Open** a sealed copy with an age identity file's text. The identity is never said.
pub fn open(identity: &str, sealed: &[u8]) -> Result<Vec<u8>, String> {
use std::io::Read as _;
let ids = age::IdentityFile::from_buffer(identity.as_bytes())
.ok()
.and_then(|f| f.into_identities().ok())
.filter(|v| !v.is_empty())
.ok_or_else(|| "[offsite-open] the identity is not an age identity file".to_string())?;
let d = age::Decryptor::new(sealed)
.map_err(|e| format!("[offsite-open] not an age-sealed copy: {e}"))?;
let mut r = d
.decrypt(ids.iter().map(|i| i.as_ref() as &dyn age::Identity))
.map_err(|e| format!("[offsite-open] this identity does not open the copy: {e}"))?;
let mut out = Vec::new();
r.read_to_end(&mut out)
.map_err(|e| format!("[offsite-open] the copy is cut or changed: {e}"))?;
Ok(out)
}
/// What a verified copy holds.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Verified {
pub entries: usize,
pub ledger_rows: u64,
pub signups: u64,
pub custody_sealed: usize,
/// The bundled signing key's public half (hex).
pub signing_public_key: String,
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
/// **Prove a copy restores**: its export is imported into fresh stores under
/// `scratch` under the bundled signing key (monetize refuses an export signed
/// by another), the custody's KEK is there when sealed custody is. `scratch` is
/// removed after, whatever the answer.
pub fn verify(entries: &BTreeMap<String, Vec<u8>>, scratch: &Path) -> Result<Verified, String> {
let r = verify_in(entries, scratch);
let _ = std::fs::remove_dir_all(scratch);
r
}
fn mkdir(p: &Path) -> Result<(), String> {
use std::os::unix::fs::DirBuilderExt as _;
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(p)
.map_err(|e| format!("{}: {e}", p.display()))
}
fn verify_in(entries: &BTreeMap<String, Vec<u8>>, scratch: &Path) -> Result<Verified, String> {
let bad = |w: String| format!("[offsite-verify] {w}");
if entries.get(FORMAT_ENTRY).map(Vec::as_slice) != Some(FORMAT) {
return Err(bad("not a one-monetize off-box copy".into()));
}
let sk = entries
.get(&format!("{DATA}signing.key"))
.ok_or_else(|| bad("no signing root (data/signing.key) in the copy".into()))?;
let sealed = entries
.keys()
.filter(|k| k.starts_with("data/custody/") && k.ends_with(".sealed"))
.count();
if sealed > 0 && !entries.contains_key("data/custody/kek") {
return Err(bad(format!("{sealed} sealed custody file(s) and no kek (data/custody/kek): the estates' seeds would not open")));
}
let export = scratch.join("export");
let data = scratch.join("data");
mkdir(&export).map_err(bad)?;
mkdir(&data).map_err(bad)?;
for (k, v) in entries {
if let Some(f) = k.strip_prefix(EXPORT) {
if f.contains('/') || f.contains("..") {
return Err(bad(format!("`{k}` is not a file the export writes")));
}
std::fs::write(export.join(f), v).map_err(|e| bad(format!("{f}: {e}")))?;
}
}
if !export.join(mb::MANIFEST).is_file() {
return Err(bad("no export manifest in the copy".into()));
}
let cfg = monetize_server::config::Config {
data_dir: data.clone(),
..Default::default()
};
std::fs::write(cfg.signing_key_path(), sk).map_err(|e| bad(format!("signing.key: {e}")))?;
let signer = monetize::Signer::from_path(&cfg.signing_key_path())
.map_err(|e| bad(format!("the signing root does not read: {e}")))?;
let pk = hex(&signer.public_key());
let ledger =
monetize::Ledger::open(&cfg.ledger_path()).map_err(|e| bad(format!("ledger: {e}")))?;
let usage = monetize_server::state::UsageStore::open(&cfg.usage_path())
.map_err(|e| bad(format!("usage: {e}")))?;
let signups = monetize_server::signup::SignupStore::open(&cfg.signup_path())
.map_err(|e| bad(format!("signups: {e}")))?;
let reconcile = monetize_server::reconcile::ReconcileStore::open(&cfg.reconcile_path())
.map_err(|e| bad(format!("reconcile: {e}")))?;
let nonpay = monetize_server::nonpay::NonpayJournal::open(cfg.nonpay_path())
.map_err(|e| bad(format!("nonpay: {e}")))?;
let stores = mb::Stores {
ledger: &ledger,
usage: &usage,
signups: &signups,
reconcile: &reconcile,
nonpay: &nonpay,
};
let m = mb::import(&export, &stores, &pk, false).map_err(|e| {
bad(format!(
"the export does not import under the bundled signing root: {e}"
))
})?;
Ok(Verified {
entries: entries.len(),
ledger_rows: m.ledger_rows as u64,
signups: m.signups as u64,
custody_sealed: sealed,
signing_public_key: pk,
})
}
// ── the store ────────────────────────────────────────────────────────────────
/// **Where the copy goes**: written once, its size read back.
pub trait Remote: Send + Sync {
/// Where, for the lines that say so (never a credential).
fn describe(&self) -> String;
fn put(&self, key: &str, bytes: &[u8]) -> Result<(), String>;
/// The size held under `key`, `None` when nothing is.
fn size(&self, key: &str) -> Result<Option<u64>, String>;
}
/// An in-memory [`Remote`] (tests).
#[derive(Default)]
pub struct MemRemote(pub std::sync::Mutex<BTreeMap<String, Vec<u8>>>);
impl Remote for MemRemote {
fn describe(&self) -> String {
"memory".into()
}
fn put(&self, key: &str, bytes: &[u8]) -> Result<(), String> {
self.0
.lock()
.unwrap_or_else(|p| p.into_inner())
.insert(key.to_string(), bytes.to_vec());
Ok(())
}
fn size(&self, key: &str) -> Result<Option<u64>, String> {
Ok(self
.0
.lock()
.unwrap_or_else(|p| p.into_inner())
.get(key)
.map(|b| b.len() as u64))
}
}
/// Pure: an S3 key, URI-encoded as SigV4 wants it (`/` kept).
pub fn uri_encode(key: &str) -> String {
let mut out = String::with_capacity(key.len());
for b in key.bytes() {
if b.is_ascii_alphanumeric() || b"-._~/".contains(&b) {
out.push(b as char);
} else {
out.push_str(&format!("%{b:02X}"));
}
}
out
}
/// Pure: `YYYYMMDDTHHMMSSZ` of a unix time.
pub fn amz_date(unix: u64) -> String {
let days = (unix / 86_400) as i64;
let s = unix % 86_400;
let z = days + 719_468;
let era = z.div_euclid(146_097);
let doe = z - era * 146_097;
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let y = yoe + era * 400 + i64::from(m <= 2);
format!(
"{y:04}{m:02}{d:02}T{:02}{:02}{:02}Z",
s / 3600,
s % 3600 / 60,
s % 60
)
}
/// One request to sign (AWS Signature Version 4).
pub struct SigV4<'a> {
pub method: &'a str,
/// Already URI-encoded.
pub path: &'a str,
pub query: &'a str,
/// Lower-case names; every one is signed.
pub headers: &'a [(&'a str, &'a str)],
pub payload_sha256: &'a str,
pub amz_date: &'a str,
pub region: &'a str,
pub service: &'a str,
}
fn sha256_hex(b: &[u8]) -> String {
hex(ring::digest::digest(&ring::digest::SHA256, b).as_ref())
}
fn hmac(key: &[u8], msg: &[u8]) -> Vec<u8> {
ring::hmac::sign(&ring::hmac::Key::new(ring::hmac::HMAC_SHA256, key), msg)
.as_ref()
.to_vec()
}
/// Pure: the `Authorization` header of `req`.
pub fn authorization(req: &SigV4<'_>, access_key: &str, secret_key: &str) -> String {
let mut hs: Vec<(String, String)> = req
.headers
.iter()
.map(|(n, v)| (n.to_ascii_lowercase(), v.trim().to_string()))
.collect();
hs.sort();
let canonical: String = hs.iter().map(|(n, v)| format!("{n}:{v}\n")).collect();
let signed = hs
.iter()
.map(|(n, _)| n.as_str())
.collect::<Vec<_>>()
.join(";");
let creq = format!(
"{}\n{}\n{}\n{canonical}\n{signed}\n{}",
req.method, req.path, req.query, req.payload_sha256
);
let date = req.amz_date.get(..8).unwrap_or(req.amz_date);
let scope = format!("{date}/{}/{}/aws4_request", req.region, req.service);
let to_sign = format!(
"AWS4-HMAC-SHA256\n{}\n{scope}\n{}",
req.amz_date,
sha256_hex(creq.as_bytes())
);
let mut k = hmac(format!("AWS4{secret_key}").as_bytes(), date.as_bytes());
for part in [req.region, req.service, "aws4_request"] {
k = hmac(&k, part.as_bytes());
}
let sig = hex(&hmac(&k, to_sign.as_bytes()));
format!(
"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, SignedHeaders={signed}, Signature={sig}"
)
}
// ── one attempt ──────────────────────────────────────────────────────────────
/// What the last attempts did ([`STATUS_REL`]). Never a credential.
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct Status {
pub last_attempt_unix: u64,
pub last_ok_unix: u64,
pub last_object: String,
pub last_digest: String,
pub last_bytes: u64,
pub failures_in_a_row: u32,
pub last_error: String,
}
/// The status written last (the default when none is).
pub fn status(root: &Path) -> Status {
toml::from_str(&std::fs::read_to_string(root.join(STATUS_REL)).unwrap_or_default())
.unwrap_or_default()
}
fn write_status(root: &Path, st: &Status) {
let text = toml::to_string(st).unwrap_or_default();
if let Err(e) = monetize_front::write_secret(root, STATUS_REL, &text) {
tracing::warn!("[offsite-status] {}: {e}", root.join(STATUS_REL).display());
}
}
/// A failure that is not one attempt's (no settings, no HTTPS client): said in
/// the status, counted.
fn record_failure(root: &Path, e: &str, now_unix: u64) {
let mut st = status(root);
st.last_attempt_unix = now_unix;
st.failures_in_a_row = st.failures_in_a_row.saturating_add(1);
st.last_error = e.to_string();
write_status(root, &st);
}
/// What one attempt did.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
Shipped {
object: String,
bytes: u64,
digest: String,
},
/// The state is the one shipped last, under a day ago.
Unchanged { object: String },
}
/// **One attempt** (module docs, steps 1–5).
pub fn backup_once(
root: &Path,
cfg: &Config,
remote: &dyn Remote,
export: &dyn Fn(&Path) -> Result<(), String>,
now_unix: u64,
) -> Result<Outcome, String> {
let mut st = status(root);
let r = attempt(root, cfg, remote, export, now_unix, &st);
st.last_attempt_unix = now_unix;
match &r {
Ok(Outcome::Shipped {
object,
bytes,
digest,
}) => {
st.last_ok_unix = now_unix;
st.last_object = object.clone();
st.last_digest = digest.clone();
st.last_bytes = *bytes;
st.failures_in_a_row = 0;
st.last_error.clear();
}
Ok(Outcome::Unchanged { .. }) => {
st.failures_in_a_row = 0;
st.last_error.clear();
}
Err(e) => {
st.failures_in_a_row = st.failures_in_a_row.saturating_add(1);
st.last_error = e.clone();
}
}
write_status(root, &st);
r
}
fn attempt(
root: &Path,
cfg: &Config,
remote: &dyn Remote,
export: &dyn Fn(&Path) -> Result<(), String>,
now_unix: u64,
st: &Status,
) -> Result<Outcome, String> {
let dir = root.join(lb::BACKUP_REL);
if let Err(e) = export(&dir) {
if !dir.join(mb::MANIFEST).is_file() {
return Err(format!(
"[offsite-export] monetize did not export ({e}) and no hourly export is there"
));
}
tracing::warn!(
"[offsite-export] monetize did not export now ({e}): the last hourly export is copied"
);
}
let plain = lb::encode(&pack(root)?);
let digest = lb::digest(&plain);
if digest == st.last_digest
&& !st.last_object.is_empty()
&& now_unix.saturating_sub(st.last_ok_unix) < RESEND_AFTER_SECS
{
return Ok(Outcome::Unchanged {
object: st.last_object.clone(),
});
}
let sealed = seal(&cfg.recipients, &plain)?;
drop(plain);
let object = format!(
"{}{OBJECT_PREFIX}{now_unix:011}-{digest}{OBJECT_SUFFIX}",
cfg.prefix
);
let len = sealed.len() as u64;
remote
.put(&object, &sealed)
.map_err(|e| format!("[offsite-upload] {}: {e}", remote.describe()))?;
match remote.size(&object) {
Ok(Some(n)) if n == len => Ok(Outcome::Shipped {
object,
bytes: len,
digest,
}),
Ok(Some(n)) => Err(format!(
"[offsite-verify] {} holds {n} of the {len} bytes of {object}",
remote.describe()
)),
Ok(None) => Err(format!(
"[offsite-verify] {} holds nothing under {object} after the PUT",
remote.describe()
)),
Err(e) => Err(format!("[offsite-verify] {}: {e}", remote.describe())),
}
}
/// **`monetize-admin offsite open --identity <file> --archive <file> --into <dir>`**:
/// decrypt, [`verify`], then write every entry under `<dir>` (0600). `<dir>`
/// must be new or empty. What it says: counts, never a byte of a key.
pub fn open_archive(identity: &Path, archive: &Path, into: &Path) -> Result<String, String> {
use std::io::Write as _;
use std::os::unix::fs::OpenOptionsExt as _;
let o = |w: String| format!("[offsite-open] {w}");
if into.exists()
&& std::fs::read_dir(into)
.map_err(|e| o(format!("{}: {e}", into.display())))?
.next()
.is_some()
{
return Err(o(format!(
"{} is not empty: a copy is opened into a new or empty directory",
into.display()
)));
}
let id = std::fs::read_to_string(identity)
.map_err(|e| o(format!("the identity {}: {e}", identity.display())))?;
let sealed = std::fs::read(archive).map_err(|e| o(format!("{}: {e}", archive.display())))?;
let entries = lb::decode(&open(&id, &sealed)?).map_err(o)?;
drop(id);
mkdir(into).map_err(o)?;
let v = verify(&entries, &into.join(".verify"))?;
for (k, b) in &entries {
if k.starts_with('/') || k.split('/').any(|c| c.is_empty() || c == "." || c == "..") {
return Err(o(format!("`{k}` is not a name this copy writes")));
}
let p = into.join(k);
if let Some(parent) = p.parent() {
mkdir(parent).map_err(o)?;
}
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(&p)
.map_err(|e| o(format!("{}: {e}", p.display())))?;
f.write_all(b)
.and_then(|_| f.sync_all())
.map_err(|e| o(format!("{}: {e}", p.display())))?;
}
Ok(format!(
"opened {}: {} entries, {} ledger rows, {} signups, {} custody sealed, signing root {} — the export imports; written under {}",
archive.display(),
v.entries,
v.ledger_rows,
v.signups,
v.custody_sealed,
v.signing_public_key,
into.display()
))
}
const USAGE: &str = "usage: monetize-admin offsite open --identity <age identity file> --archive <one-monetize-….mfb.age> --into <new or empty directory>";
/// `monetize-admin offsite …`'s words.
pub fn cli(args: &[String]) -> Result<String, String> {
match args.first().map(String::as_str) {
Some("open") => {
let mut flags: BTreeMap<&str, &str> = BTreeMap::new();
let mut it = args[1..].iter();
while let Some(f) = it.next() {
match f.as_str() {
"--identity" | "--archive" | "--into" => {
let v = it
.next()
.ok_or_else(|| format!("{USAGE} — {f} wants a value"))?;
flags.insert(f.as_str(), v.as_str());
}
other => return Err(format!("{USAGE} — `{other}` is not a word of it")),
}
}
let need = |f: &str| {
flags
.get(f)
.map(Path::new)
.ok_or_else(|| format!("{USAGE} — {f} is missing"))
};
open_archive(need("--identity")?, need("--archive")?, need("--into")?)
}
_ => Err(USAGE.to_string()),
}
}
/// The S3 store at the fed endpoint (Cloudflare R2: region `auto`).
pub struct R2 {
client: reqwest::blocking::Client,
endpoint: String,
host: String,
bucket: String,
access_key: Secret,
secret_key: Secret,
}
impl R2 {
pub fn new(cfg: &Config) -> Result<R2, String> {
let _ = rustls::crypto::ring::default_provider().install_default();
let client = reqwest::blocking::Client::builder()
.tls_certs_only(
webpki_root_certs::TLS_SERVER_ROOT_CERTS
.iter()
.filter_map(|c| reqwest::Certificate::from_der(c).ok()),
)
.connect_timeout(Duration::from_secs(30))
.timeout(Duration::from_secs(1800))
.build()
.map_err(|e| format!("[offsite-upload] no HTTPS client: {e}"))?;
let host = cfg.endpoint.trim_start_matches("https://").to_string();
Ok(R2 {
client,
endpoint: cfg.endpoint.clone(),
host,
bucket: cfg.bucket.clone(),
access_key: cfg.access_key.clone(),
secret_key: cfg.secret_key.clone(),
})
}
fn send(
&self,
method: reqwest::Method,
key: &str,
body: Vec<u8>,
) -> Result<reqwest::blocking::Response, String> {
let path = format!("/{}/{}", self.bucket, uri_encode(key));
let payload = sha256_hex(&body);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or_default();
let date = amz_date(now);
let req = SigV4 {
method: method.as_str(),
path: &path,
query: "",
headers: &[
("host", &self.host),
("x-amz-content-sha256", &payload),
("x-amz-date", &date),
],
payload_sha256: &payload,
amz_date: &date,
region: "auto",
service: "s3",
};
let auth = authorization(&req, self.access_key.expose(), self.secret_key.expose());
self.client
.request(method, format!("{}{path}", self.endpoint))
.header("x-amz-content-sha256", payload)
.header("x-amz-date", date)
.header(reqwest::header::AUTHORIZATION, auth)
.body(body)
.send()
.map_err(|e| e.without_url().to_string())
}
}
/// `HTTP <status> <Code>` of a refused request: the S3 error's code, never its
/// body or headers.
fn refused(r: reqwest::blocking::Response) -> String {
let status = r.status();
let text = r.text().unwrap_or_default();
let code = text
.split_once("<Code>")
.and_then(|(_, t)| t.split_once("</Code>"))
.map(|(c, _)| {
c.chars()
.filter(|c| c.is_ascii_alphanumeric())
.take(64)
.collect::<String>()
})
.unwrap_or_default();
format!("HTTP {status} {code}").trim_end().to_string()
}
impl Remote for R2 {
fn describe(&self) -> String {
format!("R2 {} at {}", self.bucket, self.host)
}
fn put(&self, key: &str, bytes: &[u8]) -> Result<(), String> {
let r = self.send(reqwest::Method::PUT, key, bytes.to_vec())?;
if r.status().is_success() {
Ok(())
} else {
Err(refused(r))
}
}
fn size(&self, key: &str) -> Result<Option<u64>, String> {
let r = self.send(reqwest::Method::HEAD, key, Vec::new())?;
if r.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
if !r.status().is_success() {
return Err(refused(r));
}
r.headers()
.get(reqwest::header::CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse().ok())
.map(Some)
.ok_or_else(|| "HEAD said no content-length".to_string())
}
}
/// What PID 1 hands the copy: the feed in force now, and the export.
pub type FeedNow = std::sync::Arc<dyn Fn() -> Option<BTreeMap<String, String>> + Send + Sync>;
pub type Export = Box<dyn Fn(&Path) -> Result<(), String> + Send + Sync>;
const FIRST: Duration = Duration::from_secs(300);
const UNCONFIGURED_AGAIN: Duration = Duration::from_secs(600);
const WARN_UNCONFIGURED_EVERY: Duration = Duration::from_secs(6 * 3600);
const RETRY: [Duration; 3] = [
Duration::from_secs(60),
Duration::from_secs(300),
Duration::from_secs(900),
];
fn now_unix() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or_default()
}
/// The jera duty of the off-box copy (A32-14).
pub const OFFSITE_JOB: &str = "monetize_offsite";
/// **Run the off-box copy for the life of the box**, as a jera duty (A32-14): a copy that
/// panics is an ALERT line and a FAILED row, and the next runs as due (the raw thread ended at
/// a panic, and the box was never copied off again); the box's clean stop ends it.
pub fn start(root: PathBuf, feed: FeedNow, export: Export) {
let mut copy = Copy::default();
let started = monetize_server::jobs::every(OFFSITE_JOB, FIRST, move || {
Ok(monetize_server::jobs::Round::After(
copy.look(&root, &feed, &export),
))
});
if let Err(e) = started {
tracing::error!("[offsite-thread] the off-box copy does not run: {e}");
}
}
/// What the copy keeps between its looks.
#[derive(Default)]
struct Copy {
first_done: bool,
warned: Option<std::time::Instant>,
failed: usize,
last_failure: String,
}
impl Copy {
/// **One look**: the copy once (after the first wait), and how long until the next.
fn look(&mut self, root: &Path, feed: &FeedNow, export: &Export) -> Duration {
if !std::mem::replace(&mut self.first_done, true) {
return FIRST;
}
let cfg = match Config::of(&feed().unwrap_or_default()) {
Ok(c) => c,
Err(e) => {
if self
.warned
.is_none_or(|t| t.elapsed() >= WARN_UNCONFIGURED_EVERY)
{
tracing::warn!("{e}");
self.warned = Some(std::time::Instant::now());
}
if status(root).last_error != e {
record_failure(root, &e, now_unix());
}
return UNCONFIGURED_AGAIN;
}
};
let r = match R2::new(&cfg) {
Ok(r2) => backup_once(root, &cfg, &r2, &**export, now_unix()),
Err(e) => {
record_failure(root, &e, now_unix());
Err(e)
}
};
match r {
Ok(Outcome::Shipped { object, bytes, .. }) => {
tracing::info!(
"[offsite] shipped {object} ({bytes} bytes, age-sealed to {} recipient(s))",
cfg.recipients.len()
);
self.failed = 0;
self.last_failure.clear();
cfg.every
}
Ok(Outcome::Unchanged { object }) => {
tracing::info!("[offsite] unchanged since {object}");
self.failed = 0;
self.last_failure.clear();
cfg.every
}
Err(e) => {
// Mailed to the operator once per distinct failure (audit 6).
let said = e.to_string();
if said != self.last_failure {
tracing::warn!("ALERT offsite.failed {said}");
self.last_failure = said;
} else {
tracing::warn!("{e}");
}
let wait = RETRY[self.failed.min(RETRY.len() - 1)];
self.failed += 1;
wait
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn feed(pairs: &[(&str, &str)]) -> BTreeMap<String, String> {
pairs
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect()
}
const SECRET_VALUE: &str = "R2secretVALUEneverSAID0123456789";
const ACCESS_VALUE: &str = "R2accessKEYneverSAID";
fn throwaway() -> (String, String) {
use age::secrecy::ExposeSecret as _;
let id = age::x25519::Identity::generate();
(
id.to_string().expose_secret().to_string(),
id.to_public().to_string(),
)
}
fn full(recipient: &str) -> BTreeMap<String, String> {
feed(&[
(FEED_RECIPIENTS, recipient),
(FEED_ENDPOINT, "https://acct.r2.cloudflarestorage.com/"),
(FEED_ACCESS_KEY, ACCESS_VALUE),
(FEED_SECRET_KEY, SECRET_VALUE),
(FEED_PREFIX, "one-monetize/prod"),
])
}
/// The AWS SigV4 reference (S3 docs, "GET Object" example): the signer is
/// the one every S3 store checks against.
#[test]
fn the_signature_is_the_aws_reference_one() {
let empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
let req = SigV4 {
method: "GET",
path: "/test.txt",
query: "",
headers: &[
("host", "examplebucket.s3.amazonaws.com"),
("range", "bytes=0-9"),
("x-amz-content-sha256", empty),
("x-amz-date", "20130524T000000Z"),
],
payload_sha256: empty,
amz_date: "20130524T000000Z",
region: "us-east-1",
service: "s3",
};
let auth = authorization(
&req,
"AKIAIOSFODNN7EXAMPLE",
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
);
assert_eq!(
auth,
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
);
}
#[test]
fn the_date_and_the_key_are_written_as_sigv4_wants() {
assert_eq!(amz_date(1_369_353_600), "20130524T000000Z");
assert_eq!(amz_date(1_791_698_797), "20261011T060637Z");
assert_eq!(
uri_encode("one-monetize/prod/a b+c~.age"),
"one-monetize/prod/a%20b%2Bc~.age"
);
}
/// The bench, and a box the kit fed no off-box copy: refused by name, so
/// the box says loudly that nothing leaves it.
#[test]
fn an_unconfigured_box_is_refused_by_name() {
let e = Config::of(&BTreeMap::new()).unwrap_err();
assert!(e.starts_with("[offsite-unconfigured]"), "{e}");
assert!(e.contains(FEED_RECIPIENTS), "{e}");
}
#[test]
fn a_missing_key_is_named_and_no_value_is_said() {
let (_, r) = throwaway();
let mut f = full(&r);
f.remove(FEED_SECRET_KEY);
f.remove(FEED_ENDPOINT);
let e = Config::of(&f).unwrap_err();
assert!(e.starts_with("[offsite-config]"), "{e}");
assert!(
e.contains(FEED_SECRET_KEY) && e.contains(FEED_ENDPOINT),
"{e}"
);
assert!(!e.contains(ACCESS_VALUE), "{e}");
}
/// The vault's own recipients are YubiKeys (`age1yubikey…`): the box has no
/// plugin to encrypt to them. The off-site key's X25519 half is what it takes.
#[test]
fn a_plugin_recipient_is_refused_and_an_x25519_one_taken() {
let (_, r) = throwaway();
let e = parse_recipients(
"age1yubikey1qgx7yyt0wsd9qp2lkx6v4a4lccyvrkg24zq0jrlplkmqa0nvwwqy5wpg3v",
)
.unwrap_err();
assert!(
e.starts_with("[offsite-recipient]") && e.contains("plugin"),
"{e}"
);
assert!(parse_recipients("")
.unwrap_err()
.starts_with("[offsite-recipient]"));
assert_eq!(
parse_recipients(&format!("# the off-site key\n{r}\n")).unwrap(),
vec![r.clone()]
);
let (_, r2) = throwaway();
assert_eq!(parse_recipients(&format!("{r}, {r2}")).unwrap().len(), 2);
}
#[test]
fn the_settings_are_read_and_the_credentials_never_shown() {
let (_, r) = throwaway();
let c = Config::of(&full(&r)).unwrap();
assert_eq!(c.endpoint, "https://acct.r2.cloudflarestorage.com");
assert_eq!(c.bucket, DEFAULT_BUCKET);
assert_eq!(c.prefix, "one-monetize/prod/");
assert_eq!(c.every, Duration::from_secs(DEFAULT_EVERY_MINUTES * 60));
let shown = format!("{c:?}");
assert!(
!shown.contains(SECRET_VALUE) && !shown.contains(ACCESS_VALUE),
"{shown}"
);
let mut f = full(&r);
f.insert(
FEED_ENDPOINT.into(),
"http://acct.r2.cloudflarestorage.com".into(),
);
assert!(Config::of(&f)
.unwrap_err()
.starts_with("[offsite-endpoint]"));
let mut f = full(&r);
f.insert(FEED_PREFIX.into(), "../up".into());
assert!(Config::of(&f).unwrap_err().starts_with("[offsite-prefix]"));
let mut f = full(&r);
f.insert(FEED_EVERY.into(), "1".into());
assert!(Config::of(&f).unwrap_err().starts_with("[offsite-every]"));
let mut f = full(&r);
f.insert(FEED_BUCKET.into(), "other-bucket".into());
f.insert(FEED_EVERY.into(), "60".into());
let c = Config::of(&f).unwrap();
assert_eq!(
(c.bucket.as_str(), c.every),
("other-bucket", Duration::from_secs(3600))
);
}
/// A root shaped like the box's: the export, the data, the custody, an identity.
fn a_box(root: &Path) {
let data = root.join(intake::DATA_REL);
std::fs::create_dir_all(data.join("custody")).unwrap();
std::fs::create_dir_all(root.join(lb::BACKUP_REL)).unwrap();
std::fs::write(
root.join(lb::BACKUP_REL).join(mb::MANIFEST),
"agent = \"x\"\n",
)
.unwrap();
std::fs::write(root.join(lb::BACKUP_REL).join(mb::LEDGER), "{}\n").unwrap();
std::fs::write(data.join("signing.key"), [7u8; 32]).unwrap();
std::fs::write(data.join("ledger.redb"), b"live store").unwrap();
std::fs::write(data.join("fleet.redb"), b"fleet").unwrap();
std::fs::write(data.join("nonpay.journal"), b"j").unwrap();
std::fs::write(data.join("custody/kek"), [9u8; 32]).unwrap();
std::fs::write(data.join("custody/acme.sealed"), b"sealed seeds").unwrap();
std::fs::write(data.join("custody/acme.sealed.staging"), b"half").unwrap();
std::fs::write(root.join(intake::MONETIZE_REL).join("identity-acme"), b"id").unwrap();
std::fs::write(
root.join(intake::MONETIZE_REL).join("identity-acme.pub"),
b"pub",
)
.unwrap();
}
#[test]
fn the_pack_holds_the_export_the_signing_root_and_the_custody_not_the_live_stores() {
let d = tempfile::tempdir().unwrap();
a_box(d.path());
let p = pack(d.path()).unwrap();
for k in [
"OFFSITE",
"export/MANIFEST.toml",
"export/ledger.jsonl",
"data/signing.key",
"data/custody/kek",
"data/custody/acme.sealed",
"data/nonpay.journal",
"data/fleet.redb",
"identity/identity-acme",
] {
assert!(
p.contains_key(k),
"{k} missing from {:?}",
p.keys().collect::<Vec<_>>()
);
}
for k in [
"data/ledger.redb",
"data/custody/acme.sealed.staging",
"identity/identity-acme.pub",
] {
assert!(!p.contains_key(k), "{k} must not be packed");
}
assert_eq!(p["data/custody/kek"], vec![9u8; 32]);
}
#[test]
fn no_export_or_no_signing_key_is_refused() {
let d = tempfile::tempdir().unwrap();
a_box(d.path());
std::fs::remove_file(d.path().join(intake::DATA_REL).join("signing.key")).unwrap();
assert!(pack(d.path()).unwrap_err().starts_with("[offsite-signing]"));
let d = tempfile::tempdir().unwrap();
a_box(d.path());
std::fs::remove_file(d.path().join(lb::BACKUP_REL).join(mb::MANIFEST)).unwrap();
assert!(pack(d.path()).unwrap_err().starts_with("[offsite-export]"));
}
#[test]
fn a_sealed_copy_opens_only_with_its_identity_and_holds_no_plaintext() {
let (id, r) = throwaway();
let plain = b"MFB1\nthe ledger, the signing root, the custody".repeat(50);
let sealed = seal(std::slice::from_ref(&r), &plain).unwrap();
assert!(
!sealed.windows(10).any(|w| w == b"the ledger"),
"plaintext in the sealed copy"
);
assert_eq!(open(&id, &sealed).unwrap(), plain);
let (other, _) = throwaway();
let e = open(&other, &sealed).unwrap_err();
assert!(
e.starts_with("[offsite-open]") && !e.contains(&other),
"{e}"
);
}
/// A store that keeps fewer bytes than it was given.
struct Lossy;
impl Remote for Lossy {
fn describe(&self) -> String {
"lossy".into()
}
fn put(&self, _: &str, _: &[u8]) -> Result<(), String> {
Ok(())
}
fn size(&self, _: &str) -> Result<Option<u64>, String> {
Ok(Some(3))
}
}
#[test]
fn one_attempt_ships_under_the_prefix_then_skips_an_unchanged_state() {
let d = tempfile::tempdir().unwrap();
a_box(d.path());
let (_, r) = throwaway();
let cfg = Config::of(&full(&r)).unwrap();
let store = MemRemote::default();
let ok = |_: &Path| Ok(());
let Outcome::Shipped { object, bytes, .. } =
backup_once(d.path(), &cfg, &store, &ok, 1_791_698_797).unwrap()
else {
panic!("not shipped")
};
assert!(
object.starts_with("one-monetize/prod/one-monetize-01791698797-")
&& object.ends_with(OBJECT_SUFFIX),
"{object}"
);
assert_eq!(store.size(&object).unwrap(), Some(bytes));
let st = status(d.path());
assert_eq!(
(
st.last_ok_unix,
st.failures_in_a_row,
st.last_object.as_str()
),
(1_791_698_797, 0, object.as_str())
);
{
use std::os::unix::fs::PermissionsExt as _;
assert_eq!(
std::fs::metadata(d.path().join(STATUS_REL))
.unwrap()
.permissions()
.mode()
& 0o777,
0o600
);
}
let said = std::fs::read_to_string(d.path().join(STATUS_REL)).unwrap();
assert!(!said.contains(SECRET_VALUE) && !said.contains(ACCESS_VALUE));
assert_eq!(
backup_once(d.path(), &cfg, &store, &ok, 1_791_698_797 + 60).unwrap(),
Outcome::Unchanged {
object: object.clone()
}
);
assert!(
matches!(
backup_once(
d.path(),
&cfg,
&store,
&ok,
1_791_698_797 + RESEND_AFTER_SECS + 1
)
.unwrap(),
Outcome::Shipped { .. }
),
"a day on, shipped again"
);
}
#[test]
fn a_short_store_and_a_missing_export_fail_and_are_counted() {
let d = tempfile::tempdir().unwrap();
a_box(d.path());
let (_, r) = throwaway();
let cfg = Config::of(&full(&r)).unwrap();
let ok = |_: &Path| Ok(());
let e = backup_once(d.path(), &cfg, &Lossy, &ok, 100).unwrap_err();
assert!(e.starts_with("[offsite-verify]"), "{e}");
// monetize does not answer, and no hourly export is there either.
std::fs::remove_file(d.path().join(lb::BACKUP_REL).join(mb::MANIFEST)).unwrap();
let down = |_: &Path| Err("monetize does not answer".to_string());
let e = backup_once(d.path(), &cfg, &MemRemote::default(), &down, 200).unwrap_err();
assert!(e.starts_with("[offsite-export]"), "{e}");
let st = status(d.path());
assert_eq!((st.failures_in_a_row, st.last_ok_unix), (2, 0));
assert!(st.last_error.starts_with("[offsite-export]"));
}
#[test]
fn the_hourly_export_is_shipped_when_monetize_cannot_export_now() {
let d = tempfile::tempdir().unwrap();
a_box(d.path());
let (_, r) = throwaway();
let down = |_: &Path| Err("monetize does not answer".to_string());
assert!(matches!(
backup_once(
d.path(),
&Config::of(&full(&r)).unwrap(),
&MemRemote::default(),
&down,
300
)
.unwrap(),
Outcome::Shipped { .. }
));
}
#[test]
fn the_cli_names_its_words() {
assert!(cli(&[]).unwrap_err().contains("offsite open --identity"));
assert!(cli(&["open".into(), "--identity".into(), "x".into()])
.unwrap_err()
.contains("--archive"));
}
}