crates/monetize-box-watcher/src/lib.rs
//! **The ONE-box watcher** (lane BOX-START part 3, 2026-11-01). On 2026-11-01 UpCloud stopped
//! every server in the account over a payment issue — the ONE boxes with the rest — and
//! nothing outside UpCloud could start them again. This program runs OUTSIDE UpCloud (a
//! `systemd --user` timer on dwarf). Each run:
//!
//! 1. reads the UpCloud API token of each account from an existing secret file (a
//! `KEY=value` line; the token is never printed, logged or put in an argument);
//! 2. lists that account's servers (`GET /1.3/server`) and finds the ONE boxes it watches —
//! by uuid, or by the public address their DNS name resolves to (`GET /1.3/server/<uuid>`);
//! 3. for a ONE box that is `stopped`: in DRY-RUN (the default) logs what it would do; with
//! `apply = true` starts it (`POST /1.3/server/<uuid>/start`). Nothing else is ever
//! started, stopped, created or deleted: the ONE box, once up, starts its estates itself
//! (monetize-guardian's start pass: main first, then its twin).
//!
//! ★ SEC-S1 (sec review 2026-11-01): a ONE box its operator stopped ON PURPOSE is never
//! started. The operator holds it with a marker file named after the box (`[[box]] name`) in
//! `hold_dir` (default `~/.config/monetize-box-watcher/hold`); its first line says why. The
//! run then says `held <box>` and starts nothing of it; the marker removed, it is watched again.
//!
//! One line per box on stdout (journald keeps them).
use std::collections::BTreeMap;
use std::net::ToSocketAddrs;
use std::path::{Path, PathBuf};
use std::time::Duration;
use serde::Deserialize;
use serde_json::Value;
/// The watcher's configuration (`watcher.toml`).
#[derive(Clone, Debug, Deserialize)]
pub struct Config {
/// `false` (the default): log what would be done, start nothing.
#[serde(default)]
pub apply: bool,
/// The UpCloud API (a mock's base in tests).
#[serde(default = "default_api")]
pub api_base: String,
#[serde(default, rename = "account")]
pub accounts: Vec<Account>,
#[serde(default, rename = "box")]
pub boxes: Vec<WatchedBox>,
/// ★ SEC-S1: where the operator's hold markers are (`<hold_dir>/<box name>`); `None`: no
/// box is held (the binary sets the default).
#[serde(default)]
pub hold_dir: Option<PathBuf>,
}
/// The default hold directory, under `$HOME`.
pub fn default_hold_dir() -> Option<PathBuf> {
std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config/monetize-box-watcher/hold"))
}
/// **Why the operator holds the box `name` stopped**, when its marker is there: its first line
/// (or that it is held, when the file says nothing). A marker that cannot be read still holds.
pub fn hold_of(cfg: &Config, name: &str) -> Option<(PathBuf, String)> {
let dir = cfg.hold_dir.as_ref()?;
if name.is_empty() || name.contains('/') || name.starts_with('.') {
return None;
}
let path = dir.join(name);
if std::fs::symlink_metadata(&path).is_err() {
return None;
}
let why = std::fs::read_to_string(&path).ok().and_then(|t| t.lines().map(str::trim).find(|l| !l.is_empty()).map(str::to_string)).unwrap_or_else(|| "held by its operator".into());
Some((path, why))
}
fn default_api() -> String {
"https://api.upcloud.com".into()
}
/// One UpCloud account and where its token lives.
#[derive(Clone, Debug, Deserialize)]
pub struct Account {
pub name: String,
/// A secret file of `KEY=value` lines (`export KEY=value` and quotes allowed).
pub secrets_file: PathBuf,
/// The key whose value is the API token.
pub token_key: String,
}
/// One ONE box the watcher starts when it is stopped.
#[derive(Clone, Debug, Deserialize)]
pub struct WatchedBox {
pub name: String,
/// The server's uuid, when known.
#[serde(default)]
pub uuid: String,
/// Else a DNS name (or an IP) of the box: matched against the servers' public addresses.
#[serde(default)]
pub address: String,
/// Look only in this account (by name); empty: every account.
#[serde(default)]
pub account: String,
}
/// What one run did for one box.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Verdict {
/// It runs: nothing to do.
Running { uuid: String },
/// Stopped and started now (`apply`).
Started { uuid: String },
/// Stopped; a dry run starts nothing.
WouldStart { uuid: String },
/// ★ SEC-S1: stopped, and its operator holds it (a marker): never started.
Held { uuid: String, why: String, marker: String },
/// Starting, stopping or in maintenance: looked at again next run.
Busy { uuid: String, state: String },
/// `error` (or anything else): nothing done, a person looks.
Alert { uuid: String, state: String },
/// Not found in any account it may be in.
NotFound { why: String },
/// The start was refused, or the API could not be read.
Failed { why: String },
}
impl Verdict {
pub fn line(&self, name: &str) -> String {
match self {
Verdict::Running { uuid } => format!("ok {name}: server {uuid} is started"),
Verdict::Started { uuid } => format!("STARTED {name}: server {uuid} was stopped; asked UpCloud to start it (the ONE box starts its estates itself)"),
Verdict::WouldStart { uuid } => format!("DRY-RUN {name}: server {uuid} is stopped; with apply = true it would be started"),
Verdict::Held { uuid, why, marker } => format!("held {name}: server {uuid} is stopped and its operator holds it ({why}); not started — remove {marker} to have it watched again"),
Verdict::Busy { uuid, state } => format!("wait {name}: server {uuid} is {state}; looked at again next run"),
Verdict::Alert { uuid, state } => format!("ALERT box-watcher.error {name}: server {uuid} is {state}; nothing done, a person looks"),
Verdict::NotFound { why } => format!("ALERT box-watcher.not_found {name}: {why}"),
Verdict::Failed { why } => format!("ALERT box-watcher.failed {name}: {why}"),
}
}
}
/// A token, never printed: `Debug` says only that it is there.
pub struct Token(String);
impl std::fmt::Debug for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Token(<redacted>)")
}
}
/// **The token of `key` in a `KEY=value` secret file.** The error names the file and the key,
/// never a value.
pub fn read_token(file: &Path, key: &str) -> Result<Token, String> {
let text = std::fs::read_to_string(file).map_err(|e| format!("the secret file {} cannot be read: {e}", file.display()))?;
for line in text.lines() {
let line = line.trim();
let line = line.strip_prefix("export ").unwrap_or(line);
let Some((k, v)) = line.split_once('=') else { continue };
if k.trim() == key {
let v = v.trim().trim_matches('"').trim_matches('\'').trim();
if v.is_empty() {
return Err(format!("{key} in {} is empty", file.display()));
}
return Ok(Token(v.to_string()));
}
}
Err(format!("no {key} in {}", file.display()))
}
/// The UpCloud API, read with one account's token.
pub struct Api {
base: String,
token: Token,
client: reqwest::blocking::Client,
}
/// One server as the listing says it.
#[derive(Clone, Debug)]
pub struct Server {
pub uuid: String,
pub hostname: String,
pub state: String,
}
impl Api {
pub fn new(base: &str, token: Token) -> Result<Api, String> {
let _ = rustls::crypto::ring::default_provider().install_default();
let client = reqwest::blocking::Client::builder()
.tls_certs_only(webpki_root_certs::TLS_SERVER_ROOT_CERTS.iter().filter_map(|c| reqwest::Certificate::from_der(c).ok()))
.connect_timeout(Duration::from_secs(20))
.timeout(Duration::from_secs(60))
.user_agent("monetize-box-watcher")
.build()
.map_err(|e| format!("no HTTPS client: {e}"))?;
Ok(Api { base: base.trim_end_matches('/').to_string(), token, client })
}
fn call(&self, method: reqwest::Method, path: &str, body: Option<Value>) -> Result<Value, String> {
let mut req = self.client.request(method.clone(), format!("{}{path}", self.base)).bearer_auth(&self.token.0);
if let Some(b) = body {
req = req.json(&b);
}
let resp = req.send().map_err(|e| format!("{method} {path}: {}", e.without_url()))?;
let status = resp.status();
let text = resp.text().unwrap_or_default();
if !status.is_success() {
let code = serde_json::from_str::<Value>(&text).ok().and_then(|v| v["error"]["error_code"].as_str().map(str::to_string)).unwrap_or_default();
return Err(format!("{method} {path}: HTTP {} {code}", status.as_u16()));
}
Ok(serde_json::from_str(&text).unwrap_or(Value::Null))
}
/// `GET /1.3/server`.
pub fn servers(&self) -> Result<Vec<Server>, String> {
let v = self.call(reqwest::Method::GET, "/1.3/server", None)?;
Ok(v["servers"]["server"].as_array().map(|a| a.iter().map(|s| Server { uuid: s["uuid"].as_str().unwrap_or_default().into(), hostname: s["hostname"].as_str().unwrap_or_default().into(), state: s["state"].as_str().unwrap_or_default().into() }).collect()).unwrap_or_default())
}
/// The public addresses of one server (`GET /1.3/server/<uuid>`).
pub fn addresses(&self, uuid: &str) -> Result<Vec<String>, String> {
let v = self.call(reqwest::Method::GET, &format!("/1.3/server/{uuid}"), None)?;
Ok(v["server"]["ip_addresses"]["ip_address"].as_array().map(|a| a.iter().filter(|ip| ip["access"].as_str() == Some("public")).filter_map(|ip| ip["address"].as_str().map(str::to_string)).collect()).unwrap_or_default())
}
/// `POST /1.3/server/<uuid>/start`.
pub fn start(&self, uuid: &str) -> Result<(), String> {
self.call(reqwest::Method::POST, &format!("/1.3/server/{uuid}/start"), None).map(|_| ())
}
}
/// The addresses a box's `address` names: itself when it is an IP, else what DNS answers.
pub fn resolve(address: &str) -> Vec<String> {
if address.parse::<std::net::IpAddr>().is_ok() {
return vec![address.to_string()];
}
(address, 0).to_socket_addrs().map(|it| it.map(|a| a.ip().to_string()).collect()).unwrap_or_default()
}
/// **One run over every watched box.** `apis` by account name (an account whose token could
/// not be read is absent, its error in `refused`). Answers one verdict per box, in order.
pub fn run(cfg: &Config, apis: &BTreeMap<String, Api>, refused: &BTreeMap<String, String>, resolver: &dyn Fn(&str) -> Vec<String>) -> Vec<(String, Verdict)> {
let mut listed: BTreeMap<&str, Result<Vec<Server>, String>> = BTreeMap::new();
let mut out = Vec::new();
for b in &cfg.boxes {
let names: Vec<&str> = cfg.accounts.iter().map(|a| a.name.as_str()).filter(|n| b.account.is_empty() || *n == b.account).collect();
let mut found: Option<(&str, Server)> = None;
let mut why = Vec::new();
let wanted_ips = if b.uuid.is_empty() { resolver(&b.address) } else { Vec::new() };
if b.uuid.is_empty() && wanted_ips.is_empty() {
out.push((b.name.clone(), Verdict::NotFound { why: format!("no uuid, and {:?} resolves to no address", b.address) }));
continue;
}
for name in names {
let Some(api) = apis.get(name) else {
why.push(format!("account {name}: {}", refused.get(name).cloned().unwrap_or_else(|| "no token".into())));
continue;
};
let servers = listed.entry(name).or_insert_with(|| api.servers());
let servers = match servers {
Ok(s) => s.clone(),
Err(e) => {
why.push(format!("account {name}: {e}"));
continue;
}
};
let hit = if !b.uuid.is_empty() {
servers.into_iter().find(|s| s.uuid == b.uuid)
} else {
servers.into_iter().find(|s| api.addresses(&s.uuid).is_ok_and(|ips| ips.iter().any(|ip| wanted_ips.contains(ip))))
};
match hit {
Some(s) => {
found = Some((name, s));
break;
}
None => why.push(format!("account {name}: no such server")),
}
}
let verdict = match found {
None => Verdict::NotFound { why: if why.is_empty() { "no account to look in".into() } else { why.join("; ") } },
Some((_, s)) if s.state == "started" => Verdict::Running { uuid: s.uuid },
Some((_, s)) if s.state == "stopped" && hold_of(cfg, &b.name).is_some() => {
let (marker, why) = hold_of(cfg, &b.name).unwrap_or_default();
Verdict::Held { uuid: s.uuid, why, marker: marker.display().to_string() }
}
Some((name, s)) if s.state == "stopped" => {
if !cfg.apply {
Verdict::WouldStart { uuid: s.uuid }
} else {
match apis[name].start(&s.uuid) {
Ok(()) => Verdict::Started { uuid: s.uuid },
Err(e) => Verdict::Failed { why: format!("server {} is stopped and would not start: {e}", s.uuid) },
}
}
}
Some((_, s)) if matches!(s.state.as_str(), "maintenance" | "starting" | "stopping") => Verdict::Busy { uuid: s.uuid, state: s.state },
Some((_, s)) => Verdict::Alert { uuid: s.uuid, state: s.state },
};
out.push((b.name.clone(), verdict));
}
out
}
/// The APIs of every account whose token reads, and why the others do not.
pub fn open_accounts(cfg: &Config) -> (BTreeMap<String, Api>, BTreeMap<String, String>) {
let (mut apis, mut refused) = (BTreeMap::new(), BTreeMap::new());
for a in &cfg.accounts {
match read_token(&a.secrets_file, &a.token_key).and_then(|t| Api::new(&cfg.api_base, t)) {
Ok(api) => {
apis.insert(a.name.clone(), api);
}
Err(e) => {
refused.insert(a.name.clone(), e);
}
}
}
(apis, refused)
}
/// Read `watcher.toml`.
pub fn load(path: &Path) -> Result<Config, String> {
let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?;
toml::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))
}