crates/monetize-server/src/guests.rs
//! **Guests on the fly** — a new or rebuilt `.wasm` product guest is taken in
//! while the server runs, with no restart.
//!
//! The operator's whole procedure is the one they already follow at install:
//!
//! 1. put the module and its `.p7s` where the block will name it (every guest
//! is signed: a block without `guest_trust_roots` is refused, `[guest-unsigned]`);
//! 2. write or edit `[products.<id>]` in the config file — `guest`,
//! `guest_trust_roots` (and, to pin bytes too, `guest_digest`), `grants`, `http_hosts`,
//! `guest_grpc_hosts`, and for a NEW product its `[prices.<id>.<market>]`.
//!
//! [`watch`] notices within one period (the config file, and every file a
//! guest block names: the module, its `.p7s`, its roots), re-reads the config
//! and applies its guest blocks through [`crate::registry::Registry::apply_guests`]:
//! same gates as at start, compiled off the serving path, swapped atomically,
//! calls in flight finishing on the old guest. Every outcome is one line on
//! stderr (the journal), and a refusal keeps what was serving.
//!
//! **Guest blocks, vendors and clouds are applied live** — a vendor or a cloud
//! through [`apply_providers`]: added, replaced, retired, each cloud's spend
//! ceiling reconciled against the ledger before it is published. The policy,
//! a linked product plugin, a price change for a product already served: the
//! next restart, as before — said once per change, so nobody believes an edit
//! took effect that did not. A NEW product's price list is installed with it,
//! because a product served without its list would be sold as free.
//!
//! No crate for file notification: a stat of a handful of paths per period is
//! the whole cost, and it is pure std.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use crate::config::Config;
use crate::registry::GuestChange;
use crate::state::AppState;
/// **The watcher's journal, readable in-process** — every line [`watch`]
/// says goes to stderr AND here (a bounded ring, newest last), so a host that
/// embeds monetize can answer an operator with what the watcher decided
/// (`guest ADDED …`, `guest REPLACED …`, `guest REFUSED […] …`) rather than
/// "written, look at the journal". [`journal_mark`] then [`journal_since`].
static JOURNAL: std::sync::Mutex<(u64, std::collections::VecDeque<(u64, String)>)> = std::sync::Mutex::new((0, std::collections::VecDeque::new()));
/// How many lines the ring keeps.
pub const JOURNAL_KEEP: usize = 256;
pub(crate) fn journal_push(line: &str) {
let mut j = JOURNAL.lock().unwrap_or_else(|e| e.into_inner());
j.0 += 1;
let n = j.0;
j.1.push_back((n, line.to_string()));
while j.1.len() > JOURNAL_KEEP {
j.1.pop_front();
}
}
/// The number of the last line written so far.
pub fn journal_mark() -> u64 {
JOURNAL.lock().unwrap_or_else(|e| e.into_inner()).0
}
/// Every line written after `mark`.
pub fn journal_since(mark: u64) -> Vec<String> {
JOURNAL.lock().unwrap_or_else(|e| e.into_inner()).1.iter().filter(|(n, _)| *n > mark).map(|(_, l)| l.clone()).collect()
}
/// **Where a look says which config it applied** (R34-5): `<config>.applied`, the sha256 (hex)
/// of the config text it read, written once applied — the ONE box's door answers a feed
/// "applied" only when this names the config written for it.
pub fn applied_path(config: &Path) -> PathBuf {
let mut p = config.as_os_str().to_owned();
p.push(".applied");
PathBuf::from(p)
}
/// Pure: a config text's sha256, hex ([`applied_path`]'s words).
pub fn config_digest(bytes: &[u8]) -> String {
use sha2::Digest as _;
sha2::Sha256::digest(bytes).iter().map(|b| format!("{b:02x}")).collect()
}
/// How often [`watch`] looks. A guest is taken in within this, and the look is
/// a few `stat`s.
pub const WATCH_EVERY: Duration = Duration::from_secs(2);
/// **Apply `cfg`'s guest blocks to the running server.** A NEW product's
/// `[prices.<id>]` (and its host) is installed into the terms before the
/// product is published — a product served without the list the operator
/// wrote for it would be sold as free. A list that does not parse refuses the
/// product by name.
pub fn apply(state: &AppState, cfg: &Config) -> Vec<GuestChange> {
let prices = cfg.price_lists();
let mut admit = |id: &str| -> Result<(), String> {
let prices = prices.as_ref().map_err(|e| format!("the config's price lists do not parse, so the new product cannot be priced: {e}"))?;
// G8: a new block that pins its key signs with it from its first fact.
if let Some(path) = cfg.products.get(id).and_then(|p| p.signing_key.as_ref()) {
let key = monetize::Signer::from_path(path).map_err(|e| format!("[products.{id}] signing_key {}: {e}", path.display()))?;
state.keys.pin(id, key).map_err(|e| format!("[products.{id}] signing_key {}: {e}", path.display()))?;
}
let mut terms = state.terms.write().unwrap_or_else(|e| e.into_inner());
if let Some(list) = prices.get(id) {
terms.prices.insert(id.to_string(), list.clone());
}
terms.hosts.insert(id.to_string(), cfg.host_of(id));
Ok(())
};
let changes = state.registry.apply_guests(cfg, &mut admit);
// The shop window follows the blocks: a product that registered itself after
// start brings its bundles in the same block (`AppState::bundles`).
*state.bundles.write().unwrap_or_else(|e| e.into_inner()) = bundles_of(cfg);
changes
}
/// **Every `[signups.<product>]`, taken in live**: their plans and the meter
/// each product's caps land on replace the server's
/// ([`crate::signup::Plans`]); the blocks are handed to the server's
/// [`crate::signup::SignupReload`], which builds and swaps a product's sink
/// when its block or module changed. Answers the lines to log.
pub fn apply_signup(state: &AppState, cfg: &Config) -> Vec<String> {
let mut lines = Vec::new();
let plans = cfg.signup_plans();
let count = plans.len();
if state.signup_plans.set(plans, cfg.signup_cap_meters()) {
let meters: Vec<String> = cfg.signups.keys().map(|p| format!("{p} on {}", state.signup_plans.cap_meter_of(p))).collect();
lines.push(format!("signup plans taken in live: {count} plan(s), caps {}", meters.join(", ")));
}
if let Some(r) = &state.signup_reload {
match cfg.signup_blocks() {
Ok(blocks) => lines.extend(r.apply(&blocks)),
Err(e) => lines.push(format!("signup guest REFUSED [signup-unreadable] {e}")),
}
}
lines
}
/// `[products.<id>.bundles]`, every product that declares any.
pub fn bundles_of(cfg: &Config) -> std::collections::BTreeMap<String, std::collections::BTreeMap<String, monetize::BundleSpec>> {
cfg.products.iter().filter(|(_, p)| !p.bundles.is_empty()).map(|(id, p)| (id.clone(), p.bundles.clone())).collect()
}
/// `vendor -> checkout URL template`, as `[vendors.<name>] checkout_url` says.
pub fn checkout_urls(cfg: &Config) -> std::collections::BTreeMap<String, String> {
cfg.vendors.iter().filter(|(_, v)| !v.checkout_url.is_empty()).map(|(k, v)| (k.clone(), v.checkout_url.clone())).collect()
}
/// WHERE EACH SUPPLIER IS, from the operator's own declaration. A vendor or
/// cloud that declares nothing is simply absent from the map, and
/// `monetize::Books` reads absence as domestic — never as a reverse charge
/// nobody wrote down.
pub fn supplier_countries(cfg: &Config) -> std::collections::BTreeMap<String, String> {
cfg.vendors
.iter()
.map(|(k, v)| (k.clone(), v.country.clone()))
.chain(cfg.clouds.iter().map(|(k, c)| (k.clone(), c.country.clone())))
.filter(|(_, country)| !country.is_empty())
.collect()
}
/// **Apply `cfg`'s vendors and clouds to the running server** — the
/// registry's [`crate::registry::Registry::apply_providers`], each cloud's
/// ceiling reconciled against THIS ledger before it is published, and then
/// what the server derived from those blocks at start: the checkout URLs, the
/// suppliers' countries, the file the estate picture is drawn from.
pub fn apply_providers(state: &AppState, cfg: &Config) -> Vec<crate::registry::ProviderChange> {
let spent = |provider: &str, currency: monetize_pay::Currency| state.ledger.total_provisioned_minor_by(provider, currency).map_err(|e| e.to_string());
let changes = state.registry.apply_providers(cfg, &spent);
*state.checkout_urls.write().unwrap_or_else(|e| e.into_inner()) = checkout_urls(cfg);
state.books.write().unwrap_or_else(|e| e.into_inner()).supplier_countries = supplier_countries(cfg);
*state.config.write().unwrap_or_else(|e| e.into_inner()) = cfg.clone();
changes
}
/// One look's fingerprint: each file [`stamp`] reads, with its length and
/// mtime, or `None` when it is missing.
pub type Stamp = Vec<(PathBuf, Option<(u64, SystemTime)>)>;
/// What [`watch`] compares: the config file and every file its guest blocks
/// name, each as (path, length, mtime). Missing files are part of the stamp
/// too — a module being written is a change when it appears. Public: a host
/// that writes monetize's files (the front's plugin door) asks which files the
/// watcher reads, to know whether it will say anything at all.
pub fn stamp(config: &Path, cfg: Option<&Config>) -> Stamp {
let of = |p: &Path| std::fs::metadata(p).ok().map(|m| (m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH)));
let mut out = vec![(config.to_path_buf(), of(config))];
if let Some(cfg) = cfg {
let mut named = |module: PathBuf, roots: Vec<PathBuf>| {
let mut sig = module.clone().into_os_string();
sig.push(".p7s");
for f in [module, PathBuf::from(sig)].into_iter().chain(roots) {
let s = of(&f);
out.push((f, s));
}
};
for p in cfg.products.values() {
if let Some(module) = &p.guest {
named(module.clone(), p.guest_trust_roots.clone());
}
}
for v in cfg.vendors.values() {
if let Some(module) = &v.guest {
named(module.clone(), v.guest_trust_roots.clone());
}
}
// A cloud's `guest =` and roots, from the block's own keys.
let table_files = |t: &toml::Table| -> Option<(PathBuf, Vec<PathBuf>)> {
let module = PathBuf::from(t.get("guest")?.as_str()?);
let roots = t.get("guest_trust_roots").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|r| r.as_str().map(PathBuf::from)).collect()).unwrap_or_default();
Some((module, roots))
};
for (module, roots) in cfg.clouds.values().filter_map(|c| table_files(&c.settings)) {
named(module, roots);
}
// Each product's signup guest module, from its `[signups.<product>]`
// own keys: a new module at the same path is a change too.
for (module, roots) in cfg.signups.values().filter_map(|b| table_files(&b.settings)) {
named(module, roots);
}
// Each DNS writer's module (2026-10-17): a module written over the old one is a change.
for b in cfg.dns_writers.values() {
if let Some(module) = b.get("guest").and_then(|v| v.as_str()).map(PathBuf::from) {
let roots = b.get("guest_trust_roots").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|r| r.as_str().map(PathBuf::from)).collect()).unwrap_or_default();
named(module, roots);
}
}
}
out
}
/// **One look**: if anything in the stamp moved, re-read the config and apply
/// it. Answers the lines to log. `last` carries the previous stamp and config
/// between looks.
pub fn look(state: &AppState, config: &Path, last: &mut (Stamp, Option<Config>)) -> Vec<String> {
look_then(state, config, last, || {})
}
/// [`look`], `meanwhile` run after the config is read and before the look ends (a test's
/// writer that rewrites the config while the look applies it).
fn look_then(state: &AppState, config: &Path, last: &mut (Stamp, Option<Config>), meanwhile: impl FnOnce()) -> Vec<String> {
let now = stamp(config, last.1.as_ref());
if now == last.0 {
return Vec::new();
}
// The first look (an empty stamp) is not a "change" to announce: it only
// catches what was written between the start's load and the watcher's start.
let config_moved = !last.0.is_empty() && now.first() != last.0.first();
last.0 = now.clone();
// R34-5: what this look applies, by its sha256 (read before the load: never newer).
let read_digest = std::fs::read(config).map(|b| config_digest(&b)).ok();
let cfg = match Config::load(config) {
Ok(c) => c,
Err(e) => return vec![format!("guest REFUSED [config-unreadable] {}: {e} — every guest that served still serves", config.display())],
};
// R34-7: the stamp of what was READ — the config's from before the read, the files the new
// config names from before they are applied — so a write during the apply is a change the
// next look takes (at worst one look applies again what is current already).
let read = {
let mut s = stamp(config, Some(&cfg));
if let (Some(c), Some(was)) = (s.first_mut(), now.first()) {
*c = was.clone();
}
s
};
let mut lines: Vec<String> = apply(state, &cfg).iter().map(|c| c.to_string()).collect();
// A product guest's estate lifecycle follows its module (2026-10-17).
lines.extend(crate::estate_order::guest_lifecycle::follow(&state.registry));
// The control principals follow the config live: a console fed after the
// start is taken in, a removed one is out on the next look.
lines.extend(state.control.set(&cfg.control_principals));
lines.extend(apply_providers(state, &cfg).iter().map(|c| c.to_string()));
// Each site's zone writer follows the sites the config now declares (test3.gunnar.rs,
// 2026-10-14: a site fed after the start had none and wrote no DNS).
lines.extend(state.estates.follow_site_dns(&cfg, state.registry.linked()));
lines.extend(follow_dns(&cfg));
lines.extend(apply_signup(state, &cfg));
lines.extend(crate::fleet::apply_pulled(state, &cfg));
meanwhile();
if config_moved {
lines.push(format!(
"config {} changed: its guest blocks, vendors and clouds were applied live; the policy, the prices of a product already served and linked product plugins take effect at the next restart",
config.display()
));
}
// The stamp now covers the files the NEW config names, as they were read.
last.0 = read;
if let Some(d) = read_digest {
let marker = applied_path(config);
if let Err(e) = std::fs::write(&marker, format!("{d}\n")) {
lines.push(format!("config {} applied, but {} could not be written: {e}", config.display(), marker.display()));
}
}
last.1 = Some(cfg);
lines
}
/// **Every DNS writer built from a guest follows its block live** (2026-10-17): its module,
/// signature or roots changed → rebuilt through the same gates as at start and swapped in; a
/// refusal keeps the writer that served (`monetize_cloud_wasm::dns::HotDns`).
pub fn follow_dns(cfg: &Config) -> Vec<String> {
monetize_cloud_wasm::dns::follow_all(&cfg.data_dir, &crate::estate_order::dns_guest_blocks(&cfg.dns_writers, &cfg.secrets_dir()))
}
/// The jera duty that watches the guests' config (A32-14).
pub const GUEST_WATCH: &str = "monetize_guest_watch";
/// **The watcher**: a jera duty (A32-14), off every serving path, looking every
/// `every`; a look that panics is an ALERT line and a FAILED row and the next runs, and the
/// server's clean stop ends it. `initial` is the config the server started from, so the first
/// look compares against what is loaded rather than re-applying it.
/// A duty that cannot start is the caller's to say (R5a): the server serves on
/// without guests on the fly rather than panic at its start.
pub fn watch(state: Arc<AppState>, config: PathBuf, initial: Config, every: Duration) -> Result<Arc<crate::jobs::Job>, String> {
// An EMPTY stamp, not the file's stamp now: a config written after
// `initial` was loaded and before this line ran (a host that embeds
// monetize and writes its config as it starts — the gunnar front)
// would otherwise carry the new stamp with the OLD config and never be
// applied. The first look applies whatever differs; a guest that is
// already current is left alone (`is_current_for`), so a quiet start
// stays quiet. MEASURED 2026-09-25 by the front's plugin door test.
// ★ A32-4: the door listens: what it serves is said at once, and after every look.
crate::serving::say(&state, &initial);
let mut last = (Vec::new(), Some(initial));
let mut first = true;
crate::jobs::every(GUEST_WATCH, every, move || {
// The first look only waits, as the thread's first sleep did.
if std::mem::take(&mut first) {
return Ok(crate::jobs::Round::Again);
}
for line in look(&state, &config, &mut last) {
eprintln!("monetize-server: {line}");
journal_push(&line);
}
if let Some(cfg) = last.1.as_ref() {
crate::serving::say(&state, cfg);
}
Ok(crate::jobs::Round::Again)
})
}
#[cfg(test)]
mod bundle_reload_tests {
use super::*;
/// ★ A product's bundles are taken in with its block on a live reload, not
/// only at start: a product that registers itself after the server started
/// gets its shop window (holger's mock, 2026-09-28: `Purchase.Bundles` was `[]`).
#[test]
fn a_reload_takes_in_the_bundles_its_blocks_declare() {
let state = AppState::in_memory(crate::registry::Registry::new());
assert!(state.bundles.read().unwrap().is_empty());
let cfg = Config::from_toml("[products.holger]\nendpoint = \"127.0.0.1:1\"\n[products.holger.bundles.\"10gib\"]\ncap_delta = { disk_bytes = 10737418240 }\nperiod_days = 30\n").unwrap();
let _ = apply(&state, &cfg);
let b = state.bundles.read().unwrap();
assert_eq!(b["holger"]["10gib"].cap_delta["disk_bytes"], 10_737_418_240);
drop(b);
let _ = apply(&state, &Config::from_toml("").unwrap());
assert!(state.bundles.read().unwrap().is_empty(), "a block that is gone takes its window with it");
}
/// ★ **R34-7: a config written WHILE a look applies the last one is applied on the next look**
/// (rehearsal 2026-10-25: two sites added in one second; the box applied the first and
/// took the second's config file as seen — `site show` named the site, `test-order` said
/// "not a declared operator site" for 40 minutes, until a later add moved the file again).
/// The look stamped the file AFTER applying, so a write during the apply was never a change.
#[test]
fn a_config_written_while_a_look_applies_is_applied_on_the_next_look() {
let dir = std::env::temp_dir().join(format!("monetize-guests-r34-7-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let config = dir.join("config.toml");
let first = "[products.holger]\nendpoint = \"127.0.0.1:1\"\n";
let second = format!("{first}[products.holger.bundles.\"10gib\"]\ncap_delta = {{ disk_bytes = 10737418240 }}\nperiod_days = 30\n");
std::fs::write(&config, first).unwrap();
let state = AppState::in_memory(crate::registry::Registry::new());
let mut last = (Vec::new(), None);
let _ = look_then(&state, &config, &mut last, || std::fs::write(&config, &second).unwrap());
assert!(state.bundles.read().unwrap().is_empty(), "the look applied the config it read");
let _ = look(&state, &config, &mut last);
assert!(state.bundles.read().unwrap().get("holger").is_some_and(|b| b.contains_key("10gib")), "the config written meanwhile is applied on the next look");
let _ = std::fs::remove_dir_all(&dir);
}
/// ★ **R34-5: a look says which config it applied** — `<config>.applied` holds the sha256
/// of the config text it read, written once it is applied (the ONE box's door answers a
/// feed "applied" only when this names the config PID 1 wrote for it).
#[test]
fn a_look_says_which_config_it_applied() {
let dir = std::env::temp_dir().join(format!("monetize-guests-r34-5-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let config = dir.join("config.toml");
let text = "[products.holger]\nendpoint = \"127.0.0.1:1\"\n";
std::fs::write(&config, text).unwrap();
let state = AppState::in_memory(crate::registry::Registry::new());
let mut last = (Vec::new(), None);
let _ = look(&state, &config, &mut last);
let marker = std::path::PathBuf::from(format!("{}.applied", config.display()));
let want: String = { use sha2::Digest as _; sha2::Sha256::digest(text.as_bytes()).iter().map(|b| format!("{b:02x}")).collect() };
assert_eq!(std::fs::read_to_string(&marker).unwrap_or_default().trim(), want, "the applied config's sha256");
let _ = std::fs::remove_dir_all(&dir);
}
/// ★ **Every `[signups.<product>]` is taken in live too** — its plans, the
/// meter a plan's cap lands on (`cap_meter`, holger's `disk_bytes`;
/// gunnar's `pack_bytes` is the default), and the sink a `guest =` names,
/// each product's alone. The front's monetize starts BEFORE the installer's
/// feed, so the block that sells the base plan arrives after start; read at
/// start only, no base plan could ever be sold without a restart.
#[test]
fn a_reload_takes_in_each_products_signup_plans_meter_and_sink() {
use monetize_pay::signup::{PaidSignup, SignupOutcome};
let mut state = AppState::in_memory(crate::registry::Registry::new());
let hot = crate::signup::HotSink::per_product(Default::default());
state.signup_sink = hot.clone();
let built = std::sync::Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let b = built.clone();
let reload = crate::signup::SignupReload::new(hot, std::sync::Arc::new(move |settings: &serde_json::Value| {
b.lock().unwrap().push(settings.get("product").and_then(|p| p.as_str()).unwrap_or_default().to_owned());
Ok(std::sync::Arc::new(crate::signup::LoudSink) as crate::signup::Sink)
}));
state.signup_reload = Some(std::sync::Arc::new(reload));
assert!(state.signup_plans.is_empty());
let both = "[signups.holger]\ncap_meter = \"disk_bytes\"\nguest = \"/nowhere/holger.wasm\"\n[signups.holger.plans.\"holger/base-dk\"]\namount_minor = 7500\ncurrency = \"Dkk\"\ncap_bytes = 5368709120\nperiod_days = 30\n\
[signups.gunnar]\nguest = \"/nowhere/gunnar.wasm\"\n[signups.gunnar.plans.\"gunnar/10gb-1m\"]\namount_minor = 10000\ncurrency = \"Sek\"\ncap_bytes = 10737418240\nperiod_days = 30\n";
let cfg = Config::from_toml(both).unwrap();
let lines = apply_signup(&state, &cfg);
let plan = state.signup_plans.get("holger/base-dk").expect("the plan is taken in live");
assert_eq!((plan.amount_minor, plan.cap_bytes), (7500, 5 << 30));
assert!(state.signup_plans.get("gunnar/10gb-1m").is_some());
assert_eq!(state.signup_plans.cap_meter_of("holger"), "disk_bytes");
assert_eq!(state.signup_plans.cap_meter_of("gunnar"), crate::signup::SIGNUP_CAP_METER);
assert_eq!(*built.lock().unwrap(), ["gunnar", "holger"], "each block is built once, for its own product: {lines:?}");
assert!(lines.iter().any(|l| l.contains("[signups.holger]") && l.contains("LOADED")), "{lines:?}");
assert!(state.signup_sink.creates_accounts_for("holger") && state.signup_sink.creates_accounts_for("gunnar"), "the live sinks answer now");
let _ = apply_signup(&state, &cfg);
assert_eq!(built.lock().unwrap().len(), 2, "an unchanged block is not built again");
// The fact a signup grants lands on its product's meter.
let s = PaidSignup {
vendor: "stripe/holger".into(),
session: "cs_1".into(),
reference: "pi_1".into(),
email: "a@b.dk".into(),
username: "rust-dev".into(),
product: "holger".into(),
plan: "holger/base-dk".into(),
amount: monetize_pay::Money { minor: 9375, currency: monetize_pay::Currency::Dkk },
tax: monetize_pay::Money { minor: 1875, currency: monetize_pay::Currency::Dkk },
paid_at_unix_ms: 1_788_436_800_000,
customer_country: None,
};
let signer = monetize::Signer::generate();
let f = crate::signup::signup_fact(&state.signup_plans.snapshot(), &monetize::Policy::default(), &signer, &s, 1_788_436_800_001, &state.signup_plans.cap_meter_of("holger")).unwrap();
assert_eq!(f.caps.get("disk_bytes"), Some(&(5u64 << 30)));
assert!(!f.caps.contains_key("pack_bytes"));
let _ = SignupOutcome::Created { namespace: String::new(), username: String::new() };
// holger's block goes: its plans go, its sink goes, gunnar's stand.
let lines = apply_signup(&state, &Config::from_toml(&both[both.find("[signups.gunnar]").unwrap()..]).unwrap());
assert!(state.signup_plans.get("holger/base-dk").is_none(), "a plan that is gone is not sold");
assert_eq!(state.signup_plans.cap_meter_of("holger"), crate::signup::SIGNUP_CAP_METER);
assert!(!state.signup_sink.creates_accounts_for("holger") && state.signup_sink.creates_accounts_for("gunnar"), "{lines:?}");
assert!(lines.iter().any(|l| l.contains("[signups.holger]") && l.contains("REMOVED")), "{lines:?}");
}
}