crates/gunnar-server/tests/orphan_store_recovery.rs
//! **Three correct guards composed into a state with no exit** (2026-08-30).
//!
//! MEASURED on the live appliance, gunnar `v1.1.3-227-g6a2367a`, request ids
//! from the server log:
//!
//! ```text
//! repo create alice/linux --hash sha1 --public
//! -> failed precondition: a store already exists at /data/repos/alice/linux.git;
//! `repo delete alice/linux` reclaims it, or pass --adopt (175daccc0000000c)
//!
//! repo create alice/linux --hash sha1 --public --adopt
//! -> failed precondition: cannot adopt the store at /data/repos/alice/linux.git:
//! it carries unmarked objects, the create asked for sha1 (175daccc0000000d)
//!
//! repo delete alice/linux --purge --yes
//! -> principal alice has no admin grant on repo:alice/linux (…0b / …0e)
//! ```
//!
//! Each refusal is individually right:
//!
//! * `create` must not adopt a store it did not make — that is the
//! 2026-08-30 resurrection, where a recreated name served the deleted
//! repository's refs;
//! * `--adopt` must not put a sha256 row over a sha1 store — that serves refs
//! no client can read;
//! * `may` must answer `false` for a grant on a repository that is not in the
//! catalog — otherwise a caller can probe the ACL of repositories it cannot
//! see.
//!
//! **Composed, they are a trap.** A store directory with no catalog row and no
//! tombstone is refused by `create` because it is there, refused by `--adopt`
//! because it predates the `OBJECT_FORMAT` marker every store written before
//! the catalog lacks, and refused by `delete` because no principal can hold a grant
//! on a repository the catalog has lost. The bytes (~25-39 GB of a failed
//! kernel mirror) were stranded and the NAME was permanently unusable, on a box
//! with no shell.
//!
//! So the test is the SEQUENCE, not the parts: make a store, take its catalog
//! row away behind the API exactly as the old delete did, and then assert that
//! an operator can get out of it **using only the control plane**. The three
//! per-guard tests below it are the arms of that, each seen red on its own.
use std::sync::Arc;
use gunnar_core::{Grant, HashKind, Level, NewRepo, Principal, PrincipalKind, RepoRef, Scope};
use gunnar_grpc::{Caller, EventBus, ServerConfig, ServiceContext};
use gunnar_proto::repos_server::Repos as _;
use gunnar_proto::{
CreateRepoRequest, DeleteRepoRequest, GetRepoRequest, ListDeletedRequest, ListOrphansRequest,
ListReposRequest, ReclaimOrphanRequest, RepoPath,
};
use gunnar_server::stores::{dir_bytes, ServerStores};
use gunnar_server::CoreControlPlane;
use tonic::Request;
const NS: &str = "alice";
const NAME: &str = "linux";
/// A principal with **no** grant anywhere and no account of its own. Every
/// refusal in here is proved against him, not against an empty table.
const STRANGER: &str = "bob";
struct Rig {
_dir: tempfile::TempDir,
stores: Arc<ServerStores>,
repos: gunnar_grpc::ReposService,
rt: tokio::runtime::Runtime,
}
impl Rig {
/// A server with `alice` owning the `alice` account (that is the
/// intrinsic namespace-admin rule, the ACL model's rule 2 — no grant row is written, so
/// nothing here depends on a grant that was invented for the test), and
/// `bob` registered with nothing at all.
fn new() -> Rig {
let dir = tempfile::tempdir().expect("tempdir");
let stores = Arc::new(ServerStores::open(dir.path()).expect("stores"));
stores.catalog().create_namespace(NS).expect("namespace");
let auth = stores.auth();
auth.add_principal(Principal::new(NS, NS, PrincipalKind::User).expect("id"))
.expect("alice");
auth.add_principal(Principal::new(STRANGER, STRANGER, PrincipalKind::User).expect("id"))
.expect("bob");
let plane: Arc<dyn gunnar_grpc::ControlPlane> =
Arc::new(CoreControlPlane::new(Arc::clone(&stores)));
let ctx = ServiceContext::new(plane, Arc::new(EventBus::new(16)), ServerConfig::default());
assert_eq!(
ctx.config.auth,
gunnar_grpc::AuthMode::Required,
"the allow-all escape hatch would serve every call to everybody and make every \
refusal in this file vacuous"
);
Rig {
_dir: dir,
stores,
repos: gunnar_grpc::ReposService::new(ctx),
rt: tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime"),
}
}
fn path(&self) -> RepoRef {
RepoRef::new(NS, NAME).expect("legal")
}
fn proto_path(&self) -> Option<RepoPath> {
Some(RepoPath {
namespace: NS.to_owned(),
repo: NAME.to_owned(),
})
}
fn as_caller<T>(&self, principal: &str, message: T) -> Request<T> {
let mut request = Request::new(message);
request.extensions_mut().insert(Caller {
principal_id: principal.to_owned(),
fingerprint: "SHA256:test".to_owned(),
machine: false,
authenticated: true,
});
request
}
/// **The state the appliance was in.** A real store, then the catalog row
/// taken away behind the API — which is exactly what `repo delete` did
/// before the reclaim landed, and what the 2026-08-29 kernel-mirror reboot left.
///
/// `unmarked` also removes the `OBJECT_FORMAT` file, because every store
/// written before 2026-08-30 has none and that is the arm `--adopt`
/// refused on the live box.
fn orphan(&self, hash: HashKind, unmarked: bool) -> std::path::PathBuf {
let path = self.path();
self.stores
.create_repo(NewRepo::new(path.clone()).hash_kind(hash), false)
.expect("the repository existed once");
let dir = self.stores.repo_dir(&path).expect("dir");
// Something for the bytes to be. A reclaim that reported 0 would pass
// a test that measured nothing.
std::fs::write(dir.join("PACK-STANDIN"), vec![7u8; 64 * 1024]).expect("bytes");
if unmarked {
std::fs::remove_file(dir.join(ServerStores::HASH_MARKER)).expect("unmark");
}
self.stores.catalog().delete(&path).expect("the old delete");
self.stores.close_repo(&path);
assert!(dir.is_dir(), "the fixture needs the store to survive");
assert!(
self.stores.catalog().get(&path).expect("read").is_none(),
"the fixture needs the catalog row gone"
);
assert!(
self.stores.tombstone_of(&path).expect("read").is_none(),
"the fixture needs NO tombstone: that absence is the whole bug"
);
dir
}
}
/// **The composition, and the way out of it.**
///
/// The three refusals are asserted first, as the state the box was actually
/// in; then the operator gets out with two control-plane calls and nothing
/// else. That last part is the deliverable: not that each guard is right, but
/// that the state they compose into has an exit.
#[test]
fn an_orphan_store_can_be_seen_and_reclaimed_over_the_control_plane_alone() {
let rig = Rig::new();
let dir = rig.orphan(HashKind::Sha1, true);
let bytes_before = dir_bytes(&dir);
assert!(bytes_before > 64 * 1024, "the fixture must hold real bytes");
// ── wall 1: create refuses the name, because the directory is there ───
let refused = rig
.rt
.block_on(rig.repos.create(rig.as_caller(
NS,
CreateRepoRequest {
path: rig.proto_path(),
hash_kind: gunnar_proto::HashKind::Sha1 as i32,
..Default::default()
},
)))
.expect_err("a create must not adopt a store it did not make");
assert_eq!(refused.code(), tonic::Code::FailedPrecondition);
assert!(
refused.message().contains("a store already exists at"),
"{}",
refused.message()
);
// ── wall 2 was `--adopt` refusing an unmarked store. It is now the FIRST
// exit, and it is proved on its own fixture below
// (`adopt_takes_an_unmarked_store_at_the_callers_word_and_marks_it`),
// because taking the store consumes it.
// ── wall 3: the access check, on a repository no grant can name ───────
//
// alice OWNS this account and is intrinsically its admin. Before 2026-08-30
// this is where the box stopped: `principal alice has no admin grant on
// repo:alice/linux`.
let deleted = rig.rt.block_on(rig.repos.delete(rig.as_caller(
NS,
DeleteRepoRequest {
path: rig.proto_path(),
purge: true,
..Default::default()
},
)));
let deleted = match deleted {
Ok(response) => response.into_inner(),
Err(status) => panic!(
"the account's own admin must be able to delete an orphan; this is the appliance's \
`175daccc0000000e`: {} ({:?})",
status.message(),
status.code()
),
};
assert!(deleted.purged, "--purge on an orphan reclaims the bytes");
assert!(
deleted.bytes_reclaimed >= bytes_before,
"the reclaim reported {} bytes, the store held {bytes_before}",
deleted.bytes_reclaimed
);
assert!(!dir.exists(), "the store directory is gone");
// ── and the NAME is usable again, which is the other half of the damage ─
rig.rt
.block_on(rig.repos.create(rig.as_caller(
NS,
CreateRepoRequest {
path: rig.proto_path(),
hash_kind: gunnar_proto::HashKind::Sha256 as i32,
..Default::default()
},
)))
.expect("the name is free once the bytes are gone");
let listed = rig
.rt
.block_on(rig.repos.list(rig.as_caller(
NS,
ListReposRequest {
namespace: NS.to_owned(),
..Default::default()
},
)))
.expect("list")
.into_inner();
assert_eq!(listed.repos.len(), 1, "one live repository at the name");
}
/// **The door that would have made the incident a one-liner.** An orphan is in
/// no other listing: not `Repos.List` (no catalog row), not
/// `Repos.ListDeleted` (no tombstone). `Repos.ListOrphans` is the third, and
/// `Repos.ReclaimOrphan` is its verb.
#[test]
fn list_orphans_reports_what_no_other_listing_can_see_and_reclaim_takes_it() {
let rig = Rig::new();
let dir = rig.orphan(HashKind::Sha1, true);
let bytes_before = dir_bytes(&dir);
// Invisible to both existing listings — the fact that made the appliance's
// 25 GB unfindable.
assert!(
rig.rt
.block_on(rig.repos.list(rig.as_caller(
NS,
ListReposRequest {
namespace: NS.to_owned(),
..Default::default()
}
)))
.expect("list")
.into_inner()
.repos
.is_empty(),
"an orphan has no catalog row, so `repo list` cannot show it"
);
assert!(
rig.rt
.block_on(rig.repos.list_deleted(rig.as_caller(
NS,
ListDeletedRequest {
namespace: NS.to_owned()
}
)))
.expect("list-deleted")
.into_inner()
.repos
.is_empty(),
"an orphan has no tombstone, so `repo list-deleted` cannot show it"
);
// …and visible to the new one, with everything an operator needs to decide.
let orphans = rig
.rt
.block_on(rig.repos.list_orphans(rig.as_caller(
NS,
ListOrphansRequest {
namespace: NS.to_owned(),
},
)))
.expect("list-orphans")
.into_inner();
assert_eq!(orphans.orphans.len(), 1, "the orphan is listed");
let row = &orphans.orphans[0];
assert_eq!(
row.repo
.as_ref()
.map(|p| (p.namespace.as_str(), p.repo.as_str())),
Some((NS, NAME))
);
assert_eq!(
row.dir,
dir.display().to_string(),
"the path, for a box with no shell"
);
assert_eq!(row.bytes, bytes_before);
assert_eq!(orphans.total_bytes, bytes_before);
assert!(
row.hash_marker.is_empty(),
"this store predates the marker, and the listing must SAY so rather than guess a hash"
);
assert!(
row.modified_unix_ms > 0,
"an mtime tells wreckage from live traffic"
);
// The verb.
let done = rig
.rt
.block_on(rig.repos.reclaim_orphan(rig.as_caller(
NS,
ReclaimOrphanRequest {
path: rig.proto_path(),
},
)))
.expect("the account's own admin may reclaim an orphan")
.into_inner();
assert_eq!(done.bytes_reclaimed, bytes_before);
assert!(!dir.exists());
assert!(
rig.rt
.block_on(rig.repos.list_orphans(rig.as_caller(
NS,
ListOrphansRequest {
namespace: NS.to_owned()
}
)))
.expect("list-orphans")
.into_inner()
.orphans
.is_empty(),
"nothing is stranded any more"
);
}
/// **The fix must not be a hole.** A principal with no grant on the namespace
/// is still refused, by name, on both new-reaching verbs — and the refusal
/// names the REPOSITORY, so which of the two questions was asked is not an
/// existence oracle.
#[test]
fn a_principal_without_the_namespace_grant_is_still_refused_by_name() {
let rig = Rig::new();
let dir = rig.orphan(HashKind::Sha1, true);
for (verb, status) in [
(
"Repos.Delete",
rig.rt
.block_on(rig.repos.delete(rig.as_caller(
STRANGER,
DeleteRepoRequest {
path: rig.proto_path(),
purge: true,
..Default::default()
},
)))
.err(),
),
(
"Repos.ReclaimOrphan",
rig.rt
.block_on(rig.repos.reclaim_orphan(rig.as_caller(
STRANGER,
ReclaimOrphanRequest {
path: rig.proto_path(),
},
)))
.err(),
),
(
"Repos.ListOrphans",
rig.rt
.block_on(rig.repos.list_orphans(rig.as_caller(
STRANGER,
ListOrphansRequest {
namespace: NS.to_owned(),
},
)))
.err(),
),
] {
let status = status.unwrap_or_else(|| panic!("{verb} must refuse a stranger"));
assert_eq!(status.code(), tonic::Code::PermissionDenied, "{verb}");
assert!(
status.message().contains(STRANGER),
"{verb} must name the principal: {}",
status.message()
);
}
// The two admin verbs name the repository, not the namespace: a caller with
// no grant must not learn from the wording whether the row exists.
let status = rig
.rt
.block_on(rig.repos.delete(rig.as_caller(
STRANGER,
DeleteRepoRequest {
path: rig.proto_path(),
purge: true,
..Default::default()
},
)))
.expect_err("refused");
assert!(
status.message().contains("repo:alice/linux"),
"the refusal must name the repository asked for, not the scope that answered: {}",
status.message()
);
assert!(dir.is_dir(), "a refused delete must not touch the bytes");
// …and a namespace grant, rather than account ownership, is enough. This is
// the rule stated positively: the narrowest scope that CAN hold a grant.
rig.stores
.auth()
.grant(Grant {
principal: STRANGER.to_owned(),
scope: Scope::Namespace(NS.to_owned()),
level: Level::Admin,
})
.expect("grant");
let done = rig
.rt
.block_on(rig.repos.reclaim_orphan(rig.as_caller(
STRANGER,
ReclaimOrphanRequest {
path: rig.proto_path(),
},
)))
.expect("a namespace admin may reclaim an orphan in that namespace")
.into_inner();
assert!(done.bytes_reclaimed > 0);
}
/// **`--adopt` means "take it as it is".** The second exit, and the one that
/// KEEPS the objects: every store written before 2026-08-30 is unmarked, so an
/// adopt that requires a marker refuses precisely the stores adoption exists
/// for. The caller's `--hash` is taken at their word and WRITTEN, so the store
/// is marked from then on.
#[test]
fn adopt_takes_an_unmarked_store_at_the_callers_word_and_marks_it() {
let rig = Rig::new();
let dir = rig.orphan(HashKind::Sha1, true);
let marker = dir.join(ServerStores::HASH_MARKER);
assert!(!marker.exists(), "the fixture is an unmarked store");
let repo = rig
.rt
.block_on(rig.repos.create(rig.as_caller(
NS,
CreateRepoRequest {
path: rig.proto_path(),
hash_kind: gunnar_proto::HashKind::Sha1 as i32,
adopt: true,
..Default::default()
},
)))
.expect("an unmarked store is adopted at the caller's word")
.into_inner()
.repo
.expect("a repository");
assert_eq!(repo.hash_kind, gunnar_proto::HashKind::Sha1 as i32);
assert_eq!(
std::fs::read_to_string(&marker)
.expect("the marker is WRITTEN on adoption")
.trim(),
"sha1",
"an adopted store must be marked, so the NEXT adopt is checked rather than trusted again"
);
// The store's bytes are still there — an adopt keeps the objects, which is
// the whole difference between this exit and a reclaim.
assert!(
dir.join("PACK-STANDIN").exists(),
"adoption must not touch the objects"
);
rig.rt
.block_on(rig.repos.get(rig.as_caller(
NS,
GetRepoRequest {
path: rig.proto_path(),
},
)))
.expect("the adopted repository is in the catalog");
assert!(
rig.stores.orphans().expect("orphans").is_empty(),
"an adopted store is catalogued, so it is not an orphan any more"
);
}
/// **The other arm, unweakened.** A store that CARRIES a marker is still
/// checked against it: a sha256 create over a sha1 store serves refs no client
/// can read (`fatal: protocol error`, the site test), and `--adopt` must
/// keep refusing it.
#[test]
fn adopt_still_refuses_a_marked_store_whose_marker_disagrees() {
let rig = Rig::new();
let dir = rig.orphan(HashKind::Sha1, false);
assert_eq!(
std::fs::read_to_string(dir.join(ServerStores::HASH_MARKER))
.expect("marker")
.trim(),
"sha1"
);
let refused = rig
.rt
.block_on(rig.repos.create(rig.as_caller(
NS,
CreateRepoRequest {
path: rig.proto_path(),
hash_kind: gunnar_proto::HashKind::Sha256 as i32,
adopt: true,
..Default::default()
},
)))
.expect_err("adopting a sha1 store as sha256 must stay refused");
assert_eq!(refused.code(), tonic::Code::FailedPrecondition);
assert!(
refused.message().contains("cannot adopt"),
"{}",
refused.message()
);
assert!(
refused.message().contains("sha1"),
"the refusal must name the hash the store actually is, so the retry is obvious: {}",
refused.message()
);
// …and the same call with the marker's own hash goes through.
rig.rt
.block_on(rig.repos.create(rig.as_caller(
NS,
CreateRepoRequest {
path: rig.proto_path(),
hash_kind: gunnar_proto::HashKind::Sha1 as i32,
adopt: true,
..Default::default()
},
)))
.expect("adopting a sha1 store as sha1");
}
/// **A soft delete of an orphan is refused, and that is not a fourth wall.**
///
/// The soft delete promises `repo undelete` puts the repository back *as it
/// was*, and for an orphan the server does not know what it was — no hash
/// kind, no default branch. Writing that tombstone would put a 30-day deadline
/// on bytes nobody can restore. So it is refused, and the refusal names all
/// three exits.
#[test]
fn a_soft_delete_of_an_orphan_is_refused_and_names_the_exits() {
let rig = Rig::new();
let dir = rig.orphan(HashKind::Sha1, true);
let refused = rig
.rt
.block_on(rig.repos.delete(rig.as_caller(
NS,
DeleteRepoRequest {
path: rig.proto_path(),
purge: false,
..Default::default()
},
)))
.expect_err("a tombstone with no row promises an undelete that cannot happen");
assert_eq!(refused.code(), tonic::Code::FailedPrecondition);
let text = refused.message();
assert!(text.contains("ORPHAN"), "{text}");
assert!(text.contains("--purge"), "{text}");
assert!(text.contains("repo reclaim"), "{text}");
assert!(text.contains("--adopt"), "{text}");
assert!(dir.is_dir(), "a refused delete must not touch the bytes");
assert!(
rig.stores
.tombstone_of(&rig.path())
.expect("read")
.is_none(),
"and it must not have written the tombstone it refused to write"
);
}
/// **The listings partition the disk.** Every `<name>.git` directory under the
/// repository root is in exactly one of `repo list`, `repo list-deleted` and
/// `repo list-orphans`. That invariant is what makes "invisible bytes"
/// impossible rather than merely unlikely, and it is the reason a
/// `ReclaimFailed` tombstone counts as an orphan: `list-deleted` filters those
/// out on purpose, which before this left them in no listing at all.
#[test]
fn every_directory_on_disk_is_in_exactly_one_listing() {
let rig = Rig::new();
let stores = &rig.stores;
let live = RepoRef::new(NS, "live").expect("legal");
let held = RepoRef::new(NS, "held").expect("legal");
let lost = RepoRef::new(NS, "lost").expect("legal");
let owed = RepoRef::new(NS, "owed").expect("legal");
for path in [&live, &held, &lost, &owed] {
stores
.create_repo(NewRepo::new(path.clone()).hash_kind(HashKind::Sha1), false)
.expect("create");
}
// held: a proper soft delete.
let row = stores.catalog().get(&held).expect("read").expect("row");
stores.catalog().delete(&held).expect("delete");
stores
.soft_delete(
&held,
Some(&row),
NS,
std::time::Duration::from_secs(86_400),
)
.expect("soft delete");
// lost: the old catalog-only delete.
stores.catalog().delete(&lost).expect("delete");
stores.close_repo(&lost);
// owed: a delete whose reclaim FAILED — `list-deleted` filters this reason
// out deliberately, so without `list-orphans` it is in no listing at all.
stores.catalog().delete(&owed).expect("delete");
stores.close_repo(&owed);
let owed_dir = stores.repo_dir(&owed).expect("dir");
std::fs::write(
ServerStores::tombstone_path(&owed_dir),
b"{\"deleted_at_unix_ms\":1,\"purge_after_unix_ms\":null,\"principal_id\":\"\",\
\"reason\":\"reclaim_failed\",\"repo\":null,\"note\":\"drill\"}\n",
)
.expect("tombstone");
let catalogued: Vec<String> = stores
.catalog()
.list(Some(NS))
.expect("list")
.iter()
.map(|repo| repo.key())
.collect();
let deleted: Vec<String> = stores
.tombstones()
.expect("tombstones")
.iter()
.map(|row| row.path.key())
.collect();
let orphaned: Vec<String> = stores
.orphans()
.expect("orphans")
.iter()
.map(|row| row.path.key())
.collect();
assert_eq!(catalogued, vec![live.key()], "catalogued: {catalogued:?}");
assert_eq!(deleted, vec![held.key()], "soft-deleted: {deleted:?}");
assert_eq!(
orphaned,
vec![lost.key(), owed.key()],
"orphaned — including the FAILED reclaim, which no other listing shows: {orphaned:?}"
);
// The partition, stated as a partition: four directories, four rows, no
// row twice.
let mut all: Vec<&String> = catalogued.iter().chain(&deleted).chain(&orphaned).collect();
all.sort();
all.dedup();
assert_eq!(
all.len(),
4,
"every directory in exactly one listing: {all:?}"
);
// The owed one carries its note, so an operator sees WHY it is stranded.
let owed_row = stores
.orphans()
.expect("orphans")
.into_iter()
.find(|row| row.path.key() == owed.key())
.expect("listed");
assert_eq!(owed_row.failed_reclaim.as_deref(), Some("drill"));
}