holger-cli/tests/dependency_sets_from_the_cli.rs
//! **The dependency sets from the command line** (lane HOL-ENOSPC-SETS): `holger-cli intake
//! sets`, `intake set <id>`, `intake set approve <id>` and `intake set refuse <id> --reason`,
//! against a REAL holger that pulls Python packages from a local fake index. A pull brings in
//! its whole closure as ONE set (lane HOL-DEPSET-APPROVE); the approvers judge it here exactly
//! as the console does over `/-/intake/set*`: the same code (`pull_intake::depset`) decides,
//! through the two-person rule, and the client decides nothing.
use std::io::{Read as _, Write as _};
use std::net::{TcpListener, TcpStream};
use std::path::PathBuf;
use std::time::{Duration, Instant};
use holger_cli::CliError;
use holger_server_lib::{read_ron_config, wire_holger, Holger};
use traits::test_support::hold_port;
fn token(who: &str) -> String {
format!("{who}-token-of-sufficient-length-for-holger")
}
// ── the packages: app 1.0 needs dep; lone 1.0 needs nothing ────────────────
fn registry() -> Vec<(&'static str, &'static str, Vec<&'static str>)> {
vec![
("app", "1.0", vec!["dep>=1.0"]),
("dep", "1.0", vec![]),
("lone", "1.0", vec![]),
]
}
fn crc32(data: &[u8]) -> u32 {
let mut crc = 0xFFFF_FFFFu32;
for b in data {
crc ^= *b as u32;
for _ in 0..8 {
crc = if crc & 1 != 0 {
(crc >> 1) ^ 0xEDB8_8320
} else {
crc >> 1
};
}
}
!crc
}
/// A stored-only zip of `members`.
fn zip(members: &[(String, Vec<u8>)]) -> Vec<u8> {
let (mut out, mut central) = (Vec::new(), Vec::new());
for (name, body) in members {
let crc = crc32(body);
let offset = out.len() as u32;
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
out.extend_from_slice(&[20, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
out.extend_from_slice(&crc.to_le_bytes());
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes());
out.extend_from_slice(name.as_bytes());
out.extend_from_slice(body);
central.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
central.extend_from_slice(&[20, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
central.extend_from_slice(&crc.to_le_bytes());
central.extend_from_slice(&(body.len() as u32).to_le_bytes());
central.extend_from_slice(&(body.len() as u32).to_le_bytes());
central.extend_from_slice(&(name.len() as u16).to_le_bytes());
central.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
central.extend_from_slice(&offset.to_le_bytes());
central.extend_from_slice(name.as_bytes());
}
let cd_offset = out.len() as u32;
let n = members.len() as u16;
out.extend_from_slice(¢ral);
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&n.to_le_bytes());
out.extend_from_slice(&n.to_le_bytes());
out.extend_from_slice(&(central.len() as u32).to_le_bytes());
out.extend_from_slice(&cd_offset.to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes());
out
}
fn wheel(project: &str, version: &str) -> Vec<u8> {
let (_, _, reqs) = registry()
.into_iter()
.find(|(p, v, _)| *p == project && *v == version)
.unwrap();
let mut meta = format!(
"Metadata-Version: 2.1\nName: {project}\nVersion: {version}\nLicense-Expression: MIT\n"
);
for r in reqs {
meta.push_str(&format!("Requires-Dist: {r}\n"));
}
let dist = format!("{project}-{version}.dist-info");
zip(&[
(format!("{project}/__init__.py"), b"\n".to_vec()),
(format!("{dist}/METADATA"), meta.into_bytes()),
(
format!("{dist}/WHEEL"),
b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n".to_vec(),
),
(format!("{dist}/RECORD"), Vec::new()),
])
}
fn wheel_name(project: &str, version: &str) -> String {
format!("{project}-{version}-py3-none-any.whl")
}
fn sha256_hex(b: &[u8]) -> String {
ring::digest::digest(&ring::digest::SHA256, b)
.as_ref()
.iter()
.map(|x| format!("{x:02x}"))
.collect()
}
fn route(base: &str, path: &str) -> (u16, &'static str, Vec<u8>) {
if let Some(p) = path.strip_prefix("/simple/") {
let project = p.trim_end_matches('/');
let files: Vec<serde_json::Value> = registry()
.into_iter()
.filter(|(p, _, _)| *p == project)
.map(|(p, v, _)| {
let f = wheel_name(p, v);
serde_json::json!({ "filename": f, "url": format!("{base}/files/{f}"),
"hashes": { "sha256": sha256_hex(&wheel(p, v)) } })
})
.collect();
if !files.is_empty() {
let page = serde_json::json!({ "meta": { "api-version": "1.1" }, "name": project, "files": files });
return (
200,
"application/vnd.pypi.simple.v1+json",
page.to_string().into_bytes(),
);
}
}
if let Some(f) = path.strip_prefix("/files/") {
for (p, v, _) in registry() {
if wheel_name(p, v) == f {
return (200, "application/octet-stream", wheel(p, v));
}
}
}
(404, "text/plain", b"not found".to_vec())
}
fn serve_one(mut s: TcpStream, base: &str) {
let _ = s.set_read_timeout(Some(Duration::from_secs(10)));
let mut buf = Vec::new();
let mut chunk = [0u8; 4096];
while !buf.windows(4).any(|w| w == b"\r\n\r\n") {
match s.read(&mut chunk) {
Ok(0) | Err(_) => return,
Ok(n) => buf.extend_from_slice(&chunk[..n]),
}
}
let head = String::from_utf8_lossy(&buf).to_string();
let path = head
.lines()
.next()
.unwrap_or("")
.split_whitespace()
.nth(1)
.unwrap_or("/")
.split('?')
.next()
.unwrap_or("")
.to_string();
let (status, ctype, body) = route(base, &path);
let out = format!("HTTP/1.1 {status} X\r\nContent-Length: {}\r\nContent-Type: {ctype}\r\nConnection: close\r\n\r\n", body.len());
let _ = s.write_all(out.as_bytes());
let _ = s.write_all(&body);
}
fn index() -> String {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}", l.local_addr().unwrap());
let b = base.clone();
std::thread::spawn(move || {
for s in l.incoming().flatten() {
let b2 = b.clone();
std::thread::spawn(move || serve_one(s, &b2));
}
});
base
}
// ── one Holger ─────────────────────────────────────────────────────────────
struct Running {
_holger: Holger,
_rt: tokio::runtime::Runtime,
dir: tempfile::TempDir,
_cfg: tempfile::NamedTempFile,
grpc: u16,
http: u16,
}
impl Running {
/// `holger-cli --at <this server> --token-file <who> <args…>`.
fn cli(&self, who: &str, args: &[&str]) -> Result<String, CliError> {
let tokens: PathBuf = self.dir.path().join("tokens");
std::fs::create_dir_all(&tokens).unwrap();
let file = tokens.join(who);
std::fs::write(&file, token(who)).unwrap();
let mut argv: Vec<String> = vec![
"holger-cli".into(),
"--at".into(),
format!("127.0.0.1:{}", self.grpc),
"--token-file".into(),
file.display().to_string(),
];
argv.extend(args.iter().map(|a| a.to_string()));
let home = self.dir.path().join("cli-home").display().to_string();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(holger_cli::run_args(argv, move |k| {
(k == "HOLGER_CLI_HOME").then(|| home.clone())
}))
}
}
fn start(base: &str) -> Running {
let mut last = String::new();
for _ in 0..8 {
match try_start(hold_port(), hold_port(), base) {
Ok(r) => return r,
Err(e) => last = e,
}
}
panic!("server never started; last error: {last}");
}
fn try_start(grpc: u16, http: u16, base: &str) -> Result<Running, String> {
let dir = tempfile::tempdir().unwrap();
let d = dir.path().display().to_string();
let repo = |name: &str| {
format!(
r#"(ron_name: "{name}", ron_repo_type: "raw", ron_upstreams: [], ron_in: None,
ron_out: Some((ron_storage_endpoint: "storage", ron_exposed_endpoint: "door")),
ron_data_dir: Some("{d}/{name}")),"#
)
};
let methods: Vec<String> = ["ida", "bob", "carl", "dev"]
.iter()
.map(|p| {
format!(
r#"StaticToken(subject: "{p}", token: Some("{}"))"#,
token(p)
)
})
.collect();
let ron = format!(
r#"(
network: Online,
repositories: [ {incoming} {quarantine} {approved} {production} ],
exposed_endpoints: [ (ron_name: "door", ron_url: "127.0.0.1:{grpc}", ron_http_url: Some("127.0.0.1:{http}")) ],
storage_endpoints: [ (ron_name: "storage", ron_storage_type: "znippy", ron_path: "{d}") ],
audit: (dir: "{d}/audit", enabled: true),
auth: Some((
methods: [ {methods} ],
duties: {{ "ida": [Importer], "bob": [SecurityApprover], "carl": [SecurityApprover], "dev": [Developer] }},
)),
promotion: Some((
ladder_name: "intake",
stages: [
(name: "incoming", repo: "incoming"), (name: "quarantine", repo: "quarantine"),
(name: "approved", repo: "approved"), (name: "production", repo: "production"),
],
intake: Some((
incoming: "incoming", quarantine: "quarantine", approved: "approved", production: "production",
rules: (sandbox: off),
approvals: 2,
)),
)),
pull_through: Some((
sources: [ (name: "pypi", kind: python, url: "{base}/simple/") ],
retry_after_secs: Some(1),
)),
)"#,
incoming = repo("incoming"),
quarantine = repo("quarantine"),
approved = repo("approved"),
production =
repo("production").replacen("ron_in: None,", "ron_in: None, visibility: Public,", 1),
methods = methods.join(", "),
);
let mut cfg = tempfile::NamedTempFile::new().unwrap();
cfg.write_all(ron.as_bytes()).unwrap();
let mut holger: Holger = read_ron_config(cfg.path()).map_err(|e| format!("config: {e}"))?;
holger
.instantiate_backends()
.map_err(|e| format!("backends: {e}"))?;
wire_holger(&mut holger).map_err(|e| format!("wire: {e:#}"))?;
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(4)
.enable_all()
.build()
.unwrap();
{
let _in = rt.enter();
holger.start().map_err(|e| format!("start: {e:#}"))?;
}
let r = Running {
_holger: holger,
_rt: rt,
dir,
_cfg: cfg,
grpc,
http,
};
for _ in 0..200 {
if get(http, "/healthz", "") == 200 {
return Ok(r);
}
std::thread::sleep(Duration::from_millis(20));
}
Err("never healthy".into())
}
/// One HTTP/1.1 GET with `accept`: the status.
fn get(port: u16, path: &str, accept: &str) -> u16 {
let Ok(mut c) = TcpStream::connect(("127.0.0.1", port)) else {
return 0;
};
let head = format!(
"GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\nAccept: {accept}\r\nConnection: close\r\n\r\n"
);
c.write_all(head.as_bytes()).unwrap();
let mut out = Vec::new();
let _ = c.read_to_end(&mut out);
String::from_utf8_lossy(&out)
.split_whitespace()
.nth(1)
.and_then(|s| s.parse().ok())
.unwrap_or(0)
}
const PEP691: &str = "application/vnd.pypi.simple.v1+json";
/// Ask for `project`'s page, then wait until `holger-cli intake sets` lists its set settled.
fn pulled_set(r: &Running, project: &str, root: &str) -> String {
get(r.http, &format!("/production/simple/{project}/"), PEP691);
let t = Instant::now();
loop {
let out = r.cli("bob", &["intake", "sets"]).expect("intake sets");
if let Some(block) = out
.split("DependencySet {")
.find(|b| b.contains(&format!("root: \"{root}\"")))
{
if !block.contains("status: \"resolving\"") {
return field(block, "id");
}
}
assert!(
t.elapsed() < Duration::from_secs(120),
"no settled set for {root}: {out}"
);
std::thread::sleep(Duration::from_millis(200));
}
}
/// The string `name: "…"` in a Debug rendering.
fn field(text: &str, name: &str) -> String {
let at = text
.find(&format!("{name}: \""))
.unwrap_or_else(|| panic!("no {name} in {text}"));
let rest = &text[at + name.len() + 3..];
rest[..rest.find('"').unwrap()].to_string()
}
#[test]
fn a_dependency_set_is_listed_shown_approved_and_refused_from_the_cli() {
let r = start(&index());
// `intake sets` lists the set of app's closure; `intake set <id>` shows its members.
let id = pulled_set(&r, "app", "app==1.0");
let shown = r.cli("bob", &["intake", "set", &id]).expect("intake set");
assert!(shown.contains("status: \"awaiting-approval\""), "{shown}");
for w in [
"pypi/app-1.0-py3-none-any.whl",
"pypi/dep-1.0-py3-none-any.whl",
] {
assert!(shown.contains(w), "{w} is a member: {shown}");
}
// The vote names the revision the approver was shown (SEC-REVIEW D4).
let rev = shown
.split("revision: \"")
.nth(1)
.and_then(|r| r.split('"').next())
.expect("the set shows its revision")
.to_string();
let waiting = r
.cli("bob", &["intake", "sets", "--status", "awaiting-approval"])
.expect("intake sets --status");
assert!(waiting.contains(&id), "{waiting}");
// The importer may not judge it, the server says so by name.
let e = r
.cli(
"ida",
&["intake", "set", "approve", &id, "--revision", &rev],
)
.expect_err("an importer never approves");
assert!(e.to_string().contains("holger-duty"), "{e}");
// Two approvers: the first vote waits, the second makes the whole set AVAILABLE.
let one = r
.cli(
"bob",
&[
"intake",
"set",
"approve",
&id,
"--revision",
&rev,
"--reason",
"reviewed",
],
)
.expect("bob approves");
assert!(one.contains("status: \"awaiting-approval\""), "{one}");
let two = r
.cli(
"carl",
&[
"intake",
"set",
"approve",
&id,
"--revision",
&rev,
"--reason",
"reviewed",
],
)
.expect("carl approves");
assert!(two.contains("status: \"available\""), "{two}");
assert_eq!(
get(r.http, "/production/pypi/dep-1.0-py3-none-any.whl", ""),
200,
"the dependency is served with its set"
);
// A refusal needs its reason, and blocks every waiting member.
let lone = pulled_set(&r, "lone", "lone==1.0");
let usage = r
.cli("bob", &["intake", "set", "refuse", &lone])
.expect_err("a refusal names its reason");
assert!(usage.to_string().contains("--reason"), "{usage}");
let lone_rev = {
let shown = r.cli("bob", &["intake", "set", &lone]).expect("intake set");
shown
.split("revision: \"")
.nth(1)
.and_then(|r| r.split('"').next())
.expect("the set shows its revision")
.to_string()
};
let refused = r
.cli(
"bob",
&[
"intake",
"set",
"refuse",
&lone,
"--revision",
&lone_rev,
"--reason",
"not wanted here",
],
)
.expect("bob refuses");
assert!(refused.contains("status: \"refused\""), "{refused}");
let after = r.cli("bob", &["intake", "set", &lone]).expect("intake set");
assert!(after.contains("not wanted here"), "{after}");
assert_ne!(
get(r.http, "/production/pypi/lone-1.0-py3-none-any.whl", ""),
200
);
// An unknown set is refused by the server, by name.
let e = r
.cli("bob", &["intake", "set", "no-such-set"])
.expect_err("no such set");
assert!(e.to_string().contains("no-such-set"), "{e}");
}