server/lib/src/scanner/secrets.rs
//! The SECRETS scanner (lane HOLGER-G3): keys, credentials and tokens left in a
//! package or an image layer.
//!
//! It fills the [`ScannerKind::Secrets`] slot. It walks the artifact with the
//! same inspector the INGEST walk uses ([`holger_inspect::Inspector::visit`]: zip,
//! tar, gzip, nested archives and every image layer, under the same limits) and
//! matches every member against a rule set:
//!
//! * private keys: PEM (PKCS#1, PKCS#8, EC, DSA, encrypted), OpenSSH, PGP, PuTTY;
//! * cloud credentials: AWS access key ids and secret keys, Google Cloud service
//! account keys and API keys, Azure storage account keys and Entra ID client
//! secrets;
//! * tokens: GitHub, GitLab, Slack, Stripe, npm (and `.npmrc`), PyPI, crates.io;
//! * `.env` files (and the secrets in them) and kubeconfigs with credentials;
//! * high-entropy strings assigned to a secret-like name.
//!
//! **The rules are data** ([`SecretRules`], schema [`RULES_SCHEMA`]): a RON file
//! carried over the airgap like the YARA rules. The built-in set is
//! `secrets-rules.ron`, compiled in; `promotion.intake.secrets.rules` (Tier 1:
//! `steps.secrets.rules`) names an operator file instead. The server reads and
//! checks the file at boot and hands the rules to the scan worker in the job, so
//! the match itself runs in the sandbox.
//!
//! **The evidence never holds a secret.** A finding is `secret:<rule id>`, the
//! rule's title, the member's path and the line: never the matched bytes, never a
//! hash of them.
//!
//! The scanner WARNS when it finds anything; what that means is the artifact's
//! effective policy: `rules.secrets` (the floor) and a source's
//! `gates.secrets: block | warn | off`
//! ([`crate::sources::EffectivePolicy::refusals`]). `block` refuses every high or
//! critical finding by name; medium and below stay warnings.
use std::collections::BTreeSet;
use holger_inspect::{Format, Inspector, Limits};
use regex::bytes::{Regex, RegexBuilder};
use serde::{Deserialize, Serialize};
use super::{FindingSeverity, ScanFinding, ScanOutcome, ScanSubject, Scanner, ScannerKind};
/// The scanner id in the evidence.
pub const SCANNER_ID: &str = "holger-secrets";
/// The schema label every rule file carries.
pub const RULES_SCHEMA: &str = "holger.secret-rules/v1";
/// The finding id prefix: `secret:<rule id>`.
pub const FINDING_PREFIX: &str = "secret:";
/// The built-in rule set.
pub const BUILTIN_RULES: &str = include_str!("secrets-rules.ron");
/// The most findings one artifact records; the rest are counted.
pub const MAX_FINDINGS: usize = 500;
/// The largest compiled regex, bytes (a rule file cannot exhaust the worker).
const REGEX_SIZE_LIMIT: usize = 4 << 20;
/// One rule, as the RON writes it.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SecretRule {
pub id: String,
pub title: String,
pub severity: FindingSeverity,
#[serde(default)]
pub pattern: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub path_allow: Vec<String>,
#[serde(default)]
pub keywords: Vec<String>,
#[serde(default)]
pub entropy_min: Option<f64>,
#[serde(default)]
pub allow: Vec<String>,
}
/// **A rule set** (`holger.secret-rules/v1`).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SecretRules {
pub schema: String,
/// The set's own version; it is part of the scanner's version, so stored
/// evidence made under other rules is stale.
pub version: String,
/// Placeholders for every rule (regexes over the secret).
#[serde(default)]
pub allow: Vec<String>,
/// Member paths no rule looks at (regexes).
#[serde(default)]
pub skip_paths: Vec<String>,
pub rules: Vec<SecretRule>,
}
/// `promotion.intake.secrets` (Tier 1: `steps.secrets`).
///
/// ```ron
/// secrets: (rules: "/var/lib/holger/secret-rules.ron"),
/// ```
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SecretsSettings {
/// A rule file (`holger.secret-rules/v1`). Absent: the built-in set.
#[serde(default)]
pub rules: Option<String>,
}
impl SecretRules {
/// The built-in set.
pub fn builtin() -> Self {
Self::from_ron(BUILTIN_RULES).expect("the built-in secret rules are valid (tested)")
}
/// Read and check a rule file's text: the schema, unique ids, every regex.
pub fn from_ron(text: &str) -> anyhow::Result<Self> {
let r: SecretRules = crate::sources::ron_options()
.from_str(text)
.map_err(|e| anyhow::anyhow!("not a {RULES_SCHEMA} document: {e}"))?;
r.compile()?;
Ok(r)
}
/// The rules `settings` names: the operator's file, else the built-in set.
pub fn load(settings: Option<&SecretsSettings>) -> anyhow::Result<Self> {
match settings
.and_then(|s| s.rules.as_deref())
.filter(|p| !p.trim().is_empty())
{
None => Ok(Self::builtin()),
Some(path) => {
let text = std::fs::read_to_string(path).map_err(|e| {
anyhow::anyhow!("config `promotion.intake.secrets.rules`: {path}: {e}")
})?;
Self::from_ron(&text).map_err(|e| {
anyhow::anyhow!("config `promotion.intake.secrets.rules`: {path}: {e}")
})
}
}
}
/// A short digest of the rules, for the scanner's version.
pub fn digest(&self) -> String {
let doc = serde_json::to_vec(self).unwrap_or_default();
blake3::hash(&doc).to_hex()[..16].to_string()
}
fn compile(&self) -> anyhow::Result<Compiled> {
if self.schema != RULES_SCHEMA {
anyhow::bail!("schema {:?}, not {RULES_SCHEMA:?}", self.schema);
}
let re = |what: &str, s: &str| -> anyhow::Result<Regex> {
RegexBuilder::new(s)
.size_limit(REGEX_SIZE_LIMIT)
.dfa_size_limit(REGEX_SIZE_LIMIT)
.build()
.map_err(|e| anyhow::anyhow!("{what}: the regex {s:?} does not compile: {e}"))
};
let mut ids = BTreeSet::new();
let mut rules = Vec::new();
for r in &self.rules {
let ok_id = !r.id.is_empty()
&& r.id.len() <= 64
&& r.id
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-');
if !ok_id {
anyhow::bail!("rule id {:?} is not `[a-z0-9-]`, 1 to 64 characters", r.id);
}
if !ids.insert(r.id.clone()) {
anyhow::bail!("two rules are named {:?}", r.id);
}
if r.title.trim().is_empty() || r.title.len() > 200 {
anyhow::bail!("rule {:?} needs a title (at most 200 characters)", r.id);
}
if r.pattern.is_none() && r.path.is_none() {
anyhow::bail!(
"rule {:?} has neither a pattern nor a path, so it can never fire",
r.id
);
}
if let Some(e) = r.entropy_min {
if !(0.0..=8.0).contains(&e) {
anyhow::bail!(
"rule {:?}: entropy_min {e} is not between 0 and 8 bits per byte",
r.id
);
}
}
let at = format!("rule {:?}", r.id);
let pattern = r.pattern.as_deref().map(|p| re(&at, p)).transpose()?;
let secret_group = pattern
.as_ref()
.is_some_and(|p| p.capture_names().any(|n| n == Some("secret")));
rules.push(CompiledRule {
rule: r.clone(),
pattern,
secret_group,
path: r.path.as_deref().map(|p| re(&at, p)).transpose()?,
path_allow: r
.path_allow
.iter()
.map(|p| re(&at, p))
.collect::<anyhow::Result<_>>()?,
allow: r
.allow
.iter()
.map(|p| re(&at, p))
.collect::<anyhow::Result<_>>()?,
keywords: r
.keywords
.iter()
.map(|k| k.to_ascii_lowercase().into_bytes())
.filter(|k| !k.is_empty())
.collect(),
});
}
Ok(Compiled {
rules,
allow: self
.allow
.iter()
.map(|p| re("allow", p))
.collect::<anyhow::Result<_>>()?,
skip_paths: self
.skip_paths
.iter()
.map(|p| re("skip_paths", p))
.collect::<anyhow::Result<_>>()?,
})
}
}
struct CompiledRule {
rule: SecretRule,
pattern: Option<Regex>,
secret_group: bool,
path: Option<Regex>,
path_allow: Vec<Regex>,
allow: Vec<Regex>,
keywords: Vec<Vec<u8>>,
}
struct Compiled {
rules: Vec<CompiledRule>,
allow: Vec<Regex>,
skip_paths: Vec<Regex>,
}
/// Shannon entropy, bits per byte.
pub fn entropy(b: &[u8]) -> f64 {
if b.is_empty() {
return 0.0;
}
let mut n = [0u32; 256];
for &x in b {
n[x as usize] += 1;
}
let len = b.len() as f64;
n.iter()
.filter(|&&c| c > 0)
.map(|&c| {
let p = c as f64 / len;
-p * p.log2()
})
.sum()
}
fn contains_ci(hay: &[u8], needle: &[u8]) -> bool {
needle.len() <= hay.len()
&& hay
.windows(needle.len())
.any(|w| w.eq_ignore_ascii_case(needle))
}
/// One hit, before it becomes a finding. Never holds the secret.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct Hit {
pub rule: String,
pub path: String,
/// 1-based; 0 for a rule that fires on the file alone.
pub line: usize,
}
impl Compiled {
/// The hits in one member. `path` is its path through the archives.
fn member(&self, path: &str, bytes: &[u8], out: &mut Vec<(usize, Hit)>) {
if self.skip_paths.iter().any(|r| r.is_match(path.as_bytes())) {
return;
}
for (i, r) in self.rules.iter().enumerate() {
if let Some(p) = &r.path {
if !p.is_match(path.as_bytes())
|| r.path_allow.iter().any(|a| a.is_match(path.as_bytes()))
{
continue;
}
}
if !r.keywords.is_empty() && !r.keywords.iter().any(|k| contains_ci(bytes, k)) {
continue;
}
let Some(pattern) = &r.pattern else {
out.push((
i,
Hit {
rule: r.rule.id.clone(),
path: path.to_string(),
line: 0,
},
));
continue;
};
// One finding per rule and member: the first real match.
for c in pattern.captures_iter(bytes) {
let whole = c.get(0).expect("group 0");
let secret = if r.secret_group {
c.name("secret").unwrap_or(whole)
} else {
whole
};
let s = secret.as_bytes();
if r.rule.entropy_min.is_some_and(|min| entropy(s) < min) {
continue;
}
if r.allow.iter().chain(&self.allow).any(|a| a.is_match(s)) {
continue;
}
let line = 1 + bytes[..secret.start()]
.iter()
.filter(|&&b| b == b'\n')
.count();
out.push((
i,
Hit {
rule: r.rule.id.clone(),
path: path.to_string(),
line,
},
));
break;
}
}
}
}
/// What one scan found.
#[derive(Debug, Clone, Default)]
pub struct SecretsReport {
pub hits: Vec<(FindingSeverity, String, Hit)>,
pub members: u64,
pub too_large: u64,
}
/// ★ The secrets scanner.
pub struct SecretsScanner {
rules: SecretRules,
compiled: Compiled,
inspector: Inspector,
version: String,
}
impl SecretsScanner {
/// The scanner over `rules`, walking under `limits`.
pub fn new(rules: SecretRules, limits: Limits) -> anyhow::Result<Self> {
let compiled = rules.compile()?;
let version = format!(
"holger-server-lib {}; rules {} blake3:{}",
env!("CARGO_PKG_VERSION"),
rules.version,
rules.digest()
);
Ok(Self {
rules,
compiled,
inspector: Inspector::new(limits),
version,
})
}
pub fn rules(&self) -> &SecretRules {
&self.rules
}
/// Walk one artifact and collect every hit.
pub fn report(&self, subject: &ScanSubject<'_>) -> SecretsReport {
let file = subject
.file_name
.rsplit('/')
.next()
.unwrap_or(subject.file_name);
let format = if file.ends_with(".oci.tar") {
Format::OciImage
} else {
Format::declared(file, subject.ecosystem)
};
let mut raw: Vec<(usize, Hit)> = Vec::new();
let mut members = 0u64;
let (_, too_large) =
self.inspector
.visit(subject.bytes, format, file, &mut |_name, path, bytes| {
members += 1;
self.compiled.member(path, bytes, &mut raw);
});
raw.sort_by(|a, b| a.1.cmp(&b.1));
raw.dedup_by(|a, b| a.1 == b.1);
let hits = raw
.into_iter()
.map(|(i, h)| {
let r = &self.compiled.rules[i].rule;
(r.severity, r.title.clone(), h)
})
.collect();
SecretsReport {
hits,
members,
too_large,
}
}
}
fn sev(s: FindingSeverity) -> &'static str {
match s {
FindingSeverity::Info => "info",
FindingSeverity::Low => "low",
FindingSeverity::Medium => "medium",
FindingSeverity::High => "high",
FindingSeverity::Critical => "critical",
}
}
impl Scanner for SecretsScanner {
fn id(&self) -> &str {
SCANNER_ID
}
fn version(&self) -> String {
self.version.clone()
}
fn kind(&self) -> ScannerKind {
ScannerKind::Secrets
}
fn scan(&self, subject: &ScanSubject<'_>) -> anyhow::Result<ScanOutcome> {
let rep = self.report(subject);
let mut hits = rep.hits;
hits.sort_by(|a, b| b.0.cmp(&a.0).then(a.2.cmp(&b.2)));
let total = hits.len();
let count = |s: FindingSeverity| hits.iter().filter(|h| h.0 == s).count();
let skipped = if rep.too_large > 0 {
format!(
"; {} member(s) larger than the scan cap were not read",
rep.too_large
)
} else {
String::new()
};
if total == 0 {
return Ok(ScanOutcome::pass(format!(
"no secrets in {} member(s) ({} rules, set {}){skipped}",
rep.members,
self.rules.rules.len(),
self.rules.version
)));
}
let detail = format!(
"{total} secret(s) in {} member(s): {} critical, {} high, {} medium, {} low{}{skipped}",
rep.members,
count(FindingSeverity::Critical),
count(FindingSeverity::High),
count(FindingSeverity::Medium),
count(FindingSeverity::Low),
if total > MAX_FINDINGS {
format!("; the first {MAX_FINDINGS} are listed")
} else {
String::new()
},
);
let findings = hits
.into_iter()
.take(MAX_FINDINGS)
.map(|(s, title, h)| {
let at = if h.line > 0 {
format!("{}:{}", h.path, h.line)
} else {
h.path.clone()
};
ScanFinding::new(
format!("{FINDING_PREFIX}{}", h.rule),
s,
format!("{title} ({}) at {at}", sev(s)),
)
})
.collect();
Ok(ScanOutcome::warn(findings, detail))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write as _;
// Every planted secret is assembled at run time from pieces, so no scanner
// (ours or a forge's) ever sees a whole fake secret in this source file.
fn j(parts: &[&str]) -> String {
parts.concat()
}
fn rsa_pem() -> String {
j(&[
"-----BEGIN RSA ",
"PRIVATE KEY-----\n",
"MIIEowIBAAKCAQEAxq3x9Lr2Kp8vQyTz0wN1sGm5JfXcR4bH7uYdE2aLkO9iPnV3\n",
"-----END RSA ",
"PRIVATE KEY-----\n",
])
}
fn aws_id() -> String {
j(&["AKIA", "Q3Z7XK2MRT5LW4NB"])
}
fn github() -> String {
j(&["ghp_", "R8kLm2Qx9Tz4Vb7Nc1Wd5Ys3Hf6Jg0Pa2Ue8"])
}
fn stripe_live() -> String {
j(&["sk_", "live_", "51HqT8vLkQ2mZx9Wc4Rb7Yd3N"])
}
fn npm_tok() -> String {
j(&["npm_", "Z8xQ2vLm4Tk9Rb7Wc1Yd5Ns3Hf6Jg0Pa2Ue7"])
}
fn scanner() -> SecretsScanner {
SecretsScanner::new(SecretRules::builtin(), Limits::default()).unwrap()
}
fn ids(o: &ScanOutcome) -> Vec<String> {
o.findings.iter().map(|f| f.id.clone()).collect()
}
fn wheel(files: &[(&str, &[u8])]) -> Vec<u8> {
zip_entry::fixture::zip(files)
}
fn tar_of(files: &[(&str, &[u8])]) -> Vec<u8> {
let mut b = tar::Builder::new(Vec::new());
for (p, d) in files {
let mut h = tar::Header::new_ustar();
h.set_size(d.len() as u64);
h.set_mode(0o644);
h.set_entry_type(tar::EntryType::Regular);
b.append_data(&mut h, p, *d).unwrap();
}
b.into_inner().unwrap()
}
fn gz(b: &[u8]) -> Vec<u8> {
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
e.write_all(b).unwrap();
e.finish().unwrap()
}
fn assert_no_secret_in(o: &ScanOutcome, secrets: &[String]) {
let text = format!("{} {:?}", o.detail, o.findings);
for s in secrets {
assert!(
!text.contains(s.as_str()),
"the evidence leaks a secret: {text}"
);
assert!(
!text.contains(&s[..12.min(s.len())]),
"the evidence leaks part of a secret: {text}"
);
}
}
#[test]
fn the_builtin_rules_compile_and_name_every_family() {
let r = SecretRules::builtin();
assert_eq!(r.schema, RULES_SCHEMA);
for id in [
"private-key-pem",
"private-key-openssh",
"private-key-pgp",
"aws-access-key-id",
"aws-secret-access-key",
"gcp-service-account-key",
"azure-storage-account-key",
"github-token",
"gitlab-token",
"slack-token",
"stripe-live-key",
"npm-token",
"pypi-token",
"crates-io-token",
"kubeconfig-credentials",
"dotenv-file",
"generic-high-entropy-secret",
] {
assert!(r.rules.iter().any(|x| x.id == id), "no rule {id}");
}
}
#[test]
fn a_wheel_with_planted_secrets_is_found_by_path_and_rule_never_by_value() {
let cfg = j(&[
"[default]\naws_access_key_id = ",
&aws_id(),
"\naws_secret_access_key = ",
"wJa7rXUtnFEMI/K7MDENG/bPxRfiCYzEKqLm9TzQ",
"\n",
]);
let whl = wheel(&[
("demo/__init__.py", b"print('hi')\n"),
("demo/keys/id_rsa", rsa_pem().as_bytes()),
(
"demo/settings.py",
format!("import os\n\nGITHUB = '{}'\n", github()).as_bytes(),
),
("demo/aws/credentials", cfg.as_bytes()),
]);
let s = scanner();
let o = s
.scan(&ScanSubject::new("demo-1.0-py3-none-any.whl", "pip", &whl))
.unwrap();
assert_eq!(o.verdict, crate::scanner::ScanVerdict::Warn, "{o:?}");
let got = ids(&o);
for want in [
"secret:private-key-pem",
"secret:github-token",
"secret:aws-access-key-id",
"secret:aws-secret-access-key",
] {
assert!(got.iter().any(|g| g == want), "{want} not in {got:?}");
}
let gh = o
.findings
.iter()
.find(|f| f.id == "secret:github-token")
.unwrap();
assert!(
gh.summary
.contains("demo-1.0-py3-none-any.whl!demo/settings.py:3"),
"{}",
gh.summary
);
assert_eq!(gh.severity, FindingSeverity::High);
let pem = o
.findings
.iter()
.find(|f| f.id == "secret:private-key-pem")
.unwrap();
assert_eq!(pem.severity, FindingSeverity::Critical);
assert_no_secret_in(
&o,
&[
github(),
aws_id(),
"wJa7rXUtnFEMI/K7MDENG".into(),
"MIIEowIBAAKCAQEAxq3x9".into(),
],
);
}
#[test]
fn an_npm_tarball_with_an_npmrc_a_dotenv_and_a_stripe_key_is_found() {
let env = j(&[
"NODE_ENV=production\nSTRIPE_SECRET_KEY=",
&stripe_live(),
"\n",
]);
let npmrc = j(&["//registry.npmjs.org/:_authToken=", &npm_tok(), "\n"]);
let tgz = gz(&tar_of(&[
(
"package/package.json",
br#"{"name":"demo","version":"1.0.0"}"#,
),
("package/.env", env.as_bytes()),
("package/.npmrc", npmrc.as_bytes()),
(
"package/.env.example",
b"STRIPE_SECRET_KEY=sk_live_your_key_here\n",
),
]));
let o = scanner()
.scan(&ScanSubject::new("demo-1.0.0.tgz", "npm", &tgz))
.unwrap();
let got = ids(&o);
for want in [
"secret:stripe-live-key",
"secret:dotenv-secret",
"secret:dotenv-file",
"secret:npm-token",
"secret:npmrc-auth-token",
] {
assert!(got.iter().any(|g| g == want), "{want} not in {got:?}");
}
assert!(
!o.findings
.iter()
.any(|f| f.summary.contains(".env.example")),
"an .env.example is a template, not a secret: {:?}",
o.findings
);
assert_no_secret_in(&o, &[stripe_live(), npm_tok()]);
}
#[test]
fn an_image_layer_with_a_kubeconfig_and_an_ssh_key_is_found_even_when_a_later_layer_deletes_it()
{
let kube = j(&[
"apiVersion: v1\nkind: Config\nclusters:\n- cluster:\n server: https://k8s.mock.invalid\n name: prod\nusers:\n- name: admin\n user:\n token: ",
"eyJhbGciOiJSUzI1NiIsImtpZCI6", "Ik1ZcVpXd3h0In0.eyJpc3MiOiJr\n",
]);
let ssh = j(&[
"-----BEGIN OPENSSH ",
"PRIVATE KEY-----\n",
"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\n",
"-----END OPENSSH PRIVATE KEY-----\n",
]);
let l1 = gz(&tar_of(&[
("root/.kube/config", kube.as_bytes()),
("root/.ssh/id_ed25519", ssh.as_bytes()),
("etc/os-release", b"ID=wolfi\n"),
]));
let l2 = gz(&tar_of(&[("root/.ssh/.wh.id_ed25519", b"")]));
let manifest = serde_json::to_vec(&serde_json::json!({"schemaVersion": 2, "layers": [
{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:aa"},
{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": "sha256:bb"}]}))
.unwrap();
let index = br#"{"manifests":[{"digest":"sha256:cc"}]}"#;
let layout = tar_of(&[
("index.json", index),
("blobs/sha256/cc", &manifest),
("blobs/sha256/aa", &l1),
("blobs/sha256/bb", &l2),
]);
let o = scanner()
.scan(&ScanSubject::new("demo.oci.tar", "oci", &layout))
.unwrap();
let got = ids(&o);
assert!(
got.iter().any(|g| g == "secret:kubeconfig-credentials"),
"{got:?}"
);
// The whiteout hides the key from the running container, not from whoever
// pulls the layer: the key is still in the image.
let k = o
.findings
.iter()
.find(|f| f.id == "secret:private-key-openssh")
.expect("the deleted key is still in its layer");
assert!(
k.summary.contains("blobs/sha256/aa!root/.ssh/id_ed25519"),
"{}",
k.summary
);
assert_no_secret_in(
&o,
&[
"eyJhbGciOiJSUzI1NiIsImtpZCI6".into(),
"b3BlbnNzaC1rZXktdjEAAAAABG5vbmUA".into(),
],
);
}
#[test]
fn placeholders_low_entropy_and_clean_packages_pass() {
let text = j(&[
"AWS_KEY = '",
"AKIA",
"IOSFODNN7EXAMPLE",
"'\n",
"password = 'changeme-changeme-changeme'\n",
"token = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'\n",
"api_key = os.environ['API_KEY']\n",
]);
let whl = wheel(&[("demo/conf.py", text.as_bytes())]);
let o = scanner()
.scan(&ScanSubject::new("demo-1.0-py3-none-any.whl", "pip", &whl))
.unwrap();
assert_eq!(
o.verdict,
crate::scanner::ScanVerdict::Pass,
"{:?}",
o.findings
);
}
#[test]
fn a_high_entropy_string_next_to_a_secret_name_is_a_medium_finding() {
let text = j(&[
"client_secret = \"",
"q7Zr2Lx9Vb4Nm8Tk1Wc6Yd3Hf5Jg0Pa",
"\"\n",
]);
let o = scanner()
.scan(&ScanSubject::new("conf.py", "raw", text.as_bytes()))
.unwrap();
let f = o
.findings
.iter()
.find(|f| f.id == "secret:generic-high-entropy-secret")
.expect("found");
assert_eq!(f.severity, FindingSeverity::Medium);
assert!(f.summary.ends_with("at conf.py:1"), "{}", f.summary);
}
#[test]
fn a_rule_file_is_data_and_a_bad_one_is_refused_by_name() {
let own = r#"(schema: "holger.secret-rules/v1", version: "site-1",
rules: [(id: "acme-token", title: "an ACME token", severity: critical, pattern: "acme_[a-z0-9]{16}")])"#;
let r = SecretRules::from_ron(own).unwrap();
let s = SecretsScanner::new(r, Limits::default()).unwrap();
assert!(s.version().contains("rules site-1"), "{}", s.version());
let body = j(&["x = '", "acme_", "0123456789abcdef", "'"]);
let o = s
.scan(&ScanSubject::new("f.txt", "raw", body.as_bytes()))
.unwrap();
assert_eq!(ids(&o), vec!["secret:acme-token".to_string()]);
// A GitHub token is not this site's rule.
let o = s
.scan(&ScanSubject::new("f.txt", "raw", github().as_bytes()))
.unwrap();
assert!(o.findings.is_empty());
for (bad, why) in [
(r#"(schema: "nope", version: "1", rules: [])"#, "schema"),
(
r#"(schema: "holger.secret-rules/v1", version: "1", rules: [(id: "a", title: "t", severity: high, pattern: "(")])"#,
"does not compile",
),
(
r#"(schema: "holger.secret-rules/v1", version: "1", rules: [(id: "a", title: "t", severity: high)])"#,
"can never fire",
),
(
r#"(schema: "holger.secret-rules/v1", version: "1", rules: [(id: "a", title: "t", severity: high, pattern: "x"), (id: "a", title: "t", severity: high, pattern: "y")])"#,
"two rules",
),
(
r#"(schema: "holger.secret-rules/v1", version: "1", rules: [(id: "A B", title: "t", severity: high, pattern: "x")])"#,
"rule id",
),
(
r#"(schema: "holger.secret-rules/v1", version: "1", rules: [], typo: 1)"#,
"typo",
),
] {
let e = SecretRules::from_ron(bad).unwrap_err().to_string();
assert!(e.contains(why), "{bad}: {e}");
}
}
/// A zip of every file under `dir`, by path.
fn wheel_of(dir: &str) -> Vec<u8> {
fn walk(base: &std::path::Path, d: &std::path::Path, out: &mut Vec<(String, Vec<u8>)>) {
let mut es: Vec<_> = std::fs::read_dir(d)
.unwrap_or_else(|e| panic!("{}: {e}", d.display()))
.map(|e| e.unwrap().path())
.collect();
es.sort();
for p in es {
if p.is_dir() {
walk(base, &p, out);
} else {
out.push((
p.strip_prefix(base).unwrap().to_string_lossy().into_owned(),
std::fs::read(&p).unwrap(),
));
}
}
}
let mut m = Vec::new();
walk(std::path::Path::new(dir), std::path::Path::new(dir), &mut m);
assert!(!m.is_empty(), "{dir} holds the wheel's members");
let refs: Vec<(&str, &[u8])> = m.iter().map(|(n, b)| (n.as_str(), b.as_slice())).collect();
zip_entry::fixture::zip(&refs)
}
#[test]
fn real_packages_carry_no_high_finding() {
// Real packages from the registries (the bench's intake fixtures): no false
// positive may block them under the default `secrets: block`.
let dir = concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../bench-scenarios/fixtures/intake"
);
let s = scanner();
for (f, eco) in [
("requests-2.32.3-py3-none-any.whl", "pip"),
("lodash-4.17.21.tgz", "npm"),
("serde-1.0.215.crate", "cargo"),
] {
let b = match f.strip_suffix(".whl") {
// The wheel is committed as its members (text; a .whl is a binary artifact to
// OpenSSF Scorecard): zipped here, the same members the bench makes it from.
Some(stem) => wheel_of(&format!("{dir}/{stem}")),
None => std::fs::read(format!("{dir}/{f}")).unwrap_or_else(|e| panic!("{f}: {e}")),
};
let o = s.scan(&ScanSubject::new(f, eco, &b)).unwrap();
let high: Vec<_> = o
.findings
.iter()
.filter(|x| x.severity >= FindingSeverity::High)
.collect();
assert!(high.is_empty(), "{f}: {high:?}");
assert!(
!o.detail.contains(" 0 member(s)"),
"{f} was walked: {}",
o.detail
);
}
}
#[test]
fn entropy_reads_bits_per_byte() {
assert_eq!(entropy(b"aaaa"), 0.0);
assert!((entropy(b"abcd") - 2.0).abs() < 1e-9);
}
}