src/ioc.rs
//! Threat-intelligence **IOC** (Indicator-of-Compromise) feed import (feature `scan`).
//!
//! Where [`crate::feeds`] / [`crate::osv`] import *advisories* about a known-vulnerable
//! **package**, this module imports the **garm-side** signal: network/file indicators
//! that a host or artifact is *actively malicious* — a malware-serving URL, a botnet
//! C2 IP, a known-bad file hash. modgunn owns the data usable by BOTH dwarves (its
//! supply-chain verdict) and garm (its detection platform), so the IOC corpus lands
//! in modgunn's warehouse (`ioc_indicators` table) for either consumer.
//!
//! Sources are [abuse.ch](https://abuse.ch)'s public feeds (offline, the airgap
//! sealed-znippy path, same as the advisory feeds):
//! - **URLhaus** — malware-distribution URLs. ([`parse_urlhaus_csv`])
//! - **Feodo Tracker** — botnet C2 IPv4 addresses. ([`parse_feodo_csv`])
//! - **SSLBL** — malicious-SSL-certificate SHA-1 fingerprints. ([`parse_sslbl_csv`])
//!
//! Each parser is pure (no network), tolerant of the feeds' `#`-comment headers, and
//! **validates the indicator shape** (a URL has a scheme, an IP is dotted-quad, a
//! SHA-1 is 40 hex) so a stray header/garbage row is skipped rather than landing a
//! bogus indicator. A well-formed feed with zero data rows returns an empty `Vec`;
//! the importer treats a *provided-but-empty* feed as a MISSING-FEED error (a feed
//! that fetched nothing is never a fake pass).
/// The kind of indicator — drives how a consumer matches it (a URL/domain against
/// egress, an IP against connections, a hash against a file).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum IocKind {
Url,
Ipv4,
/// A CIDR network range (`1.2.3.0/24`) — a bad-*network* indicator; an IPv4
/// observable matches by containment, not exact value (see [`IocIndex::matches`]).
Cidr,
Domain,
Sha256,
Sha1,
Md5,
}
impl IocKind {
/// Inverse of [`as_str`](IocKind::as_str): parse a stored label back to a kind.
/// Unknown ⇒ `None` (never a silent default — a mislabelled row must not be
/// silently reinterpreted as a different indicator class).
pub fn from_label(s: &str) -> Option<Self> {
Some(match s.trim() {
"url" => IocKind::Url,
"ipv4" => IocKind::Ipv4,
"cidr" => IocKind::Cidr,
"domain" => IocKind::Domain,
"sha256" => IocKind::Sha256,
"sha1" => IocKind::Sha1,
"md5" => IocKind::Md5,
_ => return None,
})
}
/// Infer the kind of a **workspace observable** from its shape — a dependency's
/// download URL, a released artifact's hash, an egress address — so a consumer
/// can hand raw strings to the b6 join without pre-classifying them.
///
/// The order is deliberate and unambiguous: a scheme ⇒ `Url`; all-hex of length
/// 64/40/32 ⇒ `Sha256`/`Sha1`/`Md5`; a canonical dotted-quad ⇒ `Ipv4`; a
/// `net/prefix` ⇒ `Cidr`; a dotted label with a non-numeric TLD ⇒ `Domain`.
/// Anything else ⇒ `None` — an unclassifiable observable is skipped rather than
/// guessed into the wrong bucket, where it would silently never match.
pub fn infer(value: &str) -> Option<Self> {
let v = value.trim();
if v.is_empty() {
return None;
}
if v.contains("://") {
return Some(IocKind::Url);
}
let hex = v.len() >= 32 && v.bytes().all(|b| b.is_ascii_hexdigit());
if hex {
return match v.len() {
64 => Some(IocKind::Sha256),
40 => Some(IocKind::Sha1),
32 => Some(IocKind::Md5),
_ => None,
};
}
if is_ipv4(v) {
return Some(IocKind::Ipv4);
}
if v.contains('/') && parse_cidr(v).is_some() {
return Some(IocKind::Cidr);
}
// A domain: dotted labels whose TLD is alphabetic (so a malformed IP like
// `1.2.3.4.5` is not silently reclassified as a domain that never matches).
if let Some(tld) = v.rsplit('.').next() {
if v.contains('.') && !tld.is_empty() && tld.bytes().all(|b| b.is_ascii_alphabetic()) {
return Some(IocKind::Domain);
}
}
None
}
/// The canonical label stored in the warehouse.
pub fn as_str(self) -> &'static str {
match self {
IocKind::Url => "url",
IocKind::Ipv4 => "ipv4",
IocKind::Cidr => "cidr",
IocKind::Domain => "domain",
IocKind::Sha256 => "sha256",
IocKind::Sha1 => "sha1",
IocKind::Md5 => "md5",
}
}
}
/// One indicator of compromise.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Ioc {
pub kind: IocKind,
/// The indicator value (the URL / IP / hash).
pub value: String,
/// The threat / malware family it is associated with (`""` if unstated).
pub threat: String,
/// The feed it came from (`urlhaus` / `feodo` / `sslbl`).
pub source: String,
/// Free tags (e.g. URLhaus `elf,mips,Mozi`); may be empty.
pub tags: Vec<String>,
}
impl Ioc {
/// Construct an indicator with its `value` **canonicalized for the kind** (see
/// [`normalize_value`]) — the ONE place every parser + the [`IocIndex`] agree on
/// the stored/compared form, so a mixed-case feed entry can't cause a b6 miss.
pub fn new(
kind: IocKind,
value: &str,
threat: impl Into<String>,
source: impl Into<String>,
tags: Vec<String>,
) -> Self {
Self {
kind,
value: normalize_value(kind, value),
threat: threat.into(),
source: source.into(),
tags,
}
}
}
/// Canonicalize an indicator value for its kind — the single normalization folder.
/// Hashes AND DNS **domains** are lowercased (both case-insensitive); a **URL** and
/// **IPv4** are kept verbatim (a URL path is case-sensitive; an IP is already
/// canonical). Always trimmed. Idempotent.
fn normalize_value(kind: IocKind, value: &str) -> String {
let v = value.trim();
match kind {
IocKind::Sha256 | IocKind::Sha1 | IocKind::Md5 | IocKind::Domain => v.to_ascii_lowercase(),
IocKind::Url | IocKind::Ipv4 | IocKind::Cidr => v.to_string(),
}
}
/// Parse a dotted-quad IPv4 into a `u32` (big-endian), or `None` if not canonical.
fn ipv4_to_u32(s: &str) -> Option<u32> {
if !is_ipv4(s) {
return None;
}
let mut n: u32 = 0;
for octet in s.split('.') {
n = (n << 8) | octet.parse::<u8>().ok()? as u32;
}
Some(n)
}
/// The narrowest network a blocklist CIDR may span: a `/8` (16M addresses) is
/// already enormous; anything broader (`/0`–`/7`) in a threat feed is a feed error
/// or a hostile line, and accepting it would flag a huge slice of the IPv4 space as
/// active-malware (a mass false positive). Legitimate DROP entries are `/9`..`/24`.
const MIN_CIDR_PREFIX: u8 = 8;
/// Parse a `network/prefix` CIDR into `(network_u32, prefix_len)`; `None` if
/// malformed, the prefix is > 32, or the block is absurdly broad (< `/8`, see
/// [`MIN_CIDR_PREFIX`]) — a defense so a bad `0.0.0.0/0` feed line can't blocklist
/// the whole internet.
fn parse_cidr(cidr: &str) -> Option<(u32, u8)> {
let (net, pfx) = cidr.split_once('/')?;
let prefix: u8 = pfx.trim().parse().ok()?;
if prefix > 32 || prefix < MIN_CIDR_PREFIX {
return None;
}
Some((ipv4_to_u32(net.trim())?, prefix))
}
/// The **match predicate** for the baddie-detection b6 join (modgunn owns this; the
/// Mímir lens calls it). Indexes an IOC corpus by `(kind, normalized value)` for O(1)
/// lookup of a workspace observable — a resolved dependency URL, an artifact hash, an
/// egress IP — against the active-malware bad-list. Normalization matches ingest
/// exactly ([`normalize_value`]): hashes and DNS domains are case-folded, URLs/IPs
/// kept verbatim — so a mixed-case domain/hash observable still hits.
#[derive(Debug, Default)]
pub struct IocIndex {
by_kv: std::collections::HashMap<(IocKind, String), usize>,
/// Parsed CIDR indicators — `(network, prefix, corpus index)` — scanned for
/// containment on an IPv4 lookup (exact `by_kv` can't express "inside a range").
cidrs: Vec<(u32, u8, usize)>,
iocs: Vec<Ioc>,
}
impl IocIndex {
/// Build the index from a corpus (later duplicates for the same `(kind, value)`
/// keep the first seen). CIDR indicators are additionally parsed for containment.
pub fn build(iocs: Vec<Ioc>) -> Self {
let mut by_kv = std::collections::HashMap::new();
let mut cidrs = Vec::new();
for (i, ioc) in iocs.iter().enumerate() {
by_kv
.entry((ioc.kind, normalize_value(ioc.kind, &ioc.value)))
.or_insert(i);
if ioc.kind == IocKind::Cidr {
if let Some((net, pfx)) = parse_cidr(&ioc.value) {
cidrs.push((net, pfx, i));
}
}
}
Self { by_kv, cidrs, iocs }
}
/// The indicator matching `(kind, value)`, if the observable is on the bad-list.
/// A hit ⇒ 🔴 refuse (active malware, no reachability gate; see b6). Exact by
/// `(kind, normalized value)` — AND, for an **`Ipv4`** query, a CIDR **containment**
/// check (a workspace IP inside a bad network like a Spamhaus DROP block), so a
/// bad-network membership counts even without an exact-IP entry.
pub fn matches(&self, kind: IocKind, value: &str) -> Option<&Ioc> {
if let Some(&i) = self.by_kv.get(&(kind, normalize_value(kind, value))) {
return Some(&self.iocs[i]);
}
if kind == IocKind::Ipv4 {
if let Some(ip) = ipv4_to_u32(value.trim()) {
for &(net, pfx, i) in &self.cidrs {
// mask = the top `pfx` bits; pfx==0 ⇒ match all (mask 0).
let mask = if pfx == 0 { 0 } else { u32::MAX << (32 - pfx) };
if (ip & mask) == (net & mask) {
return Some(&self.iocs[i]);
}
}
}
}
None
}
/// The **b6 join, end to end**: classify a raw workspace observable by shape
/// ([`IocKind::infer`]), look it up, and return the matched indicator together
/// with modgunn's decided verdict ([`crate::baddie::classify_ioc`] — an
/// unconditional 🔴 refuse, since a match IS the compromise).
///
/// This is what makes b6 a *verdict* axis rather than a bare lookup: the consumer
/// (garm for an egress address, dwarves for a dependency download URL or an
/// artifact hash) hands over strings and gets back the same
/// [`BaddieVerdict`](crate::baddie::BaddieVerdict) shape the CVE axes produce, so
/// one renderer colours every node. An observable whose kind cannot be inferred,
/// or that is not on the bad-list, yields `None` — clean.
pub fn verdict_for(&self, observable: &str) -> Option<(&Ioc, crate::baddie::BaddieVerdict)> {
let kind = IocKind::infer(observable)?;
let ioc = self.matches(kind, observable)?;
let verdict =
crate::baddie::classify_ioc(ioc.kind.as_str(), &ioc.value, &ioc.threat, &ioc.source);
Some((ioc, verdict))
}
/// Number of distinct indicators indexed.
pub fn len(&self) -> usize {
self.iocs.len()
}
/// Whether the corpus is empty.
pub fn is_empty(&self) -> bool {
self.iocs.is_empty()
}
}
/// Split one CSV line into fields, honoring RFC-4180 double-quoting (`"a,b"`,
/// `""`-escaped quotes). Works for the unquoted feeds too (no quote ⇒ plain split).
///
/// `pub(crate)` so the ONE quoted-CSV splitter serves every CSV feed in the crate —
/// [`feeds::parse_exploitdb_csv`](crate::feeds::parse_exploitdb_csv) reuses it rather
/// than growing a second, subtly-different implementation (L5: no twin). Exploit-DB's
/// `description` field routinely contains commas inside quotes, so a naive
/// `split(',')` there would shift every later column.
pub(crate) fn split_csv(line: &str) -> Vec<String> {
let mut out = Vec::new();
let mut cur = String::new();
let mut in_q = false;
let mut chars = line.chars().peekable();
while let Some(c) = chars.next() {
match c {
'"' if in_q && chars.peek() == Some(&'"') => {
cur.push('"');
chars.next();
}
'"' => in_q = !in_q,
',' if !in_q => out.push(std::mem::take(&mut cur)),
_ => cur.push(c),
}
}
out.push(cur);
out
}
/// A data line is one that is neither blank nor a `#` comment.
fn data_lines(csv: &str) -> impl Iterator<Item = &str> {
csv.lines()
.map(str::trim)
.filter(|l| !l.is_empty() && !l.starts_with('#'))
}
/// Is `s` a **canonical** dotted-quad IPv4 address? Each octet is 1–3 ASCII digits
/// with no leading zero and a value ≤255. Rejecting non-canonical forms
/// (`1.2.3.04`, `001.2.3.4`, `1.2.3.+4`) keeps a malformed feed row from landing a
/// bogus indicator (`u8::from_str` alone would accept a leading zero / `+`).
fn is_ipv4(s: &str) -> bool {
let mut n = 0;
for octet in s.split('.') {
if octet.is_empty() || octet.len() > 3 || !octet.bytes().all(|b| b.is_ascii_digit()) {
return false;
}
if octet.len() > 1 && octet.starts_with('0') {
return false;
}
if octet.parse::<u8>().is_err() {
return false;
}
n += 1;
}
n == 4
}
/// Parse a **URLhaus** CSV (`csv_recent`/`csv_online`) into malware-URL IOCs. Columns
/// (quoted): `id,dateadded,url,url_status,last_online,threat,tags,urlhaus_link,reporter`.
/// A row whose URL field carries no `://` scheme (a header/garbage row) is skipped.
pub fn parse_urlhaus_csv(csv: &str) -> Vec<Ioc> {
let mut out = Vec::new();
for line in data_lines(csv) {
let f = split_csv(line);
let Some(url) = f.get(2).map(|s| s.trim()) else {
continue;
};
if !url.contains("://") {
continue; // not a URL ⇒ header or malformed row
}
let threat = f.get(5).map(|s| s.trim().to_string()).unwrap_or_default();
let tags = f
.get(6)
.map(|s| {
s.split(',')
.map(str::trim)
.filter(|t| !t.is_empty())
.map(String::from)
.collect()
})
.unwrap_or_default();
out.push(Ioc::new(IocKind::Url, url, threat, "urlhaus", tags));
}
out
}
/// Parse a **Feodo Tracker** `ipblocklist.csv` into botnet-C2 IPv4 IOCs. Columns:
/// `first_seen_utc,dst_ip,dst_port,c2_status,last_online,malware`. A row whose
/// `dst_ip` is not a dotted-quad (the header) is skipped.
pub fn parse_feodo_csv(csv: &str) -> Vec<Ioc> {
let mut out = Vec::new();
for line in data_lines(csv) {
let f = split_csv(line);
let Some(ip) = f.get(1).map(|s| s.trim()) else {
continue;
};
if !is_ipv4(ip) {
continue;
}
let threat = f.get(5).map(|s| s.trim().to_string()).unwrap_or_default();
out.push(Ioc::new(IocKind::Ipv4, ip, threat, "feodo", Vec::new()));
}
out
}
/// Parse an **SSLBL** `sslblacklist.csv` into malicious-cert SHA-1 IOCs. Columns:
/// `Listingdate,SHA1,Listingreason`. A row whose second field is not 40 hex chars
/// (the header) is skipped.
pub fn parse_sslbl_csv(csv: &str) -> Vec<Ioc> {
let mut out = Vec::new();
for line in data_lines(csv) {
let f = split_csv(line);
let Some(sha1) = f.get(1).map(|s| s.trim()) else {
continue;
};
if sha1.len() != 40 || !sha1.bytes().all(|b| b.is_ascii_hexdigit()) {
continue;
}
let threat = f.get(2).map(|s| s.trim().to_string()).unwrap_or_default();
out.push(Ioc::new(IocKind::Sha1, sha1, threat, "sslbl", Vec::new()));
}
out
}
/// Parse a **MalwareBazaar** `full.csv`/`recent` export into malware-sample SHA-256
/// IOCs. Columns: `first_seen_utc,sha256_hash,md5_hash,sha1_hash,reporter,file_name,
/// file_type_guess,mime_type,signature,…`. The `signature` (malware family) is the
/// threat; a row whose 2nd field is not 64 hex (the header) is skipped.
pub fn parse_malwarebazaar_csv(csv: &str) -> Vec<Ioc> {
let mut out = Vec::new();
for line in data_lines(csv) {
let f = split_csv(line);
let Some(sha256) = f.get(1).map(|s| s.trim()) else {
continue;
};
if sha256.len() != 64 || !sha256.bytes().all(|b| b.is_ascii_hexdigit()) {
continue;
}
let threat = f.get(8).map(|s| s.trim().to_string()).unwrap_or_default();
out.push(Ioc::new(
IocKind::Sha256,
sha256,
threat,
"malwarebazaar",
Vec::new(),
));
}
out
}
/// Parse a **Spamhaus DROP** list (`drop.txt` / `edrop.txt`) into CIDR IOCs. Each
/// data line is `network/prefix ; SBLref` (`212.92.127.0/24 ; SBL360262`); the SBL
/// reference is the threat. A line whose first token isn't a valid CIDR is skipped
/// (Spamhaus's own `;`-comment header lines and any garbage fail the CIDR check).
pub fn parse_spamhaus_drop(text: &str) -> Vec<Ioc> {
let mut out = Vec::new();
for line in data_lines(text) {
let mut parts = line.splitn(2, ';');
let cidr = parts.next().unwrap_or("").trim();
if parse_cidr(cidr).is_none() {
continue;
}
let threat = parts
.next()
.map(|s| s.trim().to_string())
.unwrap_or_default();
out.push(Ioc::new(
IocKind::Cidr,
cidr,
threat,
"spamhaus",
Vec::new(),
));
}
out
}
/// Parse a **plain IPv4 blocklist** — one canonical IPv4 per line, `#` comments — into
/// IOCs tagged with the given `source`/`threat`. The shared shape behind the several
/// plain-text IP feeds (Tor exit list, Emerging Threats compromised hosts, …); a
/// non-IPv4 line (a comment that slipped the `#` filter, a CIDR, an IPv6) is skipped.
pub fn parse_ip_blocklist(text: &str, source: &str, threat: &str) -> Vec<Ioc> {
data_lines(text)
.filter(|l| is_ipv4(l))
.map(|ip| Ioc::new(IocKind::Ipv4, ip, threat, source, Vec::new()))
.collect()
}
/// The **Tor bulk exit list** (`check.torproject.org/torbulkexitlist`) — exit-node
/// IPv4s. Not malware per se, but a risk indicator; `threat = "tor-exit-node"`.
pub fn parse_tor_exit_list(text: &str) -> Vec<Ioc> {
parse_ip_blocklist(text, "tor", "tor-exit-node")
}
/// **Emerging Threats** compromised-hosts list (`rules.emergingthreats.net/.../
/// compromised-ips.txt`) — IPv4s of hosts actively serving malware / running C2;
/// `threat = "compromised-host"`.
pub fn parse_emergingthreats_ips(text: &str) -> Vec<Ioc> {
parse_ip_blocklist(text, "emergingthreats", "compromised-host")
}
/// Map a MISP attribute `type` to an [`IocKind`]; `None` for a type that is not a
/// network/file indicator (email, text, registry key, …) — those are skipped.
fn misp_kind(t: &str) -> Option<IocKind> {
match t {
"ip-dst" | "ip-src" | "ip-dst|port" | "ip-src|port" => Some(IocKind::Ipv4),
"url" | "uri" => Some(IocKind::Url),
"domain" | "hostname" => Some(IocKind::Domain),
"md5" => Some(IocKind::Md5),
"sha1" => Some(IocKind::Sha1),
"sha256" => Some(IocKind::Sha256),
_ => None,
}
}
/// Parse a **MISP** event (the standard threat-intel sharing JSON) into [`Ioc`]s.
/// MISP is the interchange format for TI sharing communities; an event carries a set
/// of `Attribute` objects (`{type, value}`), plus `Object`s that group more
/// attributes. The attribute `type` drives the kind (see [`misp_kind`]); a
/// composite `…|port` IP value keeps only the IP. The event `info` is the threat.
/// Tolerant of the `{ "Event": {…} }` wrapper or a bare event object.
pub fn parse_misp_event(json: &str) -> Result<Vec<Ioc>, serde_json::Error> {
let doc: serde_json::Value = serde_json::from_str(json)?;
let event = doc.get("Event").unwrap_or(&doc);
let threat = event
.get("info")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let mut out = Vec::new();
// Top-level attributes + those nested inside Objects.
let mut attr_lists: Vec<&serde_json::Value> = Vec::new();
if let Some(a) = event.get("Attribute") {
attr_lists.push(a);
}
if let Some(objs) = event.get("Object").and_then(|v| v.as_array()) {
for o in objs {
if let Some(a) = o.get("Attribute") {
attr_lists.push(a);
}
}
}
for attrs in attr_lists {
let Some(arr) = attrs.as_array() else {
continue;
};
for a in arr {
let (Some(ty), Some(val)) = (
a.get("type").and_then(|v| v.as_str()),
a.get("value").and_then(|v| v.as_str()),
) else {
continue;
};
let Some(kind) = misp_kind(ty) else { continue };
// A composite `ip|port` value keeps just the IP.
let value = if ty.ends_with("|port") {
val.split('|').next().unwrap_or(val)
} else {
val
}
.trim();
if value.is_empty() {
continue;
}
out.push(Ioc::new(kind, value, threat.clone(), "misp", Vec::new()));
}
}
Ok(out)
}
/// Map an AlienVault **OTX** indicator `type` to an [`IocKind`]; `None` for the
/// types OTX carries that are not a network/file indicator (email, CVE, IPv6, …).
fn otx_kind(t: &str) -> Option<IocKind> {
match t {
"IPv4" => Some(IocKind::Ipv4),
"domain" | "hostname" => Some(IocKind::Domain),
"URL" | "URI" => Some(IocKind::Url),
"FileHash-MD5" => Some(IocKind::Md5),
"FileHash-SHA1" => Some(IocKind::Sha1),
"FileHash-SHA256" => Some(IocKind::Sha256),
_ => None,
}
}
/// Parse an **AlienVault OTX** pulse export into [`Ioc`]s. OTX is the other major TI
/// sharing feed; a pulse groups `indicators` (`{type, indicator}`) under a `name`
/// (the threat). Tolerant of the two shapes: a `{ "results": [pulse, …] }`
/// subscription export or a single bare pulse object. Types map via [`otx_kind`];
/// non-indicator types (email/CVE/IPv6) are skipped, hashes lowercased.
pub fn parse_otx_pulse(json: &str) -> Result<Vec<Ioc>, serde_json::Error> {
let doc: serde_json::Value = serde_json::from_str(json)?;
let pulses: Vec<&serde_json::Value> = match doc.get("results").and_then(|v| v.as_array()) {
Some(arr) => arr.iter().collect(),
None => vec![&doc],
};
let mut out = Vec::new();
for pulse in pulses {
let threat = pulse
.get("name")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
let Some(inds) = pulse.get("indicators").and_then(|v| v.as_array()) else {
continue;
};
for ind in inds {
let (Some(ty), Some(val)) = (
ind.get("type").and_then(|v| v.as_str()),
ind.get("indicator").and_then(|v| v.as_str()),
) else {
continue;
};
let Some(kind) = otx_kind(ty) else { continue };
let val = val.trim();
if val.is_empty() {
continue;
}
out.push(Ioc::new(kind, val, threat.clone(), "otx", Vec::new()));
}
}
Ok(out)
}
/// Parse a **GreyNoise** result document into [`Ioc`]s — the *internet-background-
/// noise* classifier, and the one feed on this list whose payload is mostly things
/// you must **NOT** blocklist.
///
/// Tolerant of the shapes GreyNoise returns: a GNQL page `{ "data": [ … ] }`, a bare
/// `[ … ]` array, or a single IP-context object. Each record carries an `ip` and a
/// `classification` ∈ `malicious` / `suspicious` / `benign` / `unknown`.
///
/// **Only `malicious` and `suspicious` become indicators.** A `benign` record — and
/// anything flagged `riot: true`, GreyNoise's *Rule It Out* set of known-good common
/// business services (Google DNS, CDN health-checkers, Microsoft update ranges) — is
/// deliberately dropped. This matters more here than for any other source: a b6 IOC
/// hit is an **unconditional refuse** with no reachability gate
/// ([`crate::baddie::classify_ioc`]), so importing GreyNoise's benign half would
/// blocklist `8.8.8.8` and refuse half the fleet. `unknown` is likewise dropped —
/// "we have not classified this" is not evidence of malice.
///
/// The `actor` (or `name`) is the threat; `tags` ride along. A record whose `ip` is
/// not a canonical dotted-quad is skipped. A malformed document is an `Err`; a
/// well-formed page whose records are all benign is a **real empty** — GreyNoise
/// legitimately returns pages of noise-but-harmless scanners.
pub fn parse_greynoise(json: &str) -> Result<Vec<Ioc>, serde_json::Error> {
let doc: serde_json::Value = serde_json::from_str(json)?;
let records: Vec<&serde_json::Value> =
if let Some(arr) = doc.get("data").and_then(|v| v.as_array()) {
arr.iter().collect()
} else if let Some(arr) = doc.as_array() {
arr.iter().collect()
} else {
vec![&doc]
};
let mut out = Vec::new();
for rec in records {
// RIOT = a known-good common service. Never an indicator, whatever else says.
if rec.get("riot").and_then(|v| v.as_bool()).unwrap_or(false) {
continue;
}
let class = rec
.get("classification")
.and_then(|v| v.as_str())
.unwrap_or("")
.trim()
.to_ascii_lowercase();
if !matches!(class.as_str(), "malicious" | "suspicious") {
continue; // benign / unknown / unstated ⇒ not evidence of malice
}
let Some(ip) = rec.get("ip").and_then(|v| v.as_str()).map(str::trim) else {
continue;
};
if !is_ipv4(ip) {
continue;
}
let actor = ["actor", "name"]
.iter()
.find_map(|k| rec.get(*k).and_then(|v| v.as_str()))
.map(str::trim)
.filter(|s| !s.is_empty() && !s.eq_ignore_ascii_case("unknown"))
.unwrap_or(&class);
let tags: Vec<String> = rec
.get("tags")
.and_then(|v| v.as_array())
.map(|a| {
a.iter()
.filter_map(|t| t.as_str())
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
})
.unwrap_or_default();
out.push(Ioc::new(IocKind::Ipv4, ip, actor, "greynoise", tags));
}
Ok(out)
}
/// Map a ThreatFox `ioc_type` to an [`IocKind`]; `None` for an unrecognized type
/// (which also transparently skips the CSV header row, whose type is `ioc_type`).
fn threatfox_kind(t: &str) -> Option<IocKind> {
match t {
"url" => Some(IocKind::Url),
"domain" => Some(IocKind::Domain),
"ip:port" => Some(IocKind::Ipv4),
"md5_hash" => Some(IocKind::Md5),
"sha1_hash" => Some(IocKind::Sha1),
"sha256_hash" => Some(IocKind::Sha256),
_ => None,
}
}
/// Parse a **ThreatFox** CSV export — the multi-kind IOC feed — into [`Ioc`]s.
/// Columns: `first_seen_utc,ioc_id,ioc_value,ioc_type,threat_type,fk_malware,…,tags,…`.
/// The `ioc_type` column drives the kind (url / domain / ip:port / *_hash); a row of
/// an unknown type (or the header) is skipped. `fk_malware` is the threat.
pub fn parse_threatfox_csv(csv: &str) -> Vec<Ioc> {
let mut out = Vec::new();
for line in data_lines(csv) {
let f = split_csv(line);
let (Some(value), Some(itype)) = (f.get(2).map(|s| s.trim()), f.get(3).map(|s| s.trim()))
else {
continue;
};
let Some(kind) = threatfox_kind(itype) else {
continue;
};
if value.is_empty() {
continue;
}
// ThreatFox `ip:port` carries the C2 endpoint as `1.2.3.4:443`, but the
// Ipv4 matcher (exact / CIDR containment) keys on a BARE ip — the `:port`
// makes the indicator unreachable against an egress-IP observable (a false
// negative). Strip it to the bare ip, as the MISP `|port` path does.
let value = if kind == IocKind::Ipv4 {
value.split(':').next().unwrap_or(value)
} else {
value
};
if value.is_empty() {
continue;
}
let threat = f.get(5).map(|s| s.trim().to_string()).unwrap_or_default();
let tags = f
.get(12)
.map(|s| {
s.split(',')
.map(str::trim)
.filter(|t| !t.is_empty())
.map(String::from)
.collect()
})
.unwrap_or_default();
out.push(Ioc::new(kind, value, threat, "threatfox", tags));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn urlhaus_parses_url_threat_and_tags_skips_header() {
let csv = "# comment\n# id,dateadded,url,url_status,...\n\
\"1\",\"2026-07-26 20:29:23\",\"http://evil.example/bin.sh\",\"online\",\"2026-07-26\",\"malware_download\",\"elf,mips,Mozi\",\"https://urlhaus...\",\"reporter\"";
let iocs = parse_urlhaus_csv(csv);
assert_eq!(iocs.len(), 1, "the comment header lines are skipped");
let i = &iocs[0];
assert_eq!(i.kind, IocKind::Url);
assert_eq!(i.value, "http://evil.example/bin.sh");
assert_eq!(i.threat, "malware_download");
assert_eq!(i.source, "urlhaus");
assert_eq!(i.tags, vec!["elf", "mips", "Mozi"]);
}
#[test]
fn urlhaus_skips_a_row_with_no_scheme() {
// A stray/garbage row whose url field carries no `://` is not an indicator.
let csv = "\"1\",\"date\",\"not-a-url\",\"online\",\"x\",\"threat\",\"\",\"link\",\"rep\"";
assert!(parse_urlhaus_csv(csv).is_empty());
}
#[test]
fn feodo_parses_c2_ips_skips_header_row() {
let csv = "# Feodo Tracker\nfirst_seen_utc,dst_ip,dst_port,c2_status,last_online,malware\n\
2026-07-01 00:00:00,45.61.136.5,443,online,2026-07-26,Emotet";
let iocs = parse_feodo_csv(csv);
assert_eq!(
iocs.len(),
1,
"the `dst_ip` header word is not an IP ⇒ skipped"
);
assert_eq!(iocs[0].kind, IocKind::Ipv4);
assert_eq!(iocs[0].value, "45.61.136.5");
assert_eq!(iocs[0].threat, "Emotet");
assert_eq!(iocs[0].source, "feodo");
}
#[test]
fn sslbl_parses_sha1_lowercased_skips_header() {
let csv = "# SSLBL\nListingdate,SHA1,Listingreason\n\
2026-07-01 00:00:00,AABBCCDDEEFF00112233445566778899AABBCCDD,Malware C2";
let iocs = parse_sslbl_csv(csv);
assert_eq!(iocs.len(), 1);
assert_eq!(iocs[0].kind, IocKind::Sha1);
assert_eq!(
iocs[0].value, "aabbccddeeff00112233445566778899aabbccdd",
"normalized lowercase"
);
assert_eq!(iocs[0].threat, "Malware C2");
}
#[test]
fn sslbl_rejects_a_non_hex_or_wrong_length_field() {
assert!(parse_sslbl_csv("2026,not-a-hash,reason").is_empty());
assert!(
parse_sslbl_csv("2026,AABBCC,reason").is_empty(),
"too short"
);
}
#[test]
fn threatfox_maps_multi_kind_and_skips_header_and_unknown() {
let csv = "# ThreatFox\n\
\"first_seen_utc\",\"ioc_id\",\"ioc_value\",\"ioc_type\",\"threat_type\",\"fk_malware\",\"a\",\"b\",\"c\",\"d\",\"e\",\"f\",\"tags\",\"g\",\"h\"\n\
\"2026-07-26\",\"1\",\"http://evil/x\",\"url\",\"payload\",\"Cobalt Strike\",\"\",\"\",\"\",\"\",\"\",\"\",\"cs,beacon\",\"\",\"rep\"\n\
\"2026-07-26\",\"2\",\"1.2.3.4:443\",\"ip:port\",\"c2\",\"Emotet\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"rep\"\n\
\"2026-07-26\",\"3\",\"deadbeef\",\"weird_type\",\"x\",\"y\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"rep\"";
let iocs = parse_threatfox_csv(csv);
assert_eq!(
iocs.len(),
2,
"header row (ioc_type=='ioc_type') + unknown type skipped"
);
assert_eq!(iocs[0].kind, IocKind::Url);
assert_eq!(iocs[0].value, "http://evil/x");
assert_eq!(iocs[0].threat, "Cobalt Strike");
assert_eq!(iocs[0].tags, vec!["cs", "beacon"]);
assert_eq!(iocs[1].kind, IocKind::Ipv4);
assert_eq!(
iocs[1].value, "1.2.3.4",
"ip:port stripped to the bare ip so the Ipv4 matcher can reach it"
);
assert!(iocs.iter().all(|i| i.source == "threatfox"));
// The stripped indicator is now matchable by a bare egress-IP observable.
let idx = IocIndex::build(iocs.clone());
assert!(
idx.matches(IocKind::Ipv4, "1.2.3.4").is_some(),
"the Emotet C2 host flags on its bare ip"
);
}
#[test]
fn malwarebazaar_parses_sha256_and_signature() {
let csv = "# MalwareBazaar\n\
\"first_seen_utc\",\"sha256_hash\",\"md5_hash\",\"sha1_hash\",\"reporter\",\"file_name\",\"file_type_guess\",\"mime_type\",\"signature\"\n\
\"2026-07-26\",\"9DBD3D33F93D61659BA4CF9213ED43B694F091BD08BA634595F68576F0881FCF\",\"m\",\"s\",\"rep\",\"x.exe\",\"exe\",\"app\",\"CobaltStrike\"";
let iocs = parse_malwarebazaar_csv(csv);
assert_eq!(iocs.len(), 1, "header row (sha256_hash literal) skipped");
assert_eq!(iocs[0].kind, IocKind::Sha256);
assert_eq!(
iocs[0].value,
"9dbd3d33f93d61659ba4cf9213ed43b694f091bd08ba634595f68576f0881fcf"
);
assert_eq!(iocs[0].threat, "CobaltStrike");
assert_eq!(iocs[0].source, "malwarebazaar");
}
#[test]
fn tor_exit_list_parses_ips_skips_comments() {
let text = "# This is a comment\n45.61.136.5\n\n185.220.101.1\nnot-an-ip";
let iocs = parse_tor_exit_list(text);
assert_eq!(iocs.len(), 2);
assert!(iocs
.iter()
.all(|i| i.kind == IocKind::Ipv4 && i.source == "tor" && i.threat == "tor-exit-node"));
assert_eq!(iocs[0].value, "45.61.136.5");
}
#[test]
fn emergingthreats_ip_list_tagged_distinctly() {
let iocs = parse_emergingthreats_ips("# ET compromised\n45.61.136.5\n185.220.101.1\n");
assert_eq!(iocs.len(), 2);
assert!(iocs
.iter()
.all(|i| i.source == "emergingthreats" && i.threat == "compromised-host"));
// shares the shape with tor but a distinct source/threat.
assert_eq!(parse_tor_exit_list("45.61.136.5\n")[0].source, "tor");
}
#[test]
fn is_ipv4_rejects_non_canonical_octets() {
assert!(is_ipv4("45.61.136.5"));
assert!(is_ipv4("0.0.0.0") && is_ipv4("255.255.255.255"));
assert!(!is_ipv4("1.2.3.04"), "leading zero");
assert!(!is_ipv4("001.2.3.4"), "leading zero");
assert!(!is_ipv4("1.2.3.+4"), "non-digit");
assert!(!is_ipv4("1.2.3"), "too few");
assert!(!is_ipv4("1.2.3.4.5"), "too many");
assert!(!is_ipv4("256.1.1.1"), "out of range");
}
#[test]
fn misp_event_extracts_attributes_and_object_attributes() {
let json = r#"{ "Event": {
"info": "APT29 infrastructure",
"Attribute": [
{ "type": "ip-dst", "value": "45.61.136.5", "category": "Network activity" },
{ "type": "url", "value": "http://evil/x" },
{ "type": "ip-src|port", "value": "1.2.3.4|443" },
{ "type": "email-src", "value": "bad@evil.com" }
],
"Object": [
{ "Attribute": [ { "type": "sha256", "value": "AABBCC" } ] }
]
} }"#;
let iocs = parse_misp_event(json).unwrap();
assert_eq!(
iocs.len(),
4,
"3 top-level indicators (email skipped) + 1 from the Object"
);
assert!(iocs
.iter()
.all(|i| i.source == "misp" && i.threat == "APT29 infrastructure"));
assert_eq!(iocs[0].value, "45.61.136.5");
assert_eq!(iocs[1].kind, IocKind::Url);
assert_eq!(
iocs[2].value, "1.2.3.4",
"the |port composite keeps only the IP"
);
assert!(
iocs.iter()
.any(|i| i.kind == IocKind::Sha256 && i.value == "aabbcc"),
"Object attr, hash lowercased"
);
}
#[test]
fn otx_pulse_extracts_indicators_both_shapes() {
let results = r#"{ "results": [ { "name": "Lazarus campaign", "indicators": [
{ "type": "IPv4", "indicator": "45.61.136.5" },
{ "type": "FileHash-SHA256", "indicator": "AABBCC" },
{ "type": "email", "indicator": "x@y.com" },
{ "type": "URL", "indicator": "http://evil/x" }
] } ] }"#;
let iocs = parse_otx_pulse(results).unwrap();
assert_eq!(iocs.len(), 3, "email skipped");
assert!(iocs
.iter()
.all(|i| i.source == "otx" && i.threat == "Lazarus campaign"));
assert!(iocs
.iter()
.any(|i| i.kind == IocKind::Sha256 && i.value == "aabbcc"));
// single bare pulse shape.
let bare =
r#"{ "name": "p", "indicators": [ { "type": "domain", "indicator": "evil.com" } ] }"#;
assert_eq!(parse_otx_pulse(bare).unwrap().len(), 1);
assert!(parse_otx_pulse("nope").is_err());
}
#[test]
fn misp_bare_event_without_wrapper_and_malformed() {
let bare = r#"{ "info": "x", "Attribute": [ { "type": "domain", "value": "evil.com" } ] }"#;
assert_eq!(
parse_misp_event(bare).unwrap().len(),
1,
"tolerates a bare event"
);
assert!(parse_misp_event("not json").is_err());
}
#[test]
fn ioc_index_matches_the_bad_list_with_normalization() {
let corpus = vec![
Ioc {
kind: IocKind::Url,
value: "http://evil/x".into(),
threat: "mw".into(),
source: "urlhaus".into(),
tags: vec![],
},
Ioc {
kind: IocKind::Sha256,
value: "AABBCC".into(),
threat: "cs".into(),
source: "mb".into(),
tags: vec![],
},
Ioc {
kind: IocKind::Ipv4,
value: "1.2.3.4".into(),
threat: "c2".into(),
source: "feodo".into(),
tags: vec![],
},
];
let idx = IocIndex::build(corpus);
assert_eq!(idx.len(), 3);
// exact URL hit.
assert_eq!(
idx.matches(IocKind::Url, "http://evil/x").unwrap().threat,
"mw"
);
// hash lookup is case-insensitive (a workspace hash spelled lowercase matches).
assert!(
idx.matches(IocKind::Sha256, "aabbcc").is_some(),
"hash normalized"
);
// IP hit.
assert!(idx.matches(IocKind::Ipv4, "1.2.3.4").is_some());
// a clean observable is not on the bad-list; and the kind must match.
assert!(idx.matches(IocKind::Url, "http://good/y").is_none());
assert!(
idx.matches(IocKind::Domain, "http://evil/x").is_none(),
"wrong kind ⇒ no match"
);
}
/// RED-when-reverted: DNS is case-insensitive, so a mixed-case domain on the
/// bad-list must match a differently-cased workspace observable (and vice versa)
/// — else a b6 detection is silently missed. Ingest lowercases domains, and the
/// index normalizes both sides through the same folder.
#[test]
fn spamhaus_cidr_containment_match() {
let text = "; Spamhaus DROP List\n212.92.127.0/24 ; SBL360262\n1.10.16.0/20 ; SBL256894\n";
let iocs = parse_spamhaus_drop(text);
assert_eq!(iocs.len(), 2, "the `;` header line is skipped");
assert!(iocs
.iter()
.all(|i| i.kind == IocKind::Cidr && i.source == "spamhaus"));
assert_eq!(iocs[0].value, "212.92.127.0/24");
assert_eq!(iocs[0].threat, "SBL360262");
// an IPv4 observable INSIDE a bad network matches by containment.
let idx = IocIndex::build(iocs);
assert!(
idx.matches(IocKind::Ipv4, "212.92.127.45").is_some(),
"IP inside the /24 is bad"
);
assert!(
idx.matches(IocKind::Ipv4, "212.92.128.1").is_none(),
"just outside the /24 is clean"
);
assert!(
idx.matches(IocKind::Ipv4, "1.10.31.255").is_some(),
"top of the /20"
);
assert!(
idx.matches(IocKind::Ipv4, "1.10.32.0").is_none(),
"just past the /20"
);
}
#[test]
fn spamhaus_rejects_absurdly_broad_cidr() {
// A bad/hostile `/0` (or /1../7) feed line must NOT land — it would flag a
// huge slice of the internet as active malware (a mass false positive).
let text = "0.0.0.0/0 ; SBL-evil\n10.0.0.0/7 ; SBL-broad\n45.0.0.0/8 ; SBL-ok\n192.168.1.0/24 ; SBL-ok\n";
let iocs = parse_spamhaus_drop(text);
assert_eq!(iocs.len(), 2, "/0 and /7 rejected; /8 and /24 kept");
let idx = IocIndex::build(iocs);
assert!(
idx.matches(IocKind::Ipv4, "8.8.8.8").is_none(),
"a random IP is NOT flagged by a rejected /0"
);
assert!(
idx.matches(IocKind::Ipv4, "45.1.2.3").is_some(),
"the legit /8 still matches"
);
}
#[test]
fn ioc_domain_match_is_case_insensitive() {
// ingest normalization: a mixed-case domain is stored lowercased.
let ioc = Ioc::new(IocKind::Domain, "API.Evil.COM", "c2", "otx", vec![]);
assert_eq!(ioc.value, "api.evil.com", "domain lowercased at ingest");
let idx = IocIndex::build(vec![Ioc::new(
IocKind::Domain,
"Evil.com",
"c2",
"misp",
vec![],
)]);
assert!(
idx.matches(IocKind::Domain, "evil.com").is_some(),
"lower query hits mixed-case entry"
);
assert!(
idx.matches(IocKind::Domain, "EVIL.COM").is_some(),
"upper query hits too"
);
// a URL path stays case-sensitive (not folded).
assert_eq!(
Ioc::new(IocKind::Url, "http://x/PaTh", "", "", vec![]).value,
"http://x/PaTh"
);
}
/// GreyNoise is the one feed whose document is mostly things you must NOT
/// blocklist. Reverting the classification filter (importing every record, the
/// shape every other IOC parser uses) lands `8.8.8.8` on an unconditional-refuse
/// bad-list — this test goes red on exactly that mistake.
#[test]
fn greynoise_imports_only_malicious_never_benign_or_riot() {
let json = r#"{ "complete": true, "count": 5, "data": [
{ "ip": "198.51.100.7", "classification": "malicious", "actor": "Mirai",
"tags": ["Mirai", "Telnet Bruteforcer"], "seen": true },
{ "ip": "203.0.113.9", "classification": "suspicious", "actor": "unknown", "tags": [] },
{ "ip": "8.8.8.8", "classification": "benign", "name": "Google Public DNS", "riot": true },
{ "ip": "192.0.2.5", "classification": "benign", "name": "Shodan.io" },
{ "ip": "192.0.2.6", "classification": "unknown" },
{ "ip": "not-an-ip", "classification": "malicious", "actor": "X" },
{ "ip": "192.0.2.7", "classification": "malicious", "riot": true, "actor": "Y" }
] }"#;
let iocs = parse_greynoise(json).unwrap();
let values: Vec<&str> = iocs.iter().map(|i| i.value.as_str()).collect();
assert_eq!(
values,
vec!["198.51.100.7", "203.0.113.9"],
"only malicious + suspicious land; benign / RIOT / unknown / malformed-IP are dropped"
);
assert!(iocs
.iter()
.all(|i| i.kind == IocKind::Ipv4 && i.source == "greynoise"));
assert_eq!(iocs[0].threat, "Mirai");
assert_eq!(
iocs[0].tags,
vec!["Mirai".to_string(), "Telnet Bruteforcer".to_string()]
);
assert_eq!(
iocs[1].threat, "suspicious",
"an `unknown` actor falls back to the classification"
);
// The b6 predicate must therefore MISS the benign IP and HIT the C2 — the
// difference between a working gate and refusing the whole fleet.
let idx = IocIndex::build(iocs);
assert!(
idx.matches(IocKind::Ipv4, "198.51.100.7").is_some(),
"C2 egress is caught"
);
assert!(
idx.matches(IocKind::Ipv4, "8.8.8.8").is_none(),
"public DNS is NOT a baddie"
);
// Shape tolerance: a bare array and a single IP-context object both parse.
assert_eq!(
parse_greynoise(r#"[{ "ip": "198.51.100.8", "classification": "malicious" }]"#)
.unwrap()
.len(),
1
);
assert_eq!(
parse_greynoise(r#"{ "ip": "198.51.100.9", "classification": "malicious" }"#)
.unwrap()
.len(),
1
);
// A page of purely-benign noise is a REAL empty (not an error) …
assert!(parse_greynoise(
r#"{ "data": [ { "ip": "1.1.1.1", "classification": "benign" } ] }"#
)
.unwrap()
.is_empty());
// … but a malformed document is an error, never a fake-empty bad-list.
assert!(parse_greynoise("<html>403</html>").is_err());
}
#[test]
fn csv_split_honors_quoted_commas() {
let f = split_csv("\"a\",\"b,c\",\"d\"\"e\"");
assert_eq!(f, vec!["a", "b,c", "d\"e"]);
}
}