mikronova/modgunn

tools/fetch-advisories.sh

#!/usr/bin/env sh
# fetch-advisories.sh — the INTERNET-SIDE half of the modgunn advisory pipeline.
#
# Downloads the major vulnerability feeds into a staging dir, ready to be sealed
# into a .znippy and carried across the airgap, where `modgunn warehouse import`
# (the SECURE side) reads them. This script does the network I/O the leaf modgunn
# crate deliberately does NOT — so the crate stays pure and offline.
#
# Usage:  tools/fetch-advisories.sh <staging-dir> [ecosystem ...]
#   default ecosystems: crates.io PyPI npm Go Maven NuGet RubyGems Debian Alpine
#
# Then, on the secure side:
#   modgunn warehouse import --warehouse <root> --rev "$(date +%F)" \
#       --osv  <staging>/osv \
#       --nvd  <staging>/nvd  --nvd-backfill <staging>/nvd \
#       --kev  <staging>/kev/known_exploited_vulnerabilities.json \
#       --epss <staging>/epss/epss_scores-current.csv
#
# (Seal/carry step — the agent's job: znippy the <staging> dir, verify on import.)

set -eu

STAGE="${1:?usage: fetch-advisories.sh <staging-dir> [ecosystem ...]}"
shift || true
ECOSYSTEMS="${*:-crates.io PyPI npm Go Maven NuGet RubyGems Debian Alpine}"

OSV_BASE="https://osv-vulnerabilities.storage.googleapis.com"
KEV_URL="https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
EPSS_URL="https://epss.cyentia.com/epss_scores-current.csv.gz"
# NVD CVE API 2.0 (paginate with startIndex; a mirror/snapshot is fine for airgap).
NVD_API="https://services.nvd.nist.gov/rest/json/cves/2.0"

mkdir -p "$STAGE/osv" "$STAGE/nvd" "$STAGE/kev" "$STAGE/epss"

echo "OSV.dev per-ecosystem exports → $STAGE/osv"
for eco in $ECOSYSTEMS; do
    echo "  - $eco"
    curl -fsSL "$OSV_BASE/$eco/all.zip" -o "$STAGE/osv/$eco.zip"
    # each zip is a flat set of <ID>.json OSV records.
    (cd "$STAGE/osv" && rm -rf "$eco" && mkdir "$eco" && unzip -qo "$eco.zip" -d "$eco")
done
# modgunn reads every *.json under --osv recursively? no — it reads the dir's *.json,
# so flatten the per-ecosystem records up one level (id collisions are unique CVE/GHSA ids).
find "$STAGE/osv" -mindepth 2 -name '*.json' -exec mv -f {} "$STAGE/osv/" \; 2>/dev/null || true

echo "CISA KEV → $STAGE/kev"
curl -fsSL "$KEV_URL" -o "$STAGE/kev/known_exploited_vulnerabilities.json"

echo "FIRST EPSS → $STAGE/epss"
curl -fsSL "$EPSS_URL" -o "$STAGE/epss/epss_scores-current.csv.gz"
gunzip -f "$STAGE/epss/epss_scores-current.csv.gz"

echo "NVD CVE 2.0 → $STAGE/nvd  (one page; loop startIndex for the full corpus)"
curl -fsSL "$NVD_API?resultsPerPage=2000&startIndex=0" -o "$STAGE/nvd/page-0.json" || \
    echo "  (NVD rate-limits unauthenticated pulls; use an API key + paginate, or a mirror)"

echo "done. staged under $STAGE — seal into a .znippy, carry across the gap, then run:"
echo "  modgunn warehouse import --warehouse <root> --rev $(date +%F) --osv $STAGE/osv --nvd $STAGE/nvd --nvd-backfill $STAGE/nvd --kev $STAGE/kev/known_exploited_vulnerabilities.json --epss $STAGE/epss/epss_scores-current.csv"