server/inspect/src/magic.rs
//! Magic bytes: what a byte string IS, whatever its name says.
//!
//! A small, fixed table rather than a MIME database. Every entry is a format
//! the inspector acts on: an archive it opens, a binary or script the format
//! policy rules on, or a common file type a renamed artifact hides behind.
use serde::{Deserialize, Serialize};
/// What the leading bytes say a file is.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Kind {
Zip,
Gzip,
Tar,
Bzip2,
Xz,
Zstd,
SevenZip,
/// `!<arch>` — a Debian package, a static library.
Ar,
Rpm,
Elf,
/// A Windows PE image (`MZ` with a valid `PE\0\0` header).
Pe,
MachO,
JavaClass,
/// A `#!` script.
Script,
Png,
Jpeg,
Gif,
Pdf,
Unknown,
}
impl Kind {
/// The name in a finding.
pub fn name(&self) -> &'static str {
match self {
Kind::Zip => "zip",
Kind::Gzip => "gzip",
Kind::Tar => "tar",
Kind::Bzip2 => "bzip2",
Kind::Xz => "xz",
Kind::Zstd => "zstd",
Kind::SevenZip => "7z",
Kind::Ar => "ar",
Kind::Rpm => "rpm",
Kind::Elf => "ELF binary",
Kind::Pe => "PE binary",
Kind::MachO => "Mach-O binary",
Kind::JavaClass => "Java class",
Kind::Script => "shebang script",
Kind::Png => "PNG image",
Kind::Jpeg => "JPEG image",
Kind::Gif => "GIF image",
Kind::Pdf => "PDF",
Kind::Unknown => "unknown",
}
}
/// A native executable or library.
pub fn is_native(&self) -> bool {
matches!(self, Kind::Elf | Kind::Pe | Kind::MachO)
}
/// An archive or compressed stream: something with other files inside.
pub fn is_archive(&self) -> bool {
matches!(
self,
Kind::Zip
| Kind::Gzip
| Kind::Tar
| Kind::Bzip2
| Kind::Xz
| Kind::Zstd
| Kind::SevenZip
| Kind::Ar
| Kind::Rpm
)
}
/// An archive the inspector opens. Every other archive (7z) is BLOCKED as not
/// unpacked (lane HOL-NCSC): its members could not be inspected.
pub fn is_unpacked(&self) -> bool {
matches!(
self,
Kind::Zip
| Kind::Gzip
| Kind::Tar
| Kind::Bzip2
| Kind::Xz
| Kind::Zstd
| Kind::Ar
| Kind::Rpm
)
}
}
/// How many leading bytes [`detect`] and [`is_library`] look at.
pub const HEAD: usize = 4096;
fn u16_at(b: &[u8], at: usize, le: bool) -> Option<u16> {
let s: [u8; 2] = b.get(at..at + 2)?.try_into().ok()?;
Some(if le {
u16::from_le_bytes(s)
} else {
u16::from_be_bytes(s)
})
}
fn u32_at(b: &[u8], at: usize, le: bool) -> Option<u32> {
let s: [u8; 4] = b.get(at..at + 4)?.try_into().ok()?;
Some(if le {
u32::from_le_bytes(s)
} else {
u32::from_be_bytes(s)
})
}
/// The PE header offset, if `b` is a real PE image and not text that starts "MZ".
fn pe_header(b: &[u8]) -> Option<usize> {
if !b.starts_with(b"MZ") {
return None;
}
let at = u32_at(b, 0x3c, true)? as usize;
(b.get(at..at + 4)? == b"PE\0\0").then_some(at)
}
/// What `b` is, by its leading bytes.
pub fn detect(b: &[u8]) -> Kind {
let starts = |m: &[u8]| b.starts_with(m);
if starts(b"PK\x03\x04") || starts(b"PK\x05\x06") || starts(b"PK\x07\x08") {
Kind::Zip
} else if starts(&[0x1f, 0x8b]) {
Kind::Gzip
} else if starts(b"BZh") {
Kind::Bzip2
} else if starts(&[0xfd, b'7', b'z', b'X', b'Z', 0x00]) {
Kind::Xz
} else if starts(&[0x28, 0xb5, 0x2f, 0xfd]) {
Kind::Zstd
} else if starts(&[b'7', b'z', 0xbc, 0xaf, 0x27, 0x1c]) {
Kind::SevenZip
} else if starts(&[0xed, 0xab, 0xee, 0xdb]) {
Kind::Rpm
} else if starts(b"!<arch>\n") {
Kind::Ar
} else if starts(&[0x7f, b'E', b'L', b'F']) {
Kind::Elf
} else if pe_header(b).is_some() {
Kind::Pe
} else if starts(&[0xfe, 0xed, 0xfa, 0xce])
|| starts(&[0xfe, 0xed, 0xfa, 0xcf])
|| starts(&[0xce, 0xfa, 0xed, 0xfe])
|| starts(&[0xcf, 0xfa, 0xed, 0xfe])
{
Kind::MachO
} else if starts(&[0xca, 0xfe, 0xba, 0xbe]) {
// Shared by Java class files and fat Mach-O. A fat binary counts its
// architectures here (a handful); a class file has its version (>= 45).
match u32_at(b, 4, false) {
Some(n) if n < 45 => Kind::MachO,
_ => Kind::JavaClass,
}
} else if starts(b"#!") {
Kind::Script
} else if starts(&[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]) {
Kind::Png
} else if starts(&[0xff, 0xd8, 0xff]) {
Kind::Jpeg
} else if starts(b"GIF87a") || starts(b"GIF89a") {
Kind::Gif
} else if starts(b"%PDF-") {
Kind::Pdf
} else if b.get(257..262) == Some(b"ustar") {
Kind::Tar
} else {
Kind::Unknown
}
}
/// The kinds a file name's extension promises, or `None` when the extension
/// promises nothing the table knows.
pub fn kinds_for_name(name: &str) -> Option<&'static [Kind]> {
let lower = name.to_ascii_lowercase();
let ext = lower.rsplit('/').next().unwrap_or(&lower);
let ends = |s: &str| ext.ends_with(s);
Some(
if ends(".whl")
|| ends(".jar")
|| ends(".war")
|| ends(".ear")
|| ends(".aar")
|| ends(".nupkg")
|| ends(".zip")
|| ends(".egg")
|| ends(".vsix")
{
&[Kind::Zip]
} else if ends(".tar.gz") || ends(".tgz") || ends(".crate") || ends(".gz") {
&[Kind::Gzip]
} else if ends(".tar") || ends(".gem") {
&[Kind::Tar]
} else if ends(".bz2") {
&[Kind::Bzip2]
} else if ends(".xz") {
&[Kind::Xz]
} else if ends(".zst") {
&[Kind::Zstd]
} else if ends(".7z") {
&[Kind::SevenZip]
} else if ends(".deb") {
&[Kind::Ar]
} else if ends(".rpm") {
&[Kind::Rpm]
} else if ends(".png") {
&[Kind::Png]
} else if ends(".jpg") || ends(".jpeg") {
&[Kind::Jpeg]
} else if ends(".gif") {
&[Kind::Gif]
} else if ends(".pdf") {
&[Kind::Pdf]
} else if ends(".class") {
&[Kind::JavaClass]
} else {
return None;
},
)
}
/// Is the name one a shared library carries (`.so`, `.so.1`, `.pyd`, `.dll`,
/// `.dylib`, `.node`, `.jnilib`)?
pub fn has_library_name(name: &str) -> bool {
let lower = name.to_ascii_lowercase();
let base = lower.rsplit('/').next().unwrap_or(&lower);
[
".so", ".pyd", ".dll", ".dylib", ".node", ".jnilib", ".bundle",
]
.iter()
.any(|e| base.ends_with(e))
|| base.contains(".so.")
|| (base.contains(".cpython-") && base.contains(".so"))
}
/// Is this native binary a shared library rather than an executable?
///
/// The header must say library (ELF `ET_DYN`, a PE with the DLL flag, a Mach-O
/// dylib or bundle) AND the name must be a library's name. A PIE executable is
/// `ET_DYN` too, which is why the name is required: an `ET_DYN` without a
/// library name counts as an executable.
pub fn is_library(head: &[u8], kind: Kind, name: &str) -> bool {
if !has_library_name(name) {
return false;
}
match kind {
Kind::Elf => {
let le = head.get(5) != Some(&2);
u16_at(head, 16, le) == Some(3)
}
Kind::Pe => pe_header(head)
.and_then(|at| u16_at(head, at + 22, true))
.is_some_and(|c| c & 0x2000 != 0),
Kind::MachO => {
if head.starts_with(&[0xca, 0xfe, 0xba, 0xbe]) {
// A fat binary: the slices carry the type; the name decides.
return true;
}
let le = matches!(head.first(), Some(0xce) | Some(0xcf));
matches!(u32_at(head, 12, le), Some(6) | Some(8))
}
_ => false,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn the_table_tells_the_common_kinds_apart() {
assert_eq!(detect(b"PK\x03\x04rest"), Kind::Zip);
assert_eq!(detect(&[0x1f, 0x8b, 8, 0]), Kind::Gzip);
assert_eq!(detect(b"\x7fELF\x02\x01\x01"), Kind::Elf);
assert_eq!(detect(b"#!/bin/sh\necho hi\n"), Kind::Script);
assert_eq!(detect(b"\x89PNG\r\n\x1a\n...."), Kind::Png);
assert_eq!(
detect(b"MZ but only text"),
Kind::Unknown,
"MZ alone is not a PE"
);
assert_eq!(
detect(&[0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52]),
Kind::JavaClass
);
assert_eq!(detect(&[0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 2]), Kind::MachO);
assert_eq!(detect(b"hello"), Kind::Unknown);
}
#[test]
fn a_library_needs_both_the_header_and_the_name() {
let mut so = vec![0u8; 64];
so[..4].copy_from_slice(b"\x7fELF");
so[5] = 1;
so[16] = 3; // ET_DYN
assert!(is_library(
&so,
Kind::Elf,
"pkg/_speedups.cpython-312-x86_64-linux-gnu.so"
));
assert!(
!is_library(&so, Kind::Elf, "pkg/bin/tool"),
"a PIE executable is ET_DYN too"
);
so[16] = 2; // ET_EXEC
assert!(!is_library(&so, Kind::Elf, "pkg/libx.so"));
}
}