feat: cloud / mobile / binary-RE offensive loadouts (arsenal + detection benchmarks)
Three new category loadouts, each = arsenal + an offline-scorable benchmark, opt-in and
honestly scoped (roster marks all 🚧; pacu/frida stay catalog_only, unit-tested non-mintable):
- cloud: aws/az/gcloud + scoutsuite/cloudfox/pmapper; cloud:bench IaC-misconfig detection
- mobile: apkleaks/mobsfscan/drozer/objection/class-dump; mobile:bench static ruleset
- binary/RE: ghidra/objdump/checksec/strings/readelf/gdb; binary:bench decompiled-sink detector
Mobile/binary benches are DETECTOR REGRESSION CHECKS (rules+fixtures co-authored — a pass is a
floor, not a detection rate). No new headline numbers; verify-claims 24/24.
tsc clean; 337 tests; lint 0 errors.
This commit is contained in:
1 parent
a5667374bf
commit
537d5345dc
17 files changed
+1026
No files matched your search
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env node
|
||||
// cloud-misconfig-bench.mjs — the Cloud loadout's IaC-misconfig detection benchmark.
|
||||
//
|
||||
// Scores an IaC scanner (checkov) against a committed corpus of Terraform fixtures with KNOWN
|
||||
// misconfigurations + a locked-down control. Two axes:
|
||||
// • detection — fraction of misconfigured fixtures where the expected check fired
|
||||
// • discrimination — the control fixture must NOT trip any target misconfig check (no false-positive)
|
||||
//
|
||||
// HONEST SCOPE: this measures static-IaC misconfig DETECTION, not live cloud exploitation. The number
|
||||
// is directional (small corpus). `--self-test` validates the parser+scorer offline on committed data
|
||||
// (no checkov needed); `--live` runs real checkov for the actual number (needs `checkov` installed).
|
||||
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const CORPUS = path.join(HERE, '..', 'bench', 'cloud-misconfig', 'corpus.json');
|
||||
const SAMPLE = path.join(HERE, '..', 'bench', 'cloud-misconfig', 'sample-checkov-output.json');
|
||||
|
||||
export function loadCorpus() {
|
||||
return JSON.parse(readFileSync(CORPUS, 'utf8'));
|
||||
}
|
||||
|
||||
// Parse a `checkov -o json` result into { <fixture-id>: Set<check_id> }. Tolerant of checkov's shapes:
|
||||
// a single {results:{failed_checks}} object, or an array of them (multi-framework runs).
|
||||
export function parseCheckov(json) {
|
||||
const blocks = Array.isArray(json) ? json : [json];
|
||||
const byFixture = {};
|
||||
for (const b of blocks) {
|
||||
for (const fc of b?.results?.failed_checks ?? []) {
|
||||
const id = path.basename(String(fc.file_path || ''), '.tf');
|
||||
if (!id) continue;
|
||||
(byFixture[id] ??= new Set()).add(fc.check_id);
|
||||
}
|
||||
}
|
||||
return byFixture;
|
||||
}
|
||||
|
||||
// Score a parsed detection map against the corpus ground truth.
|
||||
export function scoreCorpus(corpus, byFixture) {
|
||||
const targets = new Set(corpus.target_checks || []);
|
||||
const bad = corpus.fixtures.filter((f) => !f.clean);
|
||||
const controls = corpus.fixtures.filter((f) => f.clean);
|
||||
const perFixture = corpus.fixtures.map((f) => {
|
||||
const detected = byFixture[f.id] || new Set();
|
||||
if (f.clean) {
|
||||
const falsePos = [...detected].filter((c) => targets.has(c));
|
||||
return { id: f.id, clean: true, ok: falsePos.length === 0, falsePos };
|
||||
}
|
||||
const hit = (f.expect || []).some((e) => detected.has(e));
|
||||
return { id: f.id, clean: false, ok: hit, expected: f.expect, detected: [...detected] };
|
||||
});
|
||||
const hits = perFixture.filter((r) => !r.clean && r.ok).length;
|
||||
const fpControls = perFixture.filter((r) => r.clean && !r.ok).length;
|
||||
return {
|
||||
detection: { hits, total: bad.length, rate: bad.length ? hits / bad.length : 0 },
|
||||
discrimination: { clean: controls.length, falsePositives: fpControls },
|
||||
perFixture,
|
||||
};
|
||||
}
|
||||
|
||||
function report(score) {
|
||||
const d = score.detection, g = score.discrimination;
|
||||
console.log(`\n detection: ${d.hits}/${d.total} misconfigs caught (${(100 * d.rate).toFixed(1)}%)`);
|
||||
console.log(` discrimination: ${g.clean - g.falsePositives}/${g.clean} controls clean (${g.falsePositives} false-positive)`);
|
||||
for (const r of score.perFixture) {
|
||||
const mark = r.ok ? '✅' : '❌';
|
||||
const detail = r.clean ? (r.ok ? 'control clean' : `FALSE-POSITIVE: ${r.falsePos.join(',')}`)
|
||||
: (r.ok ? `caught ${r.expected.join('/')}` : `MISSED ${r.expected.join('/')} (got: ${r.detected.join(',') || 'nothing'})`);
|
||||
console.log(` ${mark} ${r.id.padEnd(20)} ${detail}`);
|
||||
}
|
||||
}
|
||||
|
||||
function runLive() {
|
||||
const corpus = loadCorpus();
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'cloud-bench-'));
|
||||
try {
|
||||
for (const f of corpus.fixtures) writeFileSync(path.join(dir, `${f.id}.tf`), f.hcl);
|
||||
let out;
|
||||
try {
|
||||
out = execFileSync('checkov', ['-d', dir, '-o', 'json', '--compact', '--quiet'], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 });
|
||||
} catch (e) {
|
||||
// checkov exits non-zero when it finds failures — that's expected; use its stdout.
|
||||
out = e.stdout?.toString() || '';
|
||||
if (!out) { console.error(' ❌ could not run checkov — is it installed? (pipx install checkov)'); process.exitCode = 2; return; }
|
||||
}
|
||||
const score = scoreCorpus(corpus, parseCheckov(JSON.parse(out)));
|
||||
console.log('=== Cloud IaC-misconfig benchmark — LIVE (checkov) ===');
|
||||
report(score);
|
||||
console.log('\n (directional; static-IaC detection only — not live cloud exploitation)');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function runSelfTest() {
|
||||
const corpus = loadCorpus();
|
||||
const sample = JSON.parse(readFileSync(SAMPLE, 'utf8'));
|
||||
const score = scoreCorpus(corpus, parseCheckov(sample));
|
||||
let pass = 0, fail = 0;
|
||||
const ok = (l, c) => (c ? (pass++, console.log(' ✅ ' + l)) : (fail++, console.log(' ❌ ' + l)));
|
||||
console.log('=== Cloud IaC-misconfig benchmark — SELF-TEST (offline, committed sample) ===');
|
||||
ok('parser extracts a per-fixture detection for every corpus fixture with a failed check',
|
||||
Object.keys(parseCheckov(sample)).length >= corpus.fixtures.filter((f) => !f.clean).length);
|
||||
ok('every misconfigured fixture is detected (its expected check fired)', score.detection.hits === score.detection.total);
|
||||
ok('the locked-down control produces ZERO misconfig false-positives', score.discrimination.falsePositives === 0);
|
||||
ok('target_checks are non-empty and every fixture declares expect/clean',
|
||||
(corpus.target_checks || []).length > 0 && corpus.fixtures.every((f) => f.clean || (f.expect || []).length > 0));
|
||||
report(score);
|
||||
console.log('\n NOTE: the self-test validates the parser/scorer on committed sample data — run `--live` (real checkov) for an actual detection number.');
|
||||
console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}\n`);
|
||||
process.exitCode = fail === 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
if (isMain) {
|
||||
if (process.argv.includes('--live')) runLive();
|
||||
else runSelfTest();
|
||||
}
|
||||
Reference in new issue
Block a user