t3mp3st contributors
537d5345dc
Three new category loadouts, each = arsenal + an offline-scorable benchmark, opt-in and
honestly scoped (roster marks all 🚧; pacu/frida stay catalog_only, unit-tested non-mintable):
- cloud: aws/az/gcloud + scoutsuite/cloudfox/pmapper; cloud:bench IaC-misconfig detection
- mobile: apkleaks/mobsfscan/drozer/objection/class-dump; mobile:bench static ruleset
- binary/RE: ghidra/objdump/checksec/strings/readelf/gdb; binary:bench decompiled-sink detector
Mobile/binary benches are DETECTOR REGRESSION CHECKS (rules+fixtures co-authored — a pass is a
floor, not a detection rate). No new headline numbers; verify-claims 24/24.
tsc clean; 337 tests; lint 0 errors.
38 lines
1.7 KiB
TypeScript
38 lines
1.7 KiB
TypeScript
/**
|
|
* Locks the Cloud loadout's IaC-misconfig benchmark scorer into CI: the parser must map checkov
|
|
* findings to fixtures, every seeded misconfig must be detected, and the locked-down control must
|
|
* produce zero misconfig false-positives — validated against the committed sample (no checkov needed).
|
|
*/
|
|
import { describe, it, expect } from 'vitest';
|
|
import { readFileSync } from 'fs';
|
|
import { join } from 'path';
|
|
// The benchmark logic is authored in the bench script (ESM .mjs); import its pure functions directly.
|
|
import { loadCorpus, parseCheckov, scoreCorpus } from '../../scripts/cloud-misconfig-bench.mjs';
|
|
|
|
const sample = JSON.parse(readFileSync(join(process.cwd(), 'bench/cloud-misconfig/sample-checkov-output.json'), 'utf8'));
|
|
|
|
describe('cloud IaC-misconfig benchmark scorer', () => {
|
|
const corpus = loadCorpus();
|
|
const score = scoreCorpus(corpus, parseCheckov(sample));
|
|
|
|
it('the corpus has misconfigured fixtures + a clean control, all with ground truth', () => {
|
|
expect(corpus.fixtures.length).toBeGreaterThanOrEqual(5);
|
|
expect(corpus.fixtures.some((f: { clean?: boolean }) => f.clean)).toBe(true);
|
|
expect((corpus.target_checks || []).length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it('detects every seeded misconfiguration on the committed sample', () => {
|
|
expect(score.detection.hits).toBe(score.detection.total);
|
|
expect(score.detection.total).toBeGreaterThanOrEqual(5);
|
|
});
|
|
|
|
it('does not false-positive on the locked-down control', () => {
|
|
expect(score.discrimination.falsePositives).toBe(0);
|
|
});
|
|
|
|
it('parser is tolerant of checkov multi-framework (array) output', () => {
|
|
const asArray = parseCheckov([sample]);
|
|
expect(Object.keys(asArray).length).toBe(Object.keys(parseCheckov(sample)).length);
|
|
});
|
|
});
|