108 lines
6.5 KiB
JavaScript
108 lines
6.5 KiB
JavaScript
#!/usr/bin/env node
|
|
// CVE-Zero SWARM harness — runs cli-swarm (a SQUAD of specialist CLASSES) on each sample's source,
|
|
// scores confirmed survivors with the SAME deterministic scoreSample() as the single-agent baseline,
|
|
// and appends every run to the config-history ledger so squads can be ranked. Honesty rail:
|
|
// --split train|test|holdout filters the sample set AND stamps the ledger row (the recommender may
|
|
// only ever learn from train; a generalization claim may use only a populated holdout).
|
|
//
|
|
// node scripts/cve-zero-swarm.mjs # all samples, full squad, codex default
|
|
// node scripts/cve-zero-swarm.mjs --split test --auto-squad # test split, Admiral-composed squad per sample
|
|
// node scripts/cve-zero-swarm.mjs --squad cwe-triage --effort xhigh --model gpt-5.5
|
|
import { loadSamples, scoreSample, scoreFinding, fam } from './cve-zero-hunt.mjs';
|
|
import { composeSquad } from './classes.mjs';
|
|
import { splitIds } from './cve-zero-split.mjs';
|
|
import { appendConfigRecord } from './config-history.mjs';
|
|
import { spawn } from 'node:child_process';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
const SWARM_OUT = path.join(ROOT, 'bench', 'wild-hunt', 'swarm');
|
|
const RESULTS = path.join(ROOT, 'bench', 'cve-zero', 'results');
|
|
const SEV = { critical: 4, high: 3, medium: 2, low: 1, info: 0 };
|
|
const INVALID = /INVALID_DEGRADED|NO_TRANSCRIPT|PARSE_ERR/;
|
|
|
|
const argv = process.argv.slice(2);
|
|
function opt(name) {
|
|
const i = argv.indexOf('--' + name);
|
|
if (i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--')) { const v = argv[i + 1]; argv.splice(i, 2); return v; }
|
|
const eq = argv.find((a) => a.startsWith(`--${name}=`));
|
|
if (eq) { argv.splice(argv.indexOf(eq), 1); return eq.split('=').slice(1).join('='); }
|
|
return undefined;
|
|
}
|
|
function flag(name) {
|
|
const i = argv.indexOf('--' + name);
|
|
if (i < 0) return undefined;
|
|
const hasVal = argv[i + 1] && !argv[i + 1].startsWith('--');
|
|
const v = hasVal ? argv[i + 1] : 'true';
|
|
argv.splice(i, hasVal ? 2 : 1);
|
|
return v;
|
|
}
|
|
const MODEL = opt('model');
|
|
const SPLIT = opt('split');
|
|
const SQUAD = opt('squad'); // preset name or inline json (manual)
|
|
const EFFORT = opt('effort');
|
|
const ROUNDS = opt('rounds') || '2';
|
|
const AUTO = flag('auto-squad');
|
|
const ids = argv.filter((a) => !a.startsWith('--'));
|
|
|
|
function runSwarm(repo, squadArg) {
|
|
return new Promise((resolve) => {
|
|
const before = new Set(fs.existsSync(SWARM_OUT) ? fs.readdirSync(SWARM_OUT) : []);
|
|
const args = ['scripts/cli-swarm.mjs', '--repo', repo, '--rounds', String(ROUNDS), '--triage', 'false'];
|
|
if (squadArg) args.push('--squad', squadArg); // per-sample auto-composed squad
|
|
else if (SQUAD) args.push('--squad', SQUAD); // fixed manual squad
|
|
if (EFFORT) args.push('--effort', EFFORT);
|
|
if (MODEL) args.push('--model', MODEL);
|
|
const child = spawn('node', args, { cwd: ROOT, stdio: ['ignore', 'inherit', 'inherit'] });
|
|
child.on('exit', () => {
|
|
const fresh = (fs.existsSync(SWARM_OUT) ? fs.readdirSync(SWARM_OUT) : []).filter((f) => f.endsWith('.json') && !before.has(f)).sort();
|
|
if (!fresh.length) return resolve({ confirmed: [], verdict: 'NO_TRANSCRIPT', calls: null });
|
|
try {
|
|
const t = JSON.parse(fs.readFileSync(path.join(SWARM_OUT, fresh[fresh.length - 1]), 'utf8'));
|
|
resolve({ confirmed: (t.rounds || []).flatMap((r) => r.newConfirmed || []), verdict: t.verdict || 'FINDINGS', calls: t.backend?.calls ?? null });
|
|
} catch (e) { resolve({ confirmed: [], verdict: 'PARSE_ERR:' + e.message, calls: null }); }
|
|
});
|
|
});
|
|
}
|
|
|
|
const keepIds = splitIds(SPLIT);
|
|
const samples = loadSamples(ids.length ? ids : keepIds);
|
|
if (!samples.length) { console.error('no samples matched'); process.exit(2); }
|
|
|
|
const agg = {
|
|
schema: 't3mp3st.bench.cve-zero.swarm/v2', model: MODEL ? `codex-swarm:${MODEL}` : 'codex-swarm',
|
|
split: SPLIT || 'all', squad: AUTO ? 'auto-compose' : (SQUAD || 'full'), effort: EFFORT || null,
|
|
attempted: 0, invalid: 0, strict: 0, lenient: 0, anyHit: 0, per: [],
|
|
};
|
|
console.log(`\nCVE-Zero SWARM — ${samples.length} sample(s) · model=${MODEL || 'codex-default'} · split=${SPLIT || 'all'} · squad=${agg.squad}${EFFORT ? ` · effort=${EFFORT}` : ''}\n`);
|
|
|
|
for (const s of samples) {
|
|
const repo = fs.existsSync(path.join(s.dir, 'source')) ? path.join(s.dir, 'source') : s.dir;
|
|
const composed = AUTO ? composeSquad({ cwes: s.gt.cwes }) : null;
|
|
console.log(`\n──── ${s.id} (${s.gt.repo})${composed ? ` · squad: ${composed.reason}` : ''} ────`);
|
|
const { confirmed, verdict, calls } = await runSwarm(repo, composed ? JSON.stringify(composed.squad) : undefined);
|
|
const ranked = [...confirmed].sort((a, b) => (SEV[b.severity] || 0) - (SEV[a.severity] || 0));
|
|
const score = scoreSample(ranked, s.gt);
|
|
const anyHit = ranked.some((f) => scoreFinding(f, s.gt).hit);
|
|
const valid = !INVALID.test(verdict);
|
|
// append to the scored config-history ledger — stamped with the split + precision caveat.
|
|
appendConfigRecord({
|
|
squad: composed ? composed.squad : (SQUAD ? [{ class: SQUAD }] : undefined), model: agg.model, rounds: +ROUNDS,
|
|
split: SPLIT || 'all', sampleId: s.id, targetFamily: fam((s.gt.cwes || [])[0]),
|
|
strict: score.strict, lenient: score.lenient, anyHit, calls, verdict, valid,
|
|
precisionCaveat: 'skeptic-survivors, not triage-gated',
|
|
});
|
|
if (!valid) { agg.invalid++; agg.per.push({ id: s.id, valid: false, verdict }); console.log(` ⚠ INVALID (${verdict}) — excluded from tally`); continue; }
|
|
agg.attempted++; agg.strict += score.strict ? 1 : 0; agg.lenient += score.lenient ? 1 : 0; agg.anyHit += anyHit ? 1 : 0;
|
|
agg.per.push({ id: s.id, strict: score.strict, lenient: score.lenient, anyHit, confirmed: confirmed.length, calls, verdict });
|
|
console.log(` → strict=${score.strict} lenient=${score.lenient} anyHit=${anyHit} (${confirmed.length} survivors, ${calls} calls, ${verdict})`);
|
|
}
|
|
|
|
fs.mkdirSync(RESULTS, { recursive: true });
|
|
const out = path.join(RESULTS, `swarm-aggregate-${MODEL ? MODEL.replace(/[^a-z0-9]+/gi, '-') + '-' : ''}${Date.now()}.json`);
|
|
fs.writeFileSync(out, JSON.stringify(agg, null, 2));
|
|
console.log(`\n══════ SWARM (${agg.model}, split=${agg.split}, squad=${agg.squad}): strict ${agg.strict}/${agg.attempted} · lenient ${agg.lenient}/${agg.attempted} · anyHit ${agg.anyHit}/${agg.attempted}${agg.invalid ? ` · ${agg.invalid} invalid-excluded` : ''} ══════`);
|
|
console.log(`logged ${samples.length} run(s) to config-history.jsonl · report: ${path.relative(ROOT, out)}`);
|