167 lines
11 KiB
JavaScript
167 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
||
// CVE-Zero SWARM-vs-SOLO bake-off — a CONFOUND-REDUCED, PRE-REGISTERED PILOT.
|
||
//
|
||
// The earlier swarm numbers were confounded (swarm on codex, baseline on OpenRouter, different
|
||
// models). This runs every arm on the SAME backend (codex), SAME model, SAME deterministic
|
||
// scorer. To keep "coordination" from being one big bundle, it runs THREE arms so the refuter
|
||
// is isolated from the rest of the machinery (per the design review):
|
||
// • solo = cli-swarm --solo : 1 generalist, 1 pass, NO refuter.
|
||
// • solo+skeptic = cli-swarm --solo --skeptic true : 1 generalist, then the SAME refuter. ← isolates the refuter
|
||
// • swarm = cli-swarm : 9 specialists × R rounds + board + refuter.
|
||
// Decomposition: refuter effect = (solo+skeptic − solo) ; coordination effect = (swarm − solo+skeptic) ; total = (swarm − solo).
|
||
//
|
||
// HONEST CEILING — this is a PILOT, not a significance test:
|
||
// • n=10, single run per arm, no repeated runs → the ~1-challenge noise floor is NOT measured.
|
||
// • Wilson-95 on a 10-sample proportion is ≈ ±30 points; any small Δ is inside noise. Directional only.
|
||
// • Compute is UNMATCHED: swarm ≈ 20-30 codex calls vs solo = 1. A swarm win is "a 20x-compute swarm
|
||
// scores X," not "coordination beats a single agent." Per-arm elapsed/calls are recorded so the price is visible.
|
||
// • The scorer (file+CWE+line, top-3 by severity) is precision-blind: it can't punish solo over-labeling
|
||
// and can't credit the swarm's precision. Companion metrics (finding count, any-hit recall, rank-of-first-hit) are reported alongside.
|
||
// • Both arms run --triage false, so the scored set is skeptic-survivors, NOT the triage-gated shipped pipeline.
|
||
//
|
||
// node scripts/cve-zero-bakeoff.mjs --model gpt-5.5 # all 10 samples, 3 arms
|
||
// node scripts/cve-zero-bakeoff.mjs --model gpt-5.5 --rounds 2 GHSA-xxx # subset
|
||
import { loadSamples, scoreSample, scoreFinding } from './cve-zero-hunt.mjs';
|
||
import { spawn } from 'node:child_process';
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||
const SWARM_OUT = path.join(ROOT, 'bench', 'wild-hunt', 'swarm');
|
||
const RESULTS = path.join(ROOT, 'bench', 'cve-zero', 'results');
|
||
const SEV = { critical: 4, high: 3, medium: 2, low: 1, info: 0 };
|
||
const INVALID = /INVALID_DEGRADED|NO_TRANSCRIPT|PARSE_ERR/;
|
||
|
||
const argv = process.argv.slice(2);
|
||
function optval(name, def) {
|
||
const i = argv.indexOf('--' + name);
|
||
if (i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--')) { const v = argv[i + 1]; argv.splice(i, 2); return v; }
|
||
const eq = argv.find((a) => a.startsWith(`--${name}=`));
|
||
if (eq) { argv.splice(argv.indexOf(eq), 1); return eq.split('=').slice(1).join('='); }
|
||
return def;
|
||
}
|
||
const MODEL = optval('model', null);
|
||
let ROUNDS = parseInt(optval('rounds', '2'), 10);
|
||
if (!Number.isFinite(ROUNDS) || ROUNDS < 1) { console.error('warning: --rounds clamped to 1'); ROUNDS = 1; }
|
||
const ids = argv.filter((a) => !a.startsWith('--'));
|
||
|
||
// Wilson 95% CI for a binomial proportion — the honest uncertainty at n=10.
|
||
function wilson(k, n) {
|
||
if (!n) return [0, 0];
|
||
const z = 1.96, p = k / n, d = 1 + (z * z) / n;
|
||
const c = p + (z * z) / (2 * n), m = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
|
||
return [Math.max(0, (c - m) / d), Math.min(1, (c + m) / d)];
|
||
}
|
||
const pct = (x) => `${(x * 100).toFixed(0)}%`;
|
||
|
||
// Spawn one cli-swarm arm; read its fresh transcript. Snapshot BEFORE each spawn isolates its own output.
|
||
function runArm(repo, arm) {
|
||
return new Promise((resolve) => {
|
||
const before = new Set(fs.existsSync(SWARM_OUT) ? fs.readdirSync(SWARM_OUT) : []);
|
||
const args = ['scripts/cli-swarm.mjs', '--repo', repo, '--triage', 'false'];
|
||
if (arm === 'solo') args.push('--solo', 'true');
|
||
else if (arm === 'solo+skeptic') args.push('--solo', 'true', '--skeptic', 'true');
|
||
else args.push('--rounds', String(ROUNDS)); // swarm
|
||
if (MODEL) args.push('--model', MODEL);
|
||
const child = spawn('node', args, { cwd: ROOT, stdio: ['ignore', 'inherit', 'inherit'] });
|
||
child.on('exit', () => {
|
||
const fresh = (fs.existsSync(SWARM_OUT) ? fs.readdirSync(SWARM_OUT) : [])
|
||
.filter((f) => f.endsWith('.json') && !before.has(f)).sort();
|
||
if (!fresh.length) return resolve({ confirmed: [], verdict: 'NO_TRANSCRIPT', elapsed: null, calls: null });
|
||
try {
|
||
const t = JSON.parse(fs.readFileSync(path.join(SWARM_OUT, fresh[fresh.length - 1]), 'utf8'));
|
||
const confirmed = (t.rounds || []).flatMap((r) => r.newConfirmed || []);
|
||
resolve({ confirmed, verdict: t.verdict || 'FINDINGS', model: t.model, elapsed: t.elapsed_min, calls: t.backend?.calls ?? null });
|
||
} catch (e) { resolve({ confirmed: [], verdict: 'PARSE_ERR:' + e.message, elapsed: null, calls: null }); }
|
||
});
|
||
});
|
||
}
|
||
|
||
function scoreArm(res, gt) {
|
||
const confirmed = res.confirmed || [];
|
||
const ranked = [...confirmed].sort((a, b) => (SEV[b.severity] || 0) - (SEV[a.severity] || 0));
|
||
const s = scoreSample(ranked, gt); // strict = top-1 hit, lenient = top-3 hit
|
||
// severity-INDEPENDENT recall + rank-of-first-hit over ALL confirmed (the precision-blind-scorer companion)
|
||
let anyHit = false, rankOfFirstHit = null;
|
||
for (let i = 0; i < ranked.length; i++) { if (scoreFinding(ranked[i], gt).hit) { anyHit = true; rankOfFirstHit = i + 1; break; } }
|
||
return {
|
||
valid: !INVALID.test(res.verdict || ''), verdict: res.verdict,
|
||
strict: !!s.strict, lenient: !!s.lenient, anyHit, rankOfFirstHit,
|
||
n: confirmed.length, elapsed: res.elapsed, calls: res.calls,
|
||
};
|
||
}
|
||
|
||
const samples = loadSamples(ids.length ? ids : null);
|
||
if (!samples.length) { console.error('no samples matched'); process.exit(2); }
|
||
|
||
const ARMS = ['solo', 'solo+skeptic', 'swarm'];
|
||
const agg = {
|
||
schema: 't3mp3st.bench.cve-zero.bakeoff/v2', model: MODEL || 'codex-default', rounds: ROUNDS, backend: 'codex',
|
||
framing: {
|
||
kind: 'pre-registered PILOT — directional, NOT a significance test',
|
||
confoundsRemoved: ['backend (both codex)', 'model (both same)', 'scorer (identical)'],
|
||
confoundsRemaining: ['~20x compute unmatched (solo 1 call vs swarm ~20-30)', 'n=10 underpowered (Wilson-95 ≈ ±30pts)', 'single run/arm — noise floor assumed not measured', 'scorer precision-blind (companion metrics reported)', 'scored set is skeptic-survivors, not triage-gated shipped pipeline'],
|
||
decisionRule: 'a delta counts as real ONLY if it is large and consistent across per-sample pairs; a small Δ is reported as noise-indistinguishable, not as uplift/regression',
|
||
},
|
||
attempted: 0, invalid: 0,
|
||
tally: { solo: { strict: 0, lenient: 0, anyHit: 0 }, 'solo+skeptic': { strict: 0, lenient: 0, anyHit: 0 }, swarm: { strict: 0, lenient: 0, anyHit: 0 } },
|
||
delta: { refuter_strict: 0, coordination_strict: 0, total_strict: 0 },
|
||
discordant: { swarm_beats_solo: 0, solo_beats_swarm: 0 },
|
||
per: [], invalidDetail: [],
|
||
};
|
||
|
||
console.log(`\nCVE-Zero BAKE-OFF (confound-reduced PILOT) — ${samples.length} sample(s) · model=${MODEL || 'codex-default'} · 3 arms on codex · swarm rounds=${ROUNDS}`);
|
||
console.log(` arms: solo | solo+skeptic (isolates refuter) | swarm · DIRECTIONAL pilot, not a significance test\n`);
|
||
|
||
for (const s of samples) {
|
||
const repo = fs.existsSync(path.join(s.dir, 'source')) ? path.join(s.dir, 'source') : s.dir;
|
||
console.log(`\n════════ ${s.id} (${s.gt.repo}) — gt ${s.gt.cwes.join(',')}@${s.gt.vuln_files.map((f) => f.split('/').pop()).join(',')} ════════`);
|
||
const scored = {};
|
||
for (const arm of ARMS) {
|
||
console.log(` ▸ ${arm} arm…`);
|
||
scored[arm] = scoreArm(await runArm(repo, arm), s.gt);
|
||
}
|
||
const anyInvalid = ARMS.some((a) => !scored[a].valid);
|
||
const rec = { id: s.id, ...Object.fromEntries(ARMS.map((a) => [a, scored[a]])) };
|
||
agg.per.push(rec);
|
||
if (anyInvalid) {
|
||
agg.invalid++;
|
||
agg.invalidDetail.push({ id: s.id, arms: Object.fromEntries(ARMS.filter((a) => !scored[a].valid).map((a) => [a, scored[a].verdict])) });
|
||
console.log(` ⚠ EXCLUDED from tally — invalid arm(s): ${ARMS.filter((a) => !scored[a].valid).map((a) => `${a}=${scored[a].verdict}`).join(', ')}`);
|
||
continue;
|
||
}
|
||
agg.attempted++;
|
||
for (const a of ARMS) { for (const k of ['strict', 'lenient', 'anyHit']) agg.tally[a][k] += scored[a][k] ? 1 : 0; }
|
||
const d = (scored.swarm.strict ? 1 : 0) - (scored.solo.strict ? 1 : 0);
|
||
if (d > 0) agg.discordant.swarm_beats_solo++; else if (d < 0) agg.discordant.solo_beats_swarm++;
|
||
console.log(` → strict: solo=${+scored.solo.strict} solo+skeptic=${+scored['solo+skeptic'].strict} swarm=${+scored.swarm.strict}` +
|
||
` | anyHit: ${+scored.solo.anyHit}/${+scored['solo+skeptic'].anyHit}/${+scored.swarm.anyHit}` +
|
||
` | found: ${scored.solo.n}/${scored['solo+skeptic'].n}/${scored.swarm.n}` +
|
||
` | calls: ${scored.solo.calls ?? '?'}/${scored['solo+skeptic'].calls ?? '?'}/${scored.swarm.calls ?? '?'}`);
|
||
}
|
||
|
||
const N = agg.attempted;
|
||
agg.delta.refuter_strict = agg.tally['solo+skeptic'].strict - agg.tally.solo.strict;
|
||
agg.delta.coordination_strict = agg.tally.swarm.strict - agg.tally['solo+skeptic'].strict;
|
||
agg.delta.total_strict = agg.tally.swarm.strict - agg.tally.solo.strict;
|
||
agg.ci = Object.fromEntries(ARMS.map((a) => [a, wilson(agg.tally[a].strict, N).map((x) => +x.toFixed(3))]));
|
||
|
||
fs.mkdirSync(RESULTS, { recursive: true });
|
||
const out = path.join(RESULTS, `bakeoff-${MODEL ? MODEL.replace(/[^a-z0-9]+/gi, '-') + '-' : ''}${Date.now()}.json`);
|
||
fs.writeFileSync(out, JSON.stringify(agg, null, 2));
|
||
|
||
console.log(`\n${'═'.repeat(70)}`);
|
||
console.log(` BAKE-OFF PILOT (${agg.model}, n=${N} scored${agg.invalid ? `, ${agg.invalid} excluded-invalid` : ''}, both/all arms on codex, swarm rounds=${ROUNDS}):`);
|
||
for (const a of ARMS) {
|
||
const t = agg.tally[a], ci = wilson(t.strict, N);
|
||
console.log(` ${a.padEnd(13)} strict ${t.strict}/${N} (${pct(t.strict / (N || 1))}, 95% CI ${pct(ci[0])}–${pct(ci[1])}) lenient ${t.lenient}/${N} anyHit ${t.anyHit}/${N}`);
|
||
}
|
||
console.log(` ── Δstrict: refuter (solo+skeptic−solo) = ${agg.delta.refuter_strict >= 0 ? '+' : ''}${agg.delta.refuter_strict}` +
|
||
` · coordination (swarm−solo+skeptic) = ${agg.delta.coordination_strict >= 0 ? '+' : ''}${agg.delta.coordination_strict}` +
|
||
` · total (swarm−solo) = ${agg.delta.total_strict >= 0 ? '+' : ''}${agg.delta.total_strict}`);
|
||
console.log(` ── paired swarm vs solo: ${agg.discordant.swarm_beats_solo} swarm-win / ${agg.discordant.solo_beats_swarm} solo-win (discordant pairs)`);
|
||
console.log(` DIRECTIONAL PILOT — n=${N} is underpowered (CI ≈ ±30pts) and compute is unmatched; a small Δ is inside noise, not uplift.`);
|
||
console.log(`${'═'.repeat(70)}`);
|
||
console.log(`report: ${path.relative(ROOT, out)}\n`);
|