#!/usr/bin/env node
// CVE-Zero SWARM-vs-SOLO bake-off — a CONFOUND-REDUCED, PRE-REGISTERED PILOT.
//
// The earlier swarm numbers were confounded (swarm on codex, baseline on OpenRouter, different
// models). This runs every arm on the SAME backend (codex), SAME model, SAME deterministic
// scorer. To keep "coordination" from being one big bundle, it runs THREE arms so the refuter
// is isolated from the rest of the machinery (per the design review):
//   • solo         = cli-swarm --solo               : 1 generalist, 1 pass, NO refuter.
//   • solo+skeptic = cli-swarm --solo --skeptic true : 1 generalist, then the SAME refuter.  ← isolates the refuter
//   • swarm        = cli-swarm                        : 9 specialists × R rounds + board + refuter.
// Decomposition:  refuter effect = (solo+skeptic − solo) ; coordination effect = (swarm − solo+skeptic) ; total = (swarm − solo).
//
// HONEST CEILING — this is a PILOT, not a significance test:
//   • n=10, single run per arm, no repeated runs → the ~1-challenge noise floor is NOT measured.
//   • Wilson-95 on a 10-sample proportion is ≈ ±30 points; any small Δ is inside noise. Directional only.
//   • Compute is UNMATCHED: swarm ≈ 20-30 codex calls vs solo = 1. A swarm win is "a 20x-compute swarm
//     scores X," not "coordination beats a single agent." Per-arm elapsed/calls are recorded so the price is visible.
//   • The scorer (file+CWE+line, top-3 by severity) is precision-blind: it can't punish solo over-labeling
//     and can't credit the swarm's precision. Companion metrics (finding count, any-hit recall, rank-of-first-hit) are reported alongside.
//   • Both arms run --triage false, so the scored set is skeptic-survivors, NOT the triage-gated shipped pipeline.
//
//   node scripts/cve-zero-bakeoff.mjs --model gpt-5.5                       # all 10 samples, 3 arms
//   node scripts/cve-zero-bakeoff.mjs --model gpt-5.5 --rounds 2 GHSA-xxx   # subset
import { loadSamples, scoreSample, scoreFinding } from './cve-zero-hunt.mjs';
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const SWARM_OUT = path.join(ROOT, 'bench', 'wild-hunt', 'swarm');
const RESULTS = path.join(ROOT, 'bench', 'cve-zero', 'results');
const SEV = { critical: 4, high: 3, medium: 2, low: 1, info: 0 };
const INVALID = /INVALID_DEGRADED|NO_TRANSCRIPT|PARSE_ERR/;

const argv = process.argv.slice(2);
function optval(name, def) {
  const i = argv.indexOf('--' + name);
  if (i >= 0 && argv[i + 1] && !argv[i + 1].startsWith('--')) { const v = argv[i + 1]; argv.splice(i, 2); return v; }
  const eq = argv.find((a) => a.startsWith(`--${name}=`));
  if (eq) { argv.splice(argv.indexOf(eq), 1); return eq.split('=').slice(1).join('='); }
  return def;
}
const MODEL = optval('model', null);
let ROUNDS = parseInt(optval('rounds', '2'), 10);
if (!Number.isFinite(ROUNDS) || ROUNDS < 1) { console.error('warning: --rounds clamped to 1'); ROUNDS = 1; }
const ids = argv.filter((a) => !a.startsWith('--'));

// Wilson 95% CI for a binomial proportion — the honest uncertainty at n=10.
function wilson(k, n) {
  if (!n) return [0, 0];
  const z = 1.96, p = k / n, d = 1 + (z * z) / n;
  const c = p + (z * z) / (2 * n), m = z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n));
  return [Math.max(0, (c - m) / d), Math.min(1, (c + m) / d)];
}
const pct = (x) => `${(x * 100).toFixed(0)}%`;

// Spawn one cli-swarm arm; read its fresh transcript. Snapshot BEFORE each spawn isolates its own output.
function runArm(repo, arm) {
  return new Promise((resolve) => {
    const before = new Set(fs.existsSync(SWARM_OUT) ? fs.readdirSync(SWARM_OUT) : []);
    const args = ['scripts/cli-swarm.mjs', '--repo', repo, '--triage', 'false'];
    if (arm === 'solo') args.push('--solo', 'true');
    else if (arm === 'solo+skeptic') args.push('--solo', 'true', '--skeptic', 'true');
    else args.push('--rounds', String(ROUNDS)); // swarm
    if (MODEL) args.push('--model', MODEL);
    const child = spawn('node', args, { cwd: ROOT, stdio: ['ignore', 'inherit', 'inherit'] });
    child.on('exit', () => {
      const fresh = (fs.existsSync(SWARM_OUT) ? fs.readdirSync(SWARM_OUT) : [])
        .filter((f) => f.endsWith('.json') && !before.has(f)).sort();
      if (!fresh.length) return resolve({ confirmed: [], verdict: 'NO_TRANSCRIPT', elapsed: null, calls: null });
      try {
        const t = JSON.parse(fs.readFileSync(path.join(SWARM_OUT, fresh[fresh.length - 1]), 'utf8'));
        const confirmed = (t.rounds || []).flatMap((r) => r.newConfirmed || []);
        resolve({ confirmed, verdict: t.verdict || 'FINDINGS', model: t.model, elapsed: t.elapsed_min, calls: t.backend?.calls ?? null });
      } catch (e) { resolve({ confirmed: [], verdict: 'PARSE_ERR:' + e.message, elapsed: null, calls: null }); }
    });
  });
}

function scoreArm(res, gt) {
  const confirmed = res.confirmed || [];
  const ranked = [...confirmed].sort((a, b) => (SEV[b.severity] || 0) - (SEV[a.severity] || 0));
  const s = scoreSample(ranked, gt); // strict = top-1 hit, lenient = top-3 hit
  // severity-INDEPENDENT recall + rank-of-first-hit over ALL confirmed (the precision-blind-scorer companion)
  let anyHit = false, rankOfFirstHit = null;
  for (let i = 0; i < ranked.length; i++) { if (scoreFinding(ranked[i], gt).hit) { anyHit = true; rankOfFirstHit = i + 1; break; } }
  return {
    valid: !INVALID.test(res.verdict || ''), verdict: res.verdict,
    strict: !!s.strict, lenient: !!s.lenient, anyHit, rankOfFirstHit,
    n: confirmed.length, elapsed: res.elapsed, calls: res.calls,
  };
}

const samples = loadSamples(ids.length ? ids : null);
if (!samples.length) { console.error('no samples matched'); process.exit(2); }

const ARMS = ['solo', 'solo+skeptic', 'swarm'];
const agg = {
  schema: 't3mp3st.bench.cve-zero.bakeoff/v2', model: MODEL || 'codex-default', rounds: ROUNDS, backend: 'codex',
  framing: {
    kind: 'pre-registered PILOT — directional, NOT a significance test',
    confoundsRemoved: ['backend (both codex)', 'model (both same)', 'scorer (identical)'],
    confoundsRemaining: ['~20x compute unmatched (solo 1 call vs swarm ~20-30)', 'n=10 underpowered (Wilson-95 ≈ ±30pts)', 'single run/arm — noise floor assumed not measured', 'scorer precision-blind (companion metrics reported)', 'scored set is skeptic-survivors, not triage-gated shipped pipeline'],
    decisionRule: 'a delta counts as real ONLY if it is large and consistent across per-sample pairs; a small Δ is reported as noise-indistinguishable, not as uplift/regression',
  },
  attempted: 0, invalid: 0,
  tally: { solo: { strict: 0, lenient: 0, anyHit: 0 }, 'solo+skeptic': { strict: 0, lenient: 0, anyHit: 0 }, swarm: { strict: 0, lenient: 0, anyHit: 0 } },
  delta: { refuter_strict: 0, coordination_strict: 0, total_strict: 0 },
  discordant: { swarm_beats_solo: 0, solo_beats_swarm: 0 },
  per: [], invalidDetail: [],
};

console.log(`\nCVE-Zero BAKE-OFF (confound-reduced PILOT) — ${samples.length} sample(s) · model=${MODEL || 'codex-default'} · 3 arms on codex · swarm rounds=${ROUNDS}`);
console.log(`  arms: solo | solo+skeptic (isolates refuter) | swarm    ·    DIRECTIONAL pilot, not a significance test\n`);

for (const s of samples) {
  const repo = fs.existsSync(path.join(s.dir, 'source')) ? path.join(s.dir, 'source') : s.dir;
  console.log(`\n════════ ${s.id}  (${s.gt.repo}) — gt ${s.gt.cwes.join(',')}@${s.gt.vuln_files.map((f) => f.split('/').pop()).join(',')} ════════`);
  const scored = {};
  for (const arm of ARMS) {
    console.log(`  ▸ ${arm} arm…`);
    scored[arm] = scoreArm(await runArm(repo, arm), s.gt);
  }
  const anyInvalid = ARMS.some((a) => !scored[a].valid);
  const rec = { id: s.id, ...Object.fromEntries(ARMS.map((a) => [a, scored[a]])) };
  agg.per.push(rec);
  if (anyInvalid) {
    agg.invalid++;
    agg.invalidDetail.push({ id: s.id, arms: Object.fromEntries(ARMS.filter((a) => !scored[a].valid).map((a) => [a, scored[a].verdict])) });
    console.log(`  ⚠ EXCLUDED from tally — invalid arm(s): ${ARMS.filter((a) => !scored[a].valid).map((a) => `${a}=${scored[a].verdict}`).join(', ')}`);
    continue;
  }
  agg.attempted++;
  for (const a of ARMS) { for (const k of ['strict', 'lenient', 'anyHit']) agg.tally[a][k] += scored[a][k] ? 1 : 0; }
  const d = (scored.swarm.strict ? 1 : 0) - (scored.solo.strict ? 1 : 0);
  if (d > 0) agg.discordant.swarm_beats_solo++; else if (d < 0) agg.discordant.solo_beats_swarm++;
  console.log(`  → strict:  solo=${+scored.solo.strict} solo+skeptic=${+scored['solo+skeptic'].strict} swarm=${+scored.swarm.strict}` +
    `  | anyHit: ${+scored.solo.anyHit}/${+scored['solo+skeptic'].anyHit}/${+scored.swarm.anyHit}` +
    `  | found: ${scored.solo.n}/${scored['solo+skeptic'].n}/${scored.swarm.n}` +
    `  | calls: ${scored.solo.calls ?? '?'}/${scored['solo+skeptic'].calls ?? '?'}/${scored.swarm.calls ?? '?'}`);
}

const N = agg.attempted;
agg.delta.refuter_strict = agg.tally['solo+skeptic'].strict - agg.tally.solo.strict;
agg.delta.coordination_strict = agg.tally.swarm.strict - agg.tally['solo+skeptic'].strict;
agg.delta.total_strict = agg.tally.swarm.strict - agg.tally.solo.strict;
agg.ci = Object.fromEntries(ARMS.map((a) => [a, wilson(agg.tally[a].strict, N).map((x) => +x.toFixed(3))]));

fs.mkdirSync(RESULTS, { recursive: true });
const out = path.join(RESULTS, `bakeoff-${MODEL ? MODEL.replace(/[^a-z0-9]+/gi, '-') + '-' : ''}${Date.now()}.json`);
fs.writeFileSync(out, JSON.stringify(agg, null, 2));

console.log(`\n${'═'.repeat(70)}`);
console.log(`  BAKE-OFF PILOT (${agg.model}, n=${N} scored${agg.invalid ? `, ${agg.invalid} excluded-invalid` : ''}, both/all arms on codex, swarm rounds=${ROUNDS}):`);
for (const a of ARMS) {
  const t = agg.tally[a], ci = wilson(t.strict, N);
  console.log(`    ${a.padEnd(13)} strict ${t.strict}/${N} (${pct(t.strict / (N || 1))}, 95% CI ${pct(ci[0])}–${pct(ci[1])})   lenient ${t.lenient}/${N}   anyHit ${t.anyHit}/${N}`);
}
console.log(`    ── Δstrict:  refuter (solo+skeptic−solo) = ${agg.delta.refuter_strict >= 0 ? '+' : ''}${agg.delta.refuter_strict}` +
  `  ·  coordination (swarm−solo+skeptic) = ${agg.delta.coordination_strict >= 0 ? '+' : ''}${agg.delta.coordination_strict}` +
  `  ·  total (swarm−solo) = ${agg.delta.total_strict >= 0 ? '+' : ''}${agg.delta.total_strict}`);
console.log(`    ── paired swarm vs solo: ${agg.discordant.swarm_beats_solo} swarm-win / ${agg.discordant.solo_beats_swarm} solo-win (discordant pairs)`);
console.log(`  DIRECTIONAL PILOT — n=${N} is underpowered (CI ≈ ±30pts) and compute is unmatched; a small Δ is inside noise, not uplift.`);
console.log(`${'═'.repeat(70)}`);
console.log(`report: ${path.relative(ROOT, out)}\n`);