Merge branch 'pr-39' into pr-delivery-74-94-18-39-41

scripts/verify-claims.mjs

@@ -229,6 +229,48 @@ console.log('\nCLAIM 7 — CVE-Zero-v2 (held-out): hunt generalizes to fresh uns
   }
 }
 
+// ── INTEGRITY GATE: every scored solve is oracle-grounded (no phantom pass) ───
+// Every check above credits a solve straight from the committed `verdict` (detected /
+// score===1). This gate closes the obvious tamper hole in that trust: a scored solve must
+// still AGREE with its own committed flag oracle. A solve is oracle-grounded iff EITHER the
+// agent's self-reported flag matches the committed `expected` under the same case-insensitive
+// compare the harness uses (ciMatch), OR the harness auto-detected `expected` verbatim in tool
+// output (auto_detected — the agent didn't self-report, so `reported` may be "UNKNOWN").
+// Anything scored as a solve that is NEITHER is a phantom pass: a hand-edited `detected:true`
+// whose committed oracle does not back it. This wires ciMatch (until now defined but unused)
+// into an executable gate, and rejects a fabricated placeholder credited via the self-report path.
+console.log('\nINTEGRITY GATE — every committed solve is oracle-grounded (no phantom pass, no fabricated flag)');
+const oracleGrounded = (v) =>
+  ciMatch(v.reported, v.expected) || v.auto_detected === true || /auto-?detected/i.test(String(v.reason || ''));
+const scanSolves = (files) => {
+  let solves = 0, phantom = 0, fabricated = 0, grounded = 0;
+  for (const f of files) {
+    const v = (((JSON.parse(fs.readFileSync(f, 'utf8')).results) || [{}])[0] || {}).verdict || {};
+    if (!(v.detected === true || v.score === 1)) continue;
+    solves++;
+    if (oracleGrounded(v)) grounded++; else phantom++;
+    // A fabricated placeholder may only ever be "credited" via the self-report path.
+    if (ciMatch(v.reported, v.expected) && looksFabricated(v.reported)) fabricated++;
+  }
+  return { solves, phantom, fabricated, grounded };
+};
+const xbenFiles = [...BESTBALL_DIRS, ...WHITEBOX_DIRS].flatMap((sub) =>
+  glob(R('bench/xbow/results/' + sub), /^xben_\d+_24.*\.json$/));
+const cybenchFiles = [...glob(R('bench/cybench/results'), /^clean-.*\.json$/),
+                      ...glob(R('bench/cybench/results'), /^service-cybsvc_.*\.json$/)];
+const ig = scanSolves([...xbenFiles, ...cybenchFiles]);
+check('every scored solve is oracle-grounded (self-report ≡ committed oracle, or auto-detected)',
+  ig.phantom === 0, `${ig.phantom} phantom / ${ig.solves} scored solves`);
+check('no fabricated placeholder flag is credited as a self-reported solve',
+  ig.fabricated === 0, `${ig.fabricated} fabricated / ${ig.solves} scored solves`);
+// Informational — how much of the Cybench headline is backed by an oracle-bearing per-run
+// artifact vs. counted only in the hand-editable aggregate (transcripts are stripped on export).
+{
+  const cRun = scanSolves(cybenchFiles);
+  const aggSolved = fs.existsSync(cf) ? JSON.parse(fs.readFileSync(cf, 'utf8')).solved : null;
+  console.log(`  ℹ️  Cybench: aggregate reports ${aggSolved} solved · ${cRun.grounded} oracle-grounded per-run artifact(s) committed · ${(aggSolved ?? 0) - cRun.grounded} aggregate-only`);
+}
+
 // ── verdict ─────────────────────────────────────────────────────────────────
 console.log(`\n════════ ${fail === 0 ? '✅ ALL CLAIMS VERIFIED' : `❌ ${fail} CHECK(S) FAILED`} — ${pass} passed, ${fail} failed ════════\n`);
 process.exit(fail === 0 ? 0 : 1);

src/__tests__/oracle-consistency.test.ts

@@ -0,0 +1,140 @@
+import { describe, it, expect } from 'vitest';
+import fs from 'node:fs';
+import path from 'node:path';
+
+// ─────────────────────────────────────────────────────────────────────────────
+// NO PHANTOM PASS — the "scored = oracle-backed" guard for the benchmark receipts.
+//
+// `verify-claims` credits a solve straight from the committed `verdict` (detected /
+// score===1). The launch risk is the mirror of a phantom tool: a phantom PASS — a
+// committed `detected:true` that its own committed flag oracle does not actually back
+// (a hand-edit, a merge slip, a fabricated placeholder flag). This test makes that fail
+// the build by construction — the same discipline as no-phantom-tools, applied to the
+// bench artifacts the headline numbers are re-derived from.
+//
+// A scored solve is ORACLE-GROUNDED iff EITHER the agent's self-reported flag matches the
+// committed `expected` under the harness's case-insensitive compare (ciMatch), OR the
+// harness auto-detected `expected` verbatim in tool output (auto_detected — the agent did
+// not self-report, so `reported` may be "UNKNOWN"). Anything else scored as a solve is a
+// phantom pass.
+// ─────────────────────────────────────────────────────────────────────────────
+
+// Same helpers verify-claims.mjs uses, re-derived here so the gate is independent of it.
+const ciMatch = (a: unknown, b: unknown) =>
+  !!a && !!b && String(a).trim().toLowerCase() === String(b).trim().toLowerCase();
+
+function looksFabricated(flag: unknown): boolean {
+  if (!flag) return false;
+  const inner = (String(flag).match(/\{([\s\S]*)\}/) || ['', ''])[1]
+    .toLowerCase()
+    .replace(/4/g, 'a').replace(/3/g, 'e').replace(/0/g, 'o')
+    .replace(/1/g, 'i').replace(/5/g, 's').replace(/7/g, 't');
+  return /\b(fake|placeholder|dummy|redacted|todo|tbd)\b|fake_?flag|for_?test|_?test(ing)?_?(flag|local|stub|only)|local_?test|real_[a-z]+_proof|proof_?xyz|sample_?flag|your_?flag_?here|example_?flag|replace_?me|flag_?here/.test(inner);
+}
+
+interface Verdict {
+  detected?: boolean;
+  score?: number;
+  reported?: string;
+  expected?: string;
+  auto_detected?: boolean;
+  reason?: string;
+}
+
+const isScoredSolve = (v: Verdict) => v.detected === true || v.score === 1;
+const isOracleGrounded = (v: Verdict) =>
+  ciMatch(v.reported, v.expected) ||
+  v.auto_detected === true ||
+  /auto-?detected/i.test(String(v.reason || ''));
+// A phantom pass: credited as a solve, but not backed by its own committed oracle.
+const isPhantomPass = (v: Verdict) => isScoredSolve(v) && !isOracleGrounded(v);
+// A fabricated credit: a placeholder flag credited via the self-report (oracle-match) path.
+const isFabricatedCredit = (v: Verdict) =>
+  isScoredSolve(v) && ciMatch(v.reported, v.expected) && looksFabricated(v.reported);
+
+// ── Load the committed XBEN artifacts the headline sweeps are pinned to ──────
+const REPO = process.cwd();
+const XBEN = path.join(REPO, 'bench', 'xbow', 'results');
+const PINNED_DIRS = [
+  'blackbox-golden', 'blackbox-golden-v2', 'blackbox-golden-final',
+  'whitebox-golden', 'whitebox-golden-v2', 'venice-whitebox',
+];
+
+function firstVerdict(file: string): Verdict {
+  const d = JSON.parse(fs.readFileSync(file, 'utf8'));
+  return ((d.results || [{}])[0] || {}).verdict || {};
+}
+
+function committedSolves(): Verdict[] {
+  const out: Verdict[] = [];
+  for (const dir of PINNED_DIRS) {
+    const abs = path.join(XBEN, dir);
+    if (!fs.existsSync(abs)) continue;
+    for (const f of fs.readdirSync(abs).filter((n) => /^xben_\d+_24.*\.json$/.test(n))) {
+      const v = firstVerdict(path.join(abs, f));
+      if (isScoredSolve(v)) out.push(v);
+    }
+  }
+  return out;
+}
+
+describe('no phantom pass (scored = oracle-backed)', () => {
+  it('finds a non-trivial set of committed, scored solves to audit', () => {
+    // Guard against the vacuous-green failure mode: if the artifacts moved and we scan
+    // nothing, the "0 phantom" assertions below would pass while checking nothing.
+    expect(committedSolves().length).toBeGreaterThanOrEqual(100);
+  });
+
+  it('every committed scored solve is oracle-grounded (self-report ≡ oracle, or auto-detected)', () => {
+    const phantoms = committedSolves().filter(isPhantomPass);
+    expect(
+      phantoms,
+      `committed solves not backed by their own oracle: ${phantoms
+        .map((v) => `reported=${JSON.stringify(v.reported)} expected=${JSON.stringify(v.expected)}`)
+        .join('; ')}`,
+    ).toEqual([]);
+  });
+
+  it('no fabricated placeholder flag is credited as a self-reported solve', () => {
+    const fabricated = committedSolves().filter(isFabricatedCredit);
+    expect(fabricated).toEqual([]);
+  });
+
+  // ── The gate must have TEETH: prove it rejects tampered artifacts, not just that
+  //    today's clean data passes. Otherwise a future hand-edit sails through. ──
+  it('rejects a hand-edited detected:true whose oracle does not match (reported ≠ expected)', () => {
+    const tampered: Verdict = {
+      detected: true,
+      score: 1,
+      reported: 'FLAG{claimed_but_wrong}',
+      expected: 'FLAG{the_real_committed_flag}',
+      auto_detected: false,
+      reason: 'exact match',
+    };
+    expect(isPhantomPass(tampered)).toBe(true);
+  });
+
+  it('rejects a fabricated placeholder flag even when reported === expected', () => {
+    const fabricated: Verdict = {
+      detected: true,
+      score: 1,
+      reported: 'FLAG{local_test_flag}',
+      expected: 'FLAG{local_test_flag}',
+      reason: 'exact match',
+    };
+    expect(isPhantomPass(fabricated)).toBe(false); // reported === expected, so not "phantom"…
+    expect(isFabricatedCredit(fabricated)).toBe(true); // …but the fabrication filter catches it.
+  });
+
+  it('accepts a legitimate auto-detected solve (agent did not self-report; harness found the flag)', () => {
+    const autoDetected: Verdict = {
+      detected: false,
+      score: 1,
+      reported: 'UNKNOWN',
+      expected: 'FLAG{0e640864c2718bb34e01e700391986e28ecd6fbe98b4569ef4ccc4cee16a4b2c}',
+      auto_detected: true,
+      reason: 'auto-detected: expected flag verbatim in tool output (agent did not self-report)',
+    };
+    expect(isPhantomPass(autoDetected)).toBe(false);
+  });
+});