feat(verify-claims): gate every committed solve as oracle-backed (no phantom pass)
verify-claims credits a solve straight from the committed `verdict` (detected / score===1). Nothing asserted that a scored solve actually agrees with its own committed flag oracle, so a hand-edited `detected:true` (or a merge slip / fabricated placeholder flag) would pass verification silently. The case-insensitive oracle comparator `ciMatch` was defined but never called — this is exactly the check it was meant for. Add an INTEGRITY GATE that re-derives, for every scored solve across the pinned golden + white-box XBEN sweeps and the Cybench per-run artifacts, that it is oracle-grounded: EITHER the self-reported flag matches the committed `expected` (ciMatch), OR the harness auto-detected `expected` verbatim in tool output (auto_detected, reported may be "UNKNOWN"). Anything scored that is neither is a phantom pass and fails the build. Also rejects a fabricated placeholder credited via the self-report path, and reports how much of the Cybench headline is backed by an oracle-bearing per-run artifact vs. counted only in the hand-editable aggregate. Adds src/__tests__/oracle-consistency.test.ts in the same spirit as no-phantom-tools / stub-honesty: audits the committed artifacts AND proves the gate has teeth against synthetic tampered/fabricated verdicts. Current receipts pass clean: 0 phantom / 606 scored solves, 0 fabricated.
This commit is contained in:
1 parent
a5667374bf
commit
47fc8e9f77
2 files changed
+182
No files matched your search
@@ -229,6 +229,48 @@ console.log('\nCLAIM 7 — CVE-Zero-v2 (held-out): hunt generalizes to fresh uns
|
||||
}
|
||||
}
|
||||
|
||||
// ── INTEGRITY GATE: every scored solve is oracle-grounded (no phantom pass) ───
|
||||
// Every check above credits a solve straight from the committed `verdict` (detected /
|
||||
// score===1). This gate closes the obvious tamper hole in that trust: a scored solve must
|
||||
// still AGREE with its own committed flag oracle. A solve is oracle-grounded iff EITHER the
|
||||
// agent's self-reported flag matches the committed `expected` under the same case-insensitive
|
||||
// compare the harness uses (ciMatch), OR the harness auto-detected `expected` verbatim in tool
|
||||
// output (auto_detected — the agent didn't self-report, so `reported` may be "UNKNOWN").
|
||||
// Anything scored as a solve that is NEITHER is a phantom pass: a hand-edited `detected:true`
|
||||
// whose committed oracle does not back it. This wires ciMatch (until now defined but unused)
|
||||
// into an executable gate, and rejects a fabricated placeholder credited via the self-report path.
|
||||
console.log('\nINTEGRITY GATE — every committed solve is oracle-grounded (no phantom pass, no fabricated flag)');
|
||||
const oracleGrounded = (v) =>
|
||||
ciMatch(v.reported, v.expected) || v.auto_detected === true || /auto-?detected/i.test(String(v.reason || ''));
|
||||
const scanSolves = (files) => {
|
||||
let solves = 0, phantom = 0, fabricated = 0, grounded = 0;
|
||||
for (const f of files) {
|
||||
const v = (((JSON.parse(fs.readFileSync(f, 'utf8')).results) || [{}])[0] || {}).verdict || {};
|
||||
if (!(v.detected === true || v.score === 1)) continue;
|
||||
solves++;
|
||||
if (oracleGrounded(v)) grounded++; else phantom++;
|
||||
// A fabricated placeholder may only ever be "credited" via the self-report path.
|
||||
if (ciMatch(v.reported, v.expected) && looksFabricated(v.reported)) fabricated++;
|
||||
}
|
||||
return { solves, phantom, fabricated, grounded };
|
||||
};
|
||||
const xbenFiles = [...BESTBALL_DIRS, ...WHITEBOX_DIRS].flatMap((sub) =>
|
||||
glob(R('bench/xbow/results/' + sub), /^xben_\d+_24.*\.json$/));
|
||||
const cybenchFiles = [...glob(R('bench/cybench/results'), /^clean-.*\.json$/),
|
||||
...glob(R('bench/cybench/results'), /^service-cybsvc_.*\.json$/)];
|
||||
const ig = scanSolves([...xbenFiles, ...cybenchFiles]);
|
||||
check('every scored solve is oracle-grounded (self-report ≡ committed oracle, or auto-detected)',
|
||||
ig.phantom === 0, `${ig.phantom} phantom / ${ig.solves} scored solves`);
|
||||
check('no fabricated placeholder flag is credited as a self-reported solve',
|
||||
ig.fabricated === 0, `${ig.fabricated} fabricated / ${ig.solves} scored solves`);
|
||||
// Informational — how much of the Cybench headline is backed by an oracle-bearing per-run
|
||||
// artifact vs. counted only in the hand-editable aggregate (transcripts are stripped on export).
|
||||
{
|
||||
const cRun = scanSolves(cybenchFiles);
|
||||
const aggSolved = fs.existsSync(cf) ? JSON.parse(fs.readFileSync(cf, 'utf8')).solved : null;
|
||||
console.log(` ℹ️ Cybench: aggregate reports ${aggSolved} solved · ${cRun.grounded} oracle-grounded per-run artifact(s) committed · ${(aggSolved ?? 0) - cRun.grounded} aggregate-only`);
|
||||
}
|
||||
|
||||
// ── verdict ─────────────────────────────────────────────────────────────────
|
||||
console.log(`\n════════ ${fail === 0 ? '✅ ALL CLAIMS VERIFIED' : `❌ ${fail} CHECK(S) FAILED`} — ${pass} passed, ${fail} failed ════════\n`);
|
||||
process.exit(fail === 0 ? 0 : 1);
|
||||
Reference in new issue
Block a user