add reproducible cross-model benchmark matrix

This commit is contained in:
Joseph Magly committed 2026-07-20 14:50:17 -04:00
1 parent 58193d688d
commit 186afe6b50
10 files changed
+842 -4

No files matched your search

+6
View File
@@ -19,6 +19,7 @@ import { fileURLToPath } from 'node:url';
import { suiteFromDirs, suiteReport, fmtReport } from './passk.mjs';
import { loadSamples, scoreSample, scoreFinding } from './cve-zero-hunt.mjs';
import { SPLIT } from './cve-zero-split.mjs';
import { verifyCommitted as verifyModelMatrix } from './model-matrix.mjs';
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const R = (...p) => path.join(REPO, ...p);
@@ -41,6 +42,11 @@ const ciMatch = (a, b) => !!a && !!b && String(a).trim().toLowerCase() === Strin
console.log('\n════════ T3MP3ST — claim verification (from artifacts) ════════\n');
const matrixVerification = verifyModelMatrix();
check('model/harness matrix re-derives from committed source artifacts',
matrixVerification.ok,
`${matrixVerification.report.rows.length} models × ${matrixVerification.report.corpus_size} identical tasks`);
// ── CLAIM 1: XBEN black-box pass@1 FLOOR 91/104, 0 canary, vs XBOW's 85% ──────
// Headline is best-ball 96/104 (pass@k union, labeled in the docs); this re-derives
// the conservative single-run pass@1 floor from the committed golden artifacts.