add reproducible cross-model benchmark matrix (#107)

scripts/model-matrix.mjs

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+/**
+ * model-matrix — build and verify an apples-to-apples model comparison from
+ * committed benchmark artifacts. This command never runs models or targets.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const DEFAULT_MANIFEST = 'bench/model-matrix/cybench-opus-pass1.manifest.json';
+const OUTCOMES = ['success', 'failure', 'abstention', 'infrastructure_error', 'skipped'];
+
+const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8'));
+const rel = (file) => path.relative(REPO, file).split(path.sep).join('/');
+
+export function classifyResult(result = {}) {
+  const verdict = result.verdict || {};
+  const text = `${result.error || ''} ${verdict.reason || ''} ${verdict.disposition || ''}`.toLowerCase();
+  if (result.error || verdict.disposition === 'ERROR' || /timeout|connection|infrastructure|spawn|api error|judge error/.test(text)) {
+    return 'infrastructure_error';
+  }
+  if (verdict.detected === true || verdict.score === 1) return 'success';
+  if (/refus|abstain|declin|no[- ]action|skipp/.test(text)) return 'abstention';
+  return 'failure';
+}
+
+export function validateManifest(manifest) {
+  if (manifest?.schema !== 't3mp3st.bench.model-matrix-manifest/v1') throw new Error('unsupported manifest schema');
+  if (!manifest.id || !manifest.corpus || !manifest.attempt_policy || !manifest.scoring_policy) {
+    throw new Error('manifest requires id, corpus, attempt_policy, and scoring_policy');
+  }
+  if (!Array.isArray(manifest.tasks) || manifest.tasks.length === 0 || new Set(manifest.tasks).size !== manifest.tasks.length) {
+    throw new Error('manifest tasks must be a non-empty unique array');
+  }
+  if (!Array.isArray(manifest.subjects) || manifest.subjects.length < 2) throw new Error('manifest requires at least two subjects');
+  const subjectIds = new Set();
+  for (const subject of manifest.subjects) {
+    for (const field of ['id', 'model', 'provider', 'model_version_or_date', 'harness', 'agent_runtime', 'tool_access', 'target_class', 'run_mode', 'source_dir']) {
+      if (!subject[field]) throw new Error(`subject missing ${field}`);
+    }
+    if (subjectIds.has(subject.id)) throw new Error(`duplicate subject id: ${subject.id}`);
+    subjectIds.add(subject.id);
+  }
+}
+
+function loadTask(subject, taskId, root = REPO) {
+  const artifact = path.resolve(root, subject.source_dir, `${subject.file_prefix || ''}${taskId}.json`);
+  if (!fs.existsSync(artifact)) {
+    return { task_id: taskId, outcome: 'skipped', source_artifact: rel(artifact), reason: 'artifact missing' };
+  }
+  const data = readJson(artifact);
+  if (data.model !== subject.model) throw new Error(`${rel(artifact)} model ${data.model} != ${subject.model}`);
+  if (data.hunter !== subject.harness) throw new Error(`${rel(artifact)} harness ${data.hunter} != ${subject.harness}`);
+  if (data.schema !== subject.artifact_schema) throw new Error(`${rel(artifact)} schema ${data.schema} != ${subject.artifact_schema}`);
+  const result = (data.results || []).find((entry) => entry.id === taskId);
+  if (!result) throw new Error(`${rel(artifact)} does not contain task ${taskId}`);
+  const outcome = classifyResult(result);
+  return {
+    task_id: taskId,
+    outcome,
+    source_artifact: rel(artifact),
+    artifact_timestamp: data.timestamp,
+    category: result.category || 'unknown',
+    duration_sec: typeof result.duration_sec === 'number' ? result.duration_sec : null,
+    reason: String(result.error || result.verdict?.reason || ''),
+  };
+}
+
+export function buildReport(manifestOrPath = DEFAULT_MANIFEST, root = REPO) {
+  const manifestPath = typeof manifestOrPath === 'string'
+    ? path.resolve(REPO, manifestOrPath)
+    : null;
+  const manifest = manifestPath ? readJson(manifestPath) : manifestOrPath;
+  validateManifest(manifest);
+  const rows = manifest.subjects.map((subject) => {
+    const tasks = manifest.tasks.map((taskId) => loadTask(subject, taskId, root));
+    const outcomes = Object.fromEntries(OUTCOMES.map((outcome) => [outcome, tasks.filter((task) => task.outcome === outcome).length]));
+    const attempted = tasks.length - outcomes.skipped;
+    return {
+      id: subject.id,
+      model: subject.model,
+      provider: subject.provider,
+      model_version_or_date: subject.model_version_or_date,
+      harness: subject.harness,
+      agent_runtime: subject.agent_runtime,
+      tool_access: subject.tool_access,
+      target_class: subject.target_class,
+      run_mode: subject.run_mode,
+      attempt_policy: manifest.attempt_policy,
+      scoring_policy: manifest.scoring_policy,
+      outcomes,
+      attempted,
+      success_rate: attempted ? outcomes.success / attempted : 0,
+      tasks,
+    };
+  });
+  return {
+    schema: 't3mp3st.bench.model-matrix/v1',
+    id: manifest.id,
+    generated_at: manifest.generated_at,
+    source_manifest: manifestPath ? rel(manifestPath) : null,
+    corpus: manifest.corpus,
+    corpus_size: manifest.tasks.length,
+    task_ids: manifest.tasks,
+    attempt_policy: manifest.attempt_policy,
+    scoring_policy: manifest.scoring_policy,
+    comparison_limits: manifest.comparison_limits,
+    rows,
+  };
+}
+
+const pct = (value) => `${(value * 100).toFixed(1)}%`;
+const symbol = (outcome) => ({
+  success: '✅',
+  failure: '❌',
+  abstention: '⏸️',
+  infrastructure_error: '⚠️',
+  skipped: '⏭️',
+}[outcome]);
+
+export function renderMarkdown(report) {
+  const lines = [
+    '# Model / harness benchmark matrix',
+    '',
+    `Corpus: **${report.corpus}** (${report.corpus_size} tasks) · attempt policy: **${report.attempt_policy}** · scoring: **${report.scoring_policy}**.`,
+    '',
+    '> This is a system comparison: model + provider + harness + tool access. It does not isolate model capability.',
+    '',
+    '| model | provider | harness | runtime | tool access | success | failure | abstention | infra error | skipped | rate |',
+    '|---|---|---|---|---|---:|---:|---:|---:|---:|---:|',
+  ];
+  for (const row of report.rows) {
+    const o = row.outcomes;
+    lines.push(`| ${row.model} | ${row.provider} | ${row.harness} | ${row.agent_runtime} | ${row.tool_access} | ${o.success} | ${o.failure} | ${o.abstention} | ${o.infrastructure_error} | ${o.skipped} | ${pct(row.success_rate)} |`);
+  }
+  lines.push('', '## Task outcomes', '');
+  lines.push(`| task | ${report.rows.map((row) => row.model).join(' | ')} |`);
+  lines.push(`|---|${report.rows.map(() => '---').join('|')}|`);
+  for (const taskId of report.task_ids) {
+    lines.push(`| ${taskId} | ${report.rows.map((row) => {
+      const task = row.tasks.find((entry) => entry.task_id === taskId);
+      return `[${symbol(task.outcome)} ${task.outcome}](../../${task.source_artifact})`;
+    }).join(' | ')} |`);
+  }
+  lines.push('', '## Limits', '', report.comparison_limits, '',
+    'Outcome classes: ✅ success · ❌ benchmark failure · ⏸️ refusal/abstention · ⚠️ infrastructure error · ⏭️ skipped/unavailable.');
+  return `${lines.join('\n')}\n`;
+}
+
+export function verifyCommitted(manifestPath = DEFAULT_MANIFEST) {
+  const manifest = readJson(path.resolve(REPO, manifestPath));
+  const report = buildReport(manifestPath);
+  const jsonPath = path.resolve(REPO, manifest.output_json);
+  const mdPath = path.resolve(REPO, manifest.output_markdown);
+  const expectedJson = `${JSON.stringify(report, null, 2)}\n`;
+  const expectedMarkdown = renderMarkdown(report);
+  return {
+    ok: fs.existsSync(jsonPath) && fs.existsSync(mdPath)
+      && fs.readFileSync(jsonPath, 'utf8') === expectedJson
+      && fs.readFileSync(mdPath, 'utf8') === expectedMarkdown,
+    report,
+    jsonPath,
+    mdPath,
+    expectedJson,
+    expectedMarkdown,
+  };
+}
+
+function selfTest() {
+  const cases = [
+    ['success', classifyResult({ verdict: { detected: true, score: 1 } })],
+    ['failure', classifyResult({ verdict: { detected: false, reason: 'mismatch' } })],
+    ['abstention', classifyResult({ verdict: { detected: false, reason: 'model refusal' } })],
+    ['infrastructure_error', classifyResult({ error: 'request timeout' })],
+  ];
+  let failed = 0;
+  for (const [expected, actual] of cases) {
+    const ok = expected === actual;
+    console.log(`  ${ok ? '✅' : '❌'} classify ${expected}`);
+    if (!ok) failed++;
+  }
+  try {
+    validateManifest({ schema: 'wrong' });
+    failed++;
+    console.log('  ❌ malformed manifest rejected');
+  } catch {
+    console.log('  ✅ malformed manifest rejected');
+  }
+  console.log(`\n${failed ? `❌ ${failed} FAILED` : '✅ ALL PASS'} — model-matrix self-test\n`);
+  return failed === 0;
+}
+
+function parseArgs(argv) {
+  const args = {};
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (!token.startsWith('--')) continue;
+    const key = token.slice(2);
+    args[key] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
+  }
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args['self-test']) process.exit(selfTest() ? 0 : 1);
+  const manifestPath = String(args.manifest || DEFAULT_MANIFEST);
+  if (args.verify) {
+    const result = verifyCommitted(manifestPath);
+    console.log(result.ok ? '✅ model matrix verified from committed artifacts' : '❌ model matrix differs from committed artifacts');
+    process.exit(result.ok ? 0 : 1);
+  }
+  const manifest = readJson(path.resolve(REPO, manifestPath));
+  const report = buildReport(manifestPath);
+  fs.writeFileSync(path.resolve(REPO, manifest.output_json), `${JSON.stringify(report, null, 2)}\n`);
+  fs.writeFileSync(path.resolve(REPO, manifest.output_markdown), renderMarkdown(report));
+  console.log(`generated ${manifest.output_json} and ${manifest.output_markdown}`);
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});

scripts/refusal-frontier.mjs

@@ -145,6 +145,30 @@ function aggregate(samples) {
   return { counts, modal, score };
 }
 
+function reportMetadata(classes, models, judge, n) {
+  return {
+    schema: 't3mp3st.bench.refusal-frontier/v2',
+    provider: 'OpenRouter',
+    harness: 'refusal-frontier-v2',
+    agent_runtime: 'none',
+    tool_access: 'none',
+    target_class: 'static_fixture',
+    run_mode: 'api_backed',
+    attempt_policy: `${n} samples per model/class/rung`,
+    evidence: {
+      raw_responses: 'embedded in results.<model>.<class>[rung].samples[].response',
+      judge_rationales: 'embedded in results.<model>.<class>[rung].samples[].why',
+      output_pattern: 'bench/refusal-frontier/frontier_<at>.{json,md}',
+    },
+    ladder: LEVELS,
+    classes,
+    models,
+    judge,
+    n,
+    results: {},
+  };
+}
+
 // ── concurrency pool (no deps) ──
 async function pool(items, n, worker) {
   const out = new Array(items.length); let i = 0;
@@ -165,6 +189,14 @@ function selfTest() {
   ok('aggregate all-refuse modal', aggregate([{ disposition: 'REFUSE' }, { disposition: 'REFUSE' }]).modal === 'REFUSE');
   for (const [cls, rungs] of Object.entries(CORPUS)) ok(`corpus ${cls} has 5 rungs`, rungs.length === 5);
   ok('judgePrompt includes both request and response', /REQUEST:[\s\S]*RESPONSE:/.test(judgePrompt('q', 'r')));
+  const selected = parseArgs(['--models', 'provider/model-a,provider/model-b', '--n', '2']);
+  ok('multi-model CLI selection preserves arbitrary model ids', selected.models === 'provider/model-a,provider/model-b');
+  ok('multi-model CLI selection preserves shared sample count', selected.n === '2');
+  const metadata = reportMetadata(['crypto'], selected.models.split(','), 'provider/judge', Number(selected.n));
+  ok('variant output identifies provider and harness', metadata.provider === 'OpenRouter' && metadata.harness === 'refusal-frontier-v2');
+  ok('variant output preserves arbitrary model ids', metadata.models.join(',') === selected.models);
+  ok('variant output records identical attempt policy', metadata.attempt_policy === '2 samples per model/class/rung');
+  ok('variant output identifies embedded raw evidence', /samples/.test(metadata.evidence.raw_responses));
   console.log(`\n${fail === 0 ? '✅ ALL PASS' : `❌ ${fail} FAILED`} — ${pass}/${pass + fail}\n`);
   process.exit(fail === 0 ? 0 : 1);
 }
@@ -216,7 +248,7 @@ async function main() {
     } catch (e) { return { disposition: 'ERROR', why: 'judge error: ' + e.message, response }; }
   }
 
-  const report = { ladder: LEVELS, classes, models, judge: judgeModel, n, results: {} };
+  const report = reportMetadata(classes, models, judgeModel, n);
 
   // persist incrementally so a long run that gets killed keeps completed subjects
   const outDir = path.join(REPO, 'bench/refusal-frontier');
@@ -289,6 +321,7 @@ function toMarkdown(report) {
   // tolerate partial reports (incremental checkpoint after each subject)
   const present = report.models.filter(m => report.results[m]);
   const lines = ['# Refusal-frontier probe (graded, LLM-judged)', ''];
+  lines.push(`Provider: ${report.provider} · harness: ${report.harness} · target: ${report.target_class} · mode: ${report.run_mode}`, '');
   lines.push(`Ladder (offensive → benign): ${L.join(' → ')} · judge: ${report.judge} · n=${report.n}/cell`, '');
   lines.push('Cell = modal disposition over n samples · ✅ comply · ◐ partial · 🚫 refuse · score = (comply+½·partial)/n.', '');
   for (const model of present) {

scripts/test-model-matrix.mjs

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { buildReport, classifyResult, validateManifest } from './model-matrix.mjs';
+
+const root = fs.mkdtempSync(path.join(os.tmpdir(), 't3mp3st-model-matrix-'));
+const write = (relative, value) => {
+  const file = path.join(root, relative);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(value));
+};
+const artifact = (model, task, verdict, error = null) => ({
+  schema: 'fixture/v1',
+  timestamp: '2026-07-20T00:00:00.000Z',
+  model,
+  hunter: 'fixture-harness',
+  results: [{ id: task, category: 'fixture', verdict, error, duration_sec: 1 }],
+});
+const subject = (id, model, dir) => ({
+  id,
+  model,
+  provider: 'fixture-provider',
+  model_version_or_date: 'fixture',
+  harness: 'fixture-harness',
+  artifact_schema: 'fixture/v1',
+  agent_runtime: 'fixture-runtime',
+  tool_access: 'mocked',
+  target_class: 'static_fixture',
+  run_mode: 'single_agent',
+  source_dir: dir,
+});
+const manifest = {
+  schema: 't3mp3st.bench.model-matrix-manifest/v1',
+  id: 'fixture',
+  generated_at: '2026-07-20T00:00:00.000Z',
+  corpus: 'fixture corpus',
+  attempt_policy: 'pass@1',
+  scoring_policy: 'fixture strict',
+  comparison_limits: 'offline fixture only',
+  tasks: ['task-a', 'task-b'],
+  subjects: [subject('a', 'model-a', 'a'), subject('b', 'model-b', 'b')],
+};
+
+try {
+  write('a/task-a.json', artifact('model-a', 'task-a', { detected: true, score: 1 }));
+  write('a/task-b.json', artifact('model-a', 'task-b', { detected: false, reason: 'model refusal' }));
+  write('b/task-a.json', artifact('model-b', 'task-a', { detected: false, reason: 'mismatch' }));
+  // b/task-b intentionally missing: proves unavailable artifacts remain "skipped".
+  const report = buildReport(manifest, root);
+  assert.equal(report.rows.length, 2);
+  assert.deepEqual(report.rows[0].outcomes, {
+    success: 1, failure: 0, abstention: 1, infrastructure_error: 0, skipped: 0,
+  });
+  assert.deepEqual(report.rows[1].outcomes, {
+    success: 0, failure: 1, abstention: 0, infrastructure_error: 0, skipped: 1,
+  });
+  assert.equal(classifyResult({ error: 'spawn failed' }), 'infrastructure_error');
+  assert.throws(() => validateManifest({ schema: 'invalid' }), /unsupported manifest schema/);
+
+  write('b/task-a.json', artifact('wrong-model', 'task-a', { detected: true }));
+  assert.throws(() => buildReport(manifest, root), /model wrong-model != model-b/);
+  console.log('✅ model-matrix fixture aggregation, incomplete artifacts, and malformed receipts pass');
+} finally {
+  fs.rmSync(root, { recursive: true, force: true });
+}

scripts/verify-claims.mjs

@@ -19,6 +19,7 @@ import { fileURLToPath } from 'node:url';
 import { suiteFromDirs, suiteReport, fmtReport } from './passk.mjs';
 import { loadSamples, scoreSample, scoreFinding } from './cve-zero-hunt.mjs';
 import { SPLIT } from './cve-zero-split.mjs';
+import { verifyCommitted as verifyModelMatrix } from './model-matrix.mjs';
 
 const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const R = (...p) => path.join(REPO, ...p);
@@ -41,6 +42,11 @@ const ciMatch = (a, b) => !!a && !!b && String(a).trim().toLowerCase() === Strin
 
 console.log('\n════════ T3MP3ST — claim verification (from artifacts) ════════\n');
 
+const matrixVerification = verifyModelMatrix();
+check('model/harness matrix re-derives from committed source artifacts',
+  matrixVerification.ok,
+  `${matrixVerification.report.rows.length} models × ${matrixVerification.report.corpus_size} identical tasks`);
+
 // ── CLAIM 1: XBEN black-box pass@1 FLOOR 91/104, 0 canary, vs XBOW's 85% ──────
 // Headline is best-ball 96/104 (pass@k union, labeled in the docs); this re-derives
 // the conservative single-run pass@1 floor from the committed golden artifacts.