add reproducible cross-model benchmark matrix (#107)

scripts/model-matrix.mjs

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+/**
+ * model-matrix — build and verify an apples-to-apples model comparison from
+ * committed benchmark artifacts. This command never runs models or targets.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const DEFAULT_MANIFEST = 'bench/model-matrix/cybench-opus-pass1.manifest.json';
+const OUTCOMES = ['success', 'failure', 'abstention', 'infrastructure_error', 'skipped'];
+
+const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8'));
+const rel = (file) => path.relative(REPO, file).split(path.sep).join('/');
+
+export function classifyResult(result = {}) {
+  const verdict = result.verdict || {};
+  const text = `${result.error || ''} ${verdict.reason || ''} ${verdict.disposition || ''}`.toLowerCase();
+  if (result.error || verdict.disposition === 'ERROR' || /timeout|connection|infrastructure|spawn|api error|judge error/.test(text)) {
+    return 'infrastructure_error';
+  }
+  if (verdict.detected === true || verdict.score === 1) return 'success';
+  if (/refus|abstain|declin|no[- ]action|skipp/.test(text)) return 'abstention';
+  return 'failure';
+}
+
+export function validateManifest(manifest) {
+  if (manifest?.schema !== 't3mp3st.bench.model-matrix-manifest/v1') throw new Error('unsupported manifest schema');
+  if (!manifest.id || !manifest.corpus || !manifest.attempt_policy || !manifest.scoring_policy) {
+    throw new Error('manifest requires id, corpus, attempt_policy, and scoring_policy');
+  }
+  if (!Array.isArray(manifest.tasks) || manifest.tasks.length === 0 || new Set(manifest.tasks).size !== manifest.tasks.length) {
+    throw new Error('manifest tasks must be a non-empty unique array');
+  }
+  if (!Array.isArray(manifest.subjects) || manifest.subjects.length < 2) throw new Error('manifest requires at least two subjects');
+  const subjectIds = new Set();
+  for (const subject of manifest.subjects) {
+    for (const field of ['id', 'model', 'provider', 'model_version_or_date', 'harness', 'agent_runtime', 'tool_access', 'target_class', 'run_mode', 'source_dir']) {
+      if (!subject[field]) throw new Error(`subject missing ${field}`);
+    }
+    if (subjectIds.has(subject.id)) throw new Error(`duplicate subject id: ${subject.id}`);
+    subjectIds.add(subject.id);
+  }
+}
+
+function loadTask(subject, taskId, root = REPO) {
+  const artifact = path.resolve(root, subject.source_dir, `${subject.file_prefix || ''}${taskId}.json`);
+  if (!fs.existsSync(artifact)) {
+    return { task_id: taskId, outcome: 'skipped', source_artifact: rel(artifact), reason: 'artifact missing' };
+  }
+  const data = readJson(artifact);
+  if (data.model !== subject.model) throw new Error(`${rel(artifact)} model ${data.model} != ${subject.model}`);
+  if (data.hunter !== subject.harness) throw new Error(`${rel(artifact)} harness ${data.hunter} != ${subject.harness}`);
+  if (data.schema !== subject.artifact_schema) throw new Error(`${rel(artifact)} schema ${data.schema} != ${subject.artifact_schema}`);
+  const result = (data.results || []).find((entry) => entry.id === taskId);
+  if (!result) throw new Error(`${rel(artifact)} does not contain task ${taskId}`);
+  const outcome = classifyResult(result);
+  return {
+    task_id: taskId,
+    outcome,
+    source_artifact: rel(artifact),
+    artifact_timestamp: data.timestamp,
+    category: result.category || 'unknown',
+    duration_sec: typeof result.duration_sec === 'number' ? result.duration_sec : null,
+    reason: String(result.error || result.verdict?.reason || ''),
+  };
+}
+
+export function buildReport(manifestOrPath = DEFAULT_MANIFEST, root = REPO) {
+  const manifestPath = typeof manifestOrPath === 'string'
+    ? path.resolve(REPO, manifestOrPath)
+    : null;
+  const manifest = manifestPath ? readJson(manifestPath) : manifestOrPath;
+  validateManifest(manifest);
+  const rows = manifest.subjects.map((subject) => {
+    const tasks = manifest.tasks.map((taskId) => loadTask(subject, taskId, root));
+    const outcomes = Object.fromEntries(OUTCOMES.map((outcome) => [outcome, tasks.filter((task) => task.outcome === outcome).length]));
+    const attempted = tasks.length - outcomes.skipped;
+    return {
+      id: subject.id,
+      model: subject.model,
+      provider: subject.provider,
+      model_version_or_date: subject.model_version_or_date,
+      harness: subject.harness,
+      agent_runtime: subject.agent_runtime,
+      tool_access: subject.tool_access,
+      target_class: subject.target_class,
+      run_mode: subject.run_mode,
+      attempt_policy: manifest.attempt_policy,
+      scoring_policy: manifest.scoring_policy,
+      outcomes,
+      attempted,
+      success_rate: attempted ? outcomes.success / attempted : 0,
+      tasks,
+    };
+  });
+  return {
+    schema: 't3mp3st.bench.model-matrix/v1',
+    id: manifest.id,
+    generated_at: manifest.generated_at,
+    source_manifest: manifestPath ? rel(manifestPath) : null,
+    corpus: manifest.corpus,
+    corpus_size: manifest.tasks.length,
+    task_ids: manifest.tasks,
+    attempt_policy: manifest.attempt_policy,
+    scoring_policy: manifest.scoring_policy,
+    comparison_limits: manifest.comparison_limits,
+    rows,
+  };
+}
+
+const pct = (value) => `${(value * 100).toFixed(1)}%`;
+const symbol = (outcome) => ({
+  success: '✅',
+  failure: '❌',
+  abstention: '⏸️',
+  infrastructure_error: '⚠️',
+  skipped: '⏭️',
+}[outcome]);
+
+export function renderMarkdown(report) {
+  const lines = [
+    '# Model / harness benchmark matrix',
+    '',
+    `Corpus: **${report.corpus}** (${report.corpus_size} tasks) · attempt policy: **${report.attempt_policy}** · scoring: **${report.scoring_policy}**.`,
+    '',
+    '> This is a system comparison: model + provider + harness + tool access. It does not isolate model capability.',
+    '',
+    '| model | provider | harness | runtime | tool access | success | failure | abstention | infra error | skipped | rate |',
+    '|---|---|---|---|---|---:|---:|---:|---:|---:|---:|',
+  ];
+  for (const row of report.rows) {
+    const o = row.outcomes;
+    lines.push(`| ${row.model} | ${row.provider} | ${row.harness} | ${row.agent_runtime} | ${row.tool_access} | ${o.success} | ${o.failure} | ${o.abstention} | ${o.infrastructure_error} | ${o.skipped} | ${pct(row.success_rate)} |`);
+  }
+  lines.push('', '## Task outcomes', '');
+  lines.push(`| task | ${report.rows.map((row) => row.model).join(' | ')} |`);
+  lines.push(`|---|${report.rows.map(() => '---').join('|')}|`);
+  for (const taskId of report.task_ids) {
+    lines.push(`| ${taskId} | ${report.rows.map((row) => {
+      const task = row.tasks.find((entry) => entry.task_id === taskId);
+      return `[${symbol(task.outcome)} ${task.outcome}](../../${task.source_artifact})`;
+    }).join(' | ')} |`);
+  }
+  lines.push('', '## Limits', '', report.comparison_limits, '',
+    'Outcome classes: ✅ success · ❌ benchmark failure · ⏸️ refusal/abstention · ⚠️ infrastructure error · ⏭️ skipped/unavailable.');
+  return `${lines.join('\n')}\n`;
+}
+
+export function verifyCommitted(manifestPath = DEFAULT_MANIFEST) {
+  const manifest = readJson(path.resolve(REPO, manifestPath));
+  const report = buildReport(manifestPath);
+  const jsonPath = path.resolve(REPO, manifest.output_json);
+  const mdPath = path.resolve(REPO, manifest.output_markdown);
+  const expectedJson = `${JSON.stringify(report, null, 2)}\n`;
+  const expectedMarkdown = renderMarkdown(report);
+  return {
+    ok: fs.existsSync(jsonPath) && fs.existsSync(mdPath)
+      && fs.readFileSync(jsonPath, 'utf8') === expectedJson
+      && fs.readFileSync(mdPath, 'utf8') === expectedMarkdown,
+    report,
+    jsonPath,
+    mdPath,
+    expectedJson,
+    expectedMarkdown,
+  };
+}
+
+function selfTest() {
+  const cases = [
+    ['success', classifyResult({ verdict: { detected: true, score: 1 } })],
+    ['failure', classifyResult({ verdict: { detected: false, reason: 'mismatch' } })],
+    ['abstention', classifyResult({ verdict: { detected: false, reason: 'model refusal' } })],
+    ['infrastructure_error', classifyResult({ error: 'request timeout' })],
+  ];
+  let failed = 0;
+  for (const [expected, actual] of cases) {
+    const ok = expected === actual;
+    console.log(`  ${ok ? '✅' : '❌'} classify ${expected}`);
+    if (!ok) failed++;
+  }
+  try {
+    validateManifest({ schema: 'wrong' });
+    failed++;
+    console.log('  ❌ malformed manifest rejected');
+  } catch {
+    console.log('  ✅ malformed manifest rejected');
+  }
+  console.log(`\n${failed ? `❌ ${failed} FAILED` : '✅ ALL PASS'} — model-matrix self-test\n`);
+  return failed === 0;
+}
+
+function parseArgs(argv) {
+  const args = {};
+  for (let i = 0; i < argv.length; i++) {
+    const token = argv[i];
+    if (!token.startsWith('--')) continue;
+    const key = token.slice(2);
+    args[key] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
+  }
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args['self-test']) process.exit(selfTest() ? 0 : 1);
+  const manifestPath = String(args.manifest || DEFAULT_MANIFEST);
+  if (args.verify) {
+    const result = verifyCommitted(manifestPath);
+    console.log(result.ok ? '✅ model matrix verified from committed artifacts' : '❌ model matrix differs from committed artifacts');
+    process.exit(result.ok ? 0 : 1);
+  }
+  const manifest = readJson(path.resolve(REPO, manifestPath));
+  const report = buildReport(manifestPath);
+  fs.writeFileSync(path.resolve(REPO, manifest.output_json), `${JSON.stringify(report, null, 2)}\n`);
+  fs.writeFileSync(path.resolve(REPO, manifest.output_markdown), renderMarkdown(report));
+  console.log(`generated ${manifest.output_json} and ${manifest.output_markdown}`);
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) main().catch((error) => {
+  console.error(error.message);
+  process.exit(1);
+});