#!/usr/bin/env node
/**
 * model-matrix — build and verify an apples-to-apples model comparison from
 * committed benchmark artifacts. This command never runs models or targets.
 */
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const DEFAULT_MANIFEST = 'bench/model-matrix/cybench-opus-pass1.manifest.json';
const OUTCOMES = ['success', 'failure', 'abstention', 'infrastructure_error', 'skipped'];

const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8'));
const rel = (file) => path.relative(REPO, file).split(path.sep).join('/');

export function classifyResult(result = {}) {
  const verdict = result.verdict || {};
  const text = `${result.error || ''} ${verdict.reason || ''} ${verdict.disposition || ''}`.toLowerCase();
  if (result.error || verdict.disposition === 'ERROR' || /timeout|connection|infrastructure|spawn|api error|judge error/.test(text)) {
    return 'infrastructure_error';
  }
  if (verdict.detected === true || verdict.score === 1) return 'success';
  if (/refus|abstain|declin|no[- ]action|skipp/.test(text)) return 'abstention';
  return 'failure';
}

export function validateManifest(manifest) {
  if (manifest?.schema !== 't3mp3st.bench.model-matrix-manifest/v1') throw new Error('unsupported manifest schema');
  if (!manifest.id || !manifest.corpus || !manifest.attempt_policy || !manifest.scoring_policy) {
    throw new Error('manifest requires id, corpus, attempt_policy, and scoring_policy');
  }
  if (!Array.isArray(manifest.tasks) || manifest.tasks.length === 0 || new Set(manifest.tasks).size !== manifest.tasks.length) {
    throw new Error('manifest tasks must be a non-empty unique array');
  }
  if (!Array.isArray(manifest.subjects) || manifest.subjects.length < 2) throw new Error('manifest requires at least two subjects');
  const subjectIds = new Set();
  for (const subject of manifest.subjects) {
    for (const field of ['id', 'model', 'provider', 'model_version_or_date', 'harness', 'agent_runtime', 'tool_access', 'target_class', 'run_mode', 'source_dir']) {
      if (!subject[field]) throw new Error(`subject missing ${field}`);
    }
    if (subjectIds.has(subject.id)) throw new Error(`duplicate subject id: ${subject.id}`);
    subjectIds.add(subject.id);
  }
}

function loadTask(subject, taskId, root = REPO) {
  const artifact = path.resolve(root, subject.source_dir, `${subject.file_prefix || ''}${taskId}.json`);
  if (!fs.existsSync(artifact)) {
    return { task_id: taskId, outcome: 'skipped', source_artifact: rel(artifact), reason: 'artifact missing' };
  }
  const data = readJson(artifact);
  if (data.model !== subject.model) throw new Error(`${rel(artifact)} model ${data.model} != ${subject.model}`);
  if (data.hunter !== subject.harness) throw new Error(`${rel(artifact)} harness ${data.hunter} != ${subject.harness}`);
  if (data.schema !== subject.artifact_schema) throw new Error(`${rel(artifact)} schema ${data.schema} != ${subject.artifact_schema}`);
  const result = (data.results || []).find((entry) => entry.id === taskId);
  if (!result) throw new Error(`${rel(artifact)} does not contain task ${taskId}`);
  const outcome = classifyResult(result);
  return {
    task_id: taskId,
    outcome,
    source_artifact: rel(artifact),
    artifact_timestamp: data.timestamp,
    category: result.category || 'unknown',
    duration_sec: typeof result.duration_sec === 'number' ? result.duration_sec : null,
    reason: String(result.error || result.verdict?.reason || ''),
  };
}

export function buildReport(manifestOrPath = DEFAULT_MANIFEST, root = REPO) {
  const manifestPath = typeof manifestOrPath === 'string'
    ? path.resolve(REPO, manifestOrPath)
    : null;
  const manifest = manifestPath ? readJson(manifestPath) : manifestOrPath;
  validateManifest(manifest);
  const rows = manifest.subjects.map((subject) => {
    const tasks = manifest.tasks.map((taskId) => loadTask(subject, taskId, root));
    const outcomes = Object.fromEntries(OUTCOMES.map((outcome) => [outcome, tasks.filter((task) => task.outcome === outcome).length]));
    const attempted = tasks.length - outcomes.skipped;
    return {
      id: subject.id,
      model: subject.model,
      provider: subject.provider,
      model_version_or_date: subject.model_version_or_date,
      harness: subject.harness,
      agent_runtime: subject.agent_runtime,
      tool_access: subject.tool_access,
      target_class: subject.target_class,
      run_mode: subject.run_mode,
      attempt_policy: manifest.attempt_policy,
      scoring_policy: manifest.scoring_policy,
      outcomes,
      attempted,
      success_rate: attempted ? outcomes.success / attempted : 0,
      tasks,
    };
  });
  return {
    schema: 't3mp3st.bench.model-matrix/v1',
    id: manifest.id,
    generated_at: manifest.generated_at,
    source_manifest: manifestPath ? rel(manifestPath) : null,
    corpus: manifest.corpus,
    corpus_size: manifest.tasks.length,
    task_ids: manifest.tasks,
    attempt_policy: manifest.attempt_policy,
    scoring_policy: manifest.scoring_policy,
    comparison_limits: manifest.comparison_limits,
    rows,
  };
}

const pct = (value) => `${(value * 100).toFixed(1)}%`;
const symbol = (outcome) => ({
  success: '✅',
  failure: '❌',
  abstention: '⏸️',
  infrastructure_error: '⚠️',
  skipped: '⏭️',
}[outcome]);

export function renderMarkdown(report) {
  const lines = [
    '# Model / harness benchmark matrix',
    '',
    `Corpus: **${report.corpus}** (${report.corpus_size} tasks) · attempt policy: **${report.attempt_policy}** · scoring: **${report.scoring_policy}**.`,
    '',
    '> This is a system comparison: model + provider + harness + tool access. It does not isolate model capability.',
    '',
    '| model | provider | harness | runtime | tool access | success | failure | abstention | infra error | skipped | rate |',
    '|---|---|---|---|---|---:|---:|---:|---:|---:|---:|',
  ];
  for (const row of report.rows) {
    const o = row.outcomes;
    lines.push(`| ${row.model} | ${row.provider} | ${row.harness} | ${row.agent_runtime} | ${row.tool_access} | ${o.success} | ${o.failure} | ${o.abstention} | ${o.infrastructure_error} | ${o.skipped} | ${pct(row.success_rate)} |`);
  }
  lines.push('', '## Task outcomes', '');
  lines.push(`| task | ${report.rows.map((row) => row.model).join(' | ')} |`);
  lines.push(`|---|${report.rows.map(() => '---').join('|')}|`);
  for (const taskId of report.task_ids) {
    lines.push(`| ${taskId} | ${report.rows.map((row) => {
      const task = row.tasks.find((entry) => entry.task_id === taskId);
      return `[${symbol(task.outcome)} ${task.outcome}](../../${task.source_artifact})`;
    }).join(' | ')} |`);
  }
  lines.push('', '## Limits', '', report.comparison_limits, '',
    'Outcome classes: ✅ success · ❌ benchmark failure · ⏸️ refusal/abstention · ⚠️ infrastructure error · ⏭️ skipped/unavailable.');
  return `${lines.join('\n')}\n`;
}

export function verifyCommitted(manifestPath = DEFAULT_MANIFEST) {
  const manifest = readJson(path.resolve(REPO, manifestPath));
  const report = buildReport(manifestPath);
  const jsonPath = path.resolve(REPO, manifest.output_json);
  const mdPath = path.resolve(REPO, manifest.output_markdown);
  const expectedJson = `${JSON.stringify(report, null, 2)}\n`;
  const expectedMarkdown = renderMarkdown(report);
  return {
    ok: fs.existsSync(jsonPath) && fs.existsSync(mdPath)
      && fs.readFileSync(jsonPath, 'utf8') === expectedJson
      && fs.readFileSync(mdPath, 'utf8') === expectedMarkdown,
    report,
    jsonPath,
    mdPath,
    expectedJson,
    expectedMarkdown,
  };
}

function selfTest() {
  const cases = [
    ['success', classifyResult({ verdict: { detected: true, score: 1 } })],
    ['failure', classifyResult({ verdict: { detected: false, reason: 'mismatch' } })],
    ['abstention', classifyResult({ verdict: { detected: false, reason: 'model refusal' } })],
    ['infrastructure_error', classifyResult({ error: 'request timeout' })],
  ];
  let failed = 0;
  for (const [expected, actual] of cases) {
    const ok = expected === actual;
    console.log(`  ${ok ? '✅' : '❌'} classify ${expected}`);
    if (!ok) failed++;
  }
  try {
    validateManifest({ schema: 'wrong' });
    failed++;
    console.log('  ❌ malformed manifest rejected');
  } catch {
    console.log('  ✅ malformed manifest rejected');
  }
  console.log(`\n${failed ? `❌ ${failed} FAILED` : '✅ ALL PASS'} — model-matrix self-test\n`);
  return failed === 0;
}

function parseArgs(argv) {
  const args = {};
  for (let i = 0; i < argv.length; i++) {
    const token = argv[i];
    if (!token.startsWith('--')) continue;
    const key = token.slice(2);
    args[key] = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[++i] : true;
  }
  return args;
}

async function main() {
  const args = parseArgs(process.argv.slice(2));
  if (args['self-test']) process.exit(selfTest() ? 0 : 1);
  const manifestPath = String(args.manifest || DEFAULT_MANIFEST);
  if (args.verify) {
    const result = verifyCommitted(manifestPath);
    console.log(result.ok ? '✅ model matrix verified from committed artifacts' : '❌ model matrix differs from committed artifacts');
    process.exit(result.ok ? 0 : 1);
  }
  const manifest = readJson(path.resolve(REPO, manifestPath));
  const report = buildReport(manifestPath);
  fs.writeFileSync(path.resolve(REPO, manifest.output_json), `${JSON.stringify(report, null, 2)}\n`);
  fs.writeFileSync(path.resolve(REPO, manifest.output_markdown), renderMarkdown(report));
  console.log(`generated ${manifest.output_json} and ${manifest.output_markdown}`);
}

const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMain) main().catch((error) => {
  console.error(error.message);
  process.exit(1);
});