T3MP3ST v1 — hardened launch (AGPL, verify-claims 24/24, leak-scrubbed)

scripts/cve-zero-split.mjs

@@ -0,0 +1,47 @@
+// cve-zero-split.mjs — the HONESTY RAIL: a deterministic, family-balanced train/test/holdout split.
+//
+// The recommender (composeSquad tuning + rankConfigs learning) may read ONLY the TRAIN split, and
+// any "config X beats config Y" comparison is reported on TEST. A generalization claim ("our tuned
+// squads win in general") may use ONLY the HOLDOUT split — fresh, never-seen CVEs (CVE-Zero-v2),
+// which is intentionally EMPTY until curated. This is what stops us from overfitting the eval and
+// then quoting the overfit number. The three families shared across the corpus (path / ssrf / authz)
+// each land >=1 in BOTH train and test so neither half is blind to them.
+export const SPLIT = {
+  train: ['GHSA-hx53-77qj-8663', 'GHSA-fqvv-jvhr-g5jc', 'GHSA-3jmg-p96m-m328', 'GHSA-hm2w-vr2p-hq7w', 'GHSA-m6xr-fvfg-5g64'],
+  test: ['GHSA-ph9p-34f9-6g65', 'GHSA-w4g9-mxgg-j532', 'GHSA-rg3m-cfq7-g6h6', 'GHSA-vv9j-gjw2-j8wp', 'GHSA-36hh-x5p5-jgc8'],
+  // CVE-Zero-v2 — fresh 2026 CVEs the hardened prompts have NEVER seen (built 2026-07-04). The ONLY
+  // split a generalization claim may use. Diverse: path/authz (gonic), authz (nezha), path (dulwich),
+  // memory (oj), proto-pollution (adonisjs, piscina), redos (jsonata), ssrf (fluentd).
+  holdout: ['GHSA-2fp4-5v5c-4448', 'GHSA-q6xx-5vr8-p898', 'GHSA-gfhv-vqv2-4544', 'GHSA-3m6q-jj5j-38c9', 'GHSA-qcm7-3vpr-hj5h', 'GHSA-x9g3-xrwr-cwfg', 'GHSA-86vw-mfpg-wwv9', 'GHSA-72f5-rr8c-r6gr', 'GHSA-wc3f-xc32-435f', 'GHSA-97vg-427p-8hx5'],
+};
+
+export function splitOf(id) {
+  for (const k of ['train', 'test', 'holdout']) if (SPLIT[k].includes(id)) return k;
+  return 'unknown';
+}
+
+// map a --split arg (train|test|holdout|all) to a set of ids to keep, or null for 'all' (back-compat)
+export function splitIds(which) {
+  if (!which || which === 'all' || which === 'true') return null;
+  return SPLIT[which] || null;
+}
+
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain && process.argv.includes('--self-test')) {
+  const run = async () => {
+    const { loadSamples, fam } = await import('./cve-zero-hunt.mjs');
+    const gt = Object.fromEntries(loadSamples().map((s) => [s.id, s.gt]));
+    const famsIn = (ids) => new Set(ids.flatMap((id) => (gt[id]?.cwes || []).map(fam)));
+    const trainF = famsIn(SPLIT.train), testF = famsIn(SPLIT.test);
+    let pass = 0, fail = 0; const ok = (l, c) => (c ? (pass++, console.log('  ✅ ' + l)) : (fail++, console.log('  ❌ ' + l)));
+    ok('train + test are 5 + 5, disjoint', SPLIT.train.length === 5 && SPLIT.test.length === 5 && !SPLIT.train.some((x) => SPLIT.test.includes(x)));
+    ok('every split id is a real sample', [...SPLIT.train, ...SPLIT.test].every((id) => gt[id]));
+    for (const f of ['path', 'ssrf', 'authz']) ok(`shared family "${f}" appears in BOTH train and test`, trainF.has(f) && testF.has(f));
+    ok('holdout is empty until CVE-Zero-v2 is curated', SPLIT.holdout.length === 0);
+    console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}\n`);
+    process.exitCode = fail === 0 ? 0 : 1;
+  };
+  run();
+}