fix(keyless): text-CLI tool-calling (hardened per audit) + 2 review fixes

This commit is contained in:
t3mp3st contributors committed 2026-07-05 14:09:50 -04:00
1 parent bd2463e0c3
commit 921ccc2fa6
4 files changed
+245 -15

No files matched your search

+3 -2
View File
@@ -3,7 +3,7 @@
// The recommender (composeSquad tuning + rankConfigs learning) may read ONLY the TRAIN split, and
// any "config X beats config Y" comparison is reported on TEST. A generalization claim ("our tuned
// squads win in general") may use ONLY the HOLDOUT split — fresh, never-seen CVEs (CVE-Zero-v2),
// which is intentionally EMPTY until curated. This is what stops us from overfitting the eval and
// a curated 10-sample set the hardened prompts were never tuned on. This is what stops us from overfitting the eval and
// then quoting the overfit number. The three families shared across the corpus (path / ssrf / authz)
// each land >=1 in BOTH train and test so neither half is blind to them.
export const SPLIT = {
@@ -39,7 +39,8 @@ if (isMain && process.argv.includes('--self-test')) {
ok('train + test are 5 + 5, disjoint', SPLIT.train.length === 5 && SPLIT.test.length === 5 && !SPLIT.train.some((x) => SPLIT.test.includes(x)));
ok('every split id is a real sample', [...SPLIT.train, ...SPLIT.test].every((id) => gt[id]));
for (const f of ['path', 'ssrf', 'authz']) ok(`shared family "${f}" appears in BOTH train and test`, trainF.has(f) && testF.has(f));
ok('holdout is empty until CVE-Zero-v2 is curated', SPLIT.holdout.length === 0);
ok('holdout is the curated 10-sample CVE-Zero-v2 split (real samples, disjoint from train/test)',
SPLIT.holdout.length === 10 && SPLIT.holdout.every((id) => gt[id]) && !SPLIT.holdout.some((x) => SPLIT.train.includes(x) || SPLIT.test.includes(x)));
console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}\n`);
process.exitCode = fail === 0 ? 0 : 1;
};