fix(keyless): text-CLI tool-calling (hardened per audit) + 2 review fixes
This commit is contained in:
1 parent
bd2463e0c3
commit
921ccc2fa6
4 files changed
+245
-15
No files matched your search
@@ -3,7 +3,7 @@
|
||||
// The recommender (composeSquad tuning + rankConfigs learning) may read ONLY the TRAIN split, and
|
||||
// any "config X beats config Y" comparison is reported on TEST. A generalization claim ("our tuned
|
||||
// squads win in general") may use ONLY the HOLDOUT split — fresh, never-seen CVEs (CVE-Zero-v2),
|
||||
// which is intentionally EMPTY until curated. This is what stops us from overfitting the eval and
|
||||
// a curated 10-sample set the hardened prompts were never tuned on. This is what stops us from overfitting the eval and
|
||||
// then quoting the overfit number. The three families shared across the corpus (path / ssrf / authz)
|
||||
// each land >=1 in BOTH train and test so neither half is blind to them.
|
||||
export const SPLIT = {
|
||||
@@ -39,7 +39,8 @@ if (isMain && process.argv.includes('--self-test')) {
|
||||
ok('train + test are 5 + 5, disjoint', SPLIT.train.length === 5 && SPLIT.test.length === 5 && !SPLIT.train.some((x) => SPLIT.test.includes(x)));
|
||||
ok('every split id is a real sample', [...SPLIT.train, ...SPLIT.test].every((id) => gt[id]));
|
||||
for (const f of ['path', 'ssrf', 'authz']) ok(`shared family "${f}" appears in BOTH train and test`, trainF.has(f) && testF.has(f));
|
||||
ok('holdout is empty until CVE-Zero-v2 is curated', SPLIT.holdout.length === 0);
|
||||
ok('holdout is the curated 10-sample CVE-Zero-v2 split (real samples, disjoint from train/test)',
|
||||
SPLIT.holdout.length === 10 && SPLIT.holdout.every((id) => gt[id]) && !SPLIT.holdout.some((x) => SPLIT.train.includes(x) || SPLIT.test.includes(x)));
|
||||
console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}\n`);
|
||||
process.exitCode = fail === 0 ? 0 : 1;
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user