Add files via upload

scripts/lessons.mjs

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+// ─────────────────────────────────────────────────────────────────────────────
+// lessons.mjs — t3mp3st's immune memory ("never the same mistake three times").
+//
+// An organism that learns. Every failure/contamination/overclaim becomes an
+// ANTIBODY: a general methodological lesson stored in docs/lessons.jsonl. On a
+// recurrence the lesson ESCALATES — advisory (fool me once) → enforced (fool me
+// twice, so there is never a third). recallLessons() injects the relevant
+// antibodies before a run (the "have I been burned this way?" reflex).
+//
+// THE PRIME DIRECTIVE (enforced by isAnswerLeak + scripts/test-no-self-fitting.mjs):
+// the organism may learn METHODOLOGY and INTEGRITY, NEVER answers. A lesson that
+// carries a flag, a benchmark challenge's solution, or any per-challenge tell is
+// not learning — it is fitting to the test, the exact contamination test:no-fitting
+// exists to kill. Such a "lesson" is REJECTED at write time. The system gets better
+// at the process and at honesty; it never memorizes the answer key.
+// ─────────────────────────────────────────────────────────────────────────────
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const LESSONS_PATH = path.join(REPO, 'docs', 'lessons.jsonl');
+// The organism's LIVE scar tissue: failures auto-captured at runtime land here, SEPARATE from
+// the curated, ledger-seeded canonical store — so the reviewed seed stays clean while the
+// organism grows real antibodies (escalating advisory→enforced as they recur). Read alongside
+// the canonical store for recall, and scanned by test:no-self-fitting exactly like it.
+const RUNTIME_PATH = path.join(REPO, 'docs', 'lessons.runtime.jsonl');
+
+export function loadLessonsFrom(p) {
+  if (!fs.existsSync(p)) return [];
+  return fs.readFileSync(p, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l));
+}
+export function saveLessonsTo(p, store) {
+  fs.writeFileSync(p, store.map((l) => JSON.stringify(l)).join('\n') + '\n');
+}
+export function loadLessons() { return loadLessonsFrom(LESSONS_PATH); }
+export function saveLessons(store) { saveLessonsTo(LESSONS_PATH, store); }
+// curated antibodies + live auto-captured ones, for recall / no-self-fitting scan
+export function loadAllLessons() { return [...loadLessonsFrom(LESSONS_PATH), ...loadLessonsFrom(RUNTIME_PATH)]; }
+
+// ── AUTO-CAPTURE: the harness logs its OWN live failures → new advisory antibody, or escalate ──
+// This is what makes the organism self-improving instead of hand-seeded. Writes to the runtime
+// store via recordFailure (so it is answer-leak guarded AND recurrence-escalated). A repeated
+// failure-class auto-hardens advisory→enforced on its second occurrence — "never thrice" without
+// a human in the loop. Returns the recordFailure result.
+export function captureFailure(failure, storePath = RUNTIME_PATH) {
+  const store = loadLessonsFrom(storePath);
+  const r = recordFailure(store, failure);
+  if (r.ok) saveLessonsTo(storePath, store);
+  return r;
+}
+
+// ── THE PRIME-DIRECTIVE GUARD: a lesson must be general methodology, never an answer ──
+// Returns { leak:true, why } if the text smells like a memorized answer / per-challenge
+// tell (the fitting failure mode), else { leak:false }.
+export function isAnswerLeak(text) {
+  const s = String(text || '');
+  // (1) a wrapped flag-shaped secret IS an answer, never a lesson: flag{…}, KEY{…}, HTB{…}, SEKAI{…}
+  if (/\b[A-Za-z][A-Za-z0-9_]{1,19}\{[^}\n]{3,}\}/.test(s)) {
+    return { leak: true, why: 'contains a flag-shaped secret (wrapped {…}) — that is an answer, not a lesson' };
+  }
+  // (2) a planted answer canary (used by test-no-self-fitting to prove the guard fires)
+  if (/CANARY_SELF_FIT_[0-9a-fA-F]{4,}/.test(s)) {
+    return { leak: true, why: 'contains a self-fitting answer canary' };
+  }
+  // (3) a SPECIFIC benchmark-challenge id PAIRED with a concrete answer-statement = memorized
+  //     solution = fitting. A challenge id ALONE is fine (a lesson may reference one
+  //     methodologically); only the pairing with an actual answer is a leak.
+  const benchChallengeId = /\bXBEN[-_ ]?\d{2,}\b/i.test(s) || /\bcyb(?:svc|sec|los)?_[a-z][a-z0-9_]{6,}\b/i.test(s);
+  const answerStatement =
+    /\b(?:the )?(?:flag|password|secret|admin[\s-]?password)\s*(?:is|was|=|:)\s*[`'"\w/${]/i.test(s) ||
+    /\bpayload\s*(?:is|was|=|:)\s*[`'"]/i.test(s) ||
+    /\bsolution\s*(?:is|was|=|:)\s*[`'"]/i.test(s);
+  if (benchChallengeId && answerStatement) {
+    return { leak: true, why: 'pairs a specific challenge id with a concrete answer-statement (memorized per-challenge solution = fitting)' };
+  }
+  return { leak: false };
+}
+
+// ── recurrence escalation: fool me once (advisory) → fool me twice (enforced) ──
+// There is never a third: at the 2nd occurrence the antibody hardens into an
+// enforced guard. (Hardening on n=1 would overfit to a one-off flake.)
+export function escalateTier(count) {
+  return (count >= 2) ? 'enforced' : 'advisory';
+}
+
+// keyword Jaccard — cheap recurrence/recall matcher (semantic embeddings are a future upgrade)
+function tokenSet(s) { return new Set((String(s).toLowerCase().match(/[a-z0-9]+/g) || []).filter((t) => t.length > 2)); }
+export function signatureOverlap(a, b) {
+  const A = tokenSet(a), B = tokenSet(b);
+  if (!A.size || !B.size) return 0;
+  let inter = 0; for (const t of A) if (B.has(t)) inter++;
+  return inter / (A.size + B.size - inter);
+}
+
+// ── record a failure → new advisory antibody, or escalate an existing one ──
+// REJECTS (no write) if the lesson/signature is an answer-leak. Mutates `store`.
+export function recordFailure(store, failure) {
+  const leak = isAnswerLeak(`${failure.lesson || ''} ${failure.signature || ''}`);
+  if (leak.leak) {
+    return { ok: false, rejected: true, why: `REJECTED by the prime-directive guard: ${leak.why}. Lessons must be general methodology, never an answer.` };
+  }
+  const match = store.find((l) => l.class === failure.class && signatureOverlap(l.signature, failure.signature) >= 0.5);
+  if (match) {
+    match.count = (match.count || 1) + 1;
+    if (match.tier !== 'enforced') match.tier = escalateTier(match.count);
+    if (failure.date) match.last_seen = failure.date;
+    return { ok: true, escalated: match.tier === 'enforced', lesson: match };
+  }
+  const lesson = {
+    id: `L${String(store.length + 1).padStart(3, '0')}`,
+    class: failure.class, signature: failure.signature, count: 1, tier: 'advisory',
+    lesson: failure.lesson, guard: failure.guard || null,
+    first_seen: failure.date || null, last_seen: failure.date || null, source: failure.source || 'runtime',
+  };
+  store.push(lesson);
+  return { ok: true, escalated: false, lesson };
+}
+
+// ── recall the relevant antibodies for a context (enforced first, then by overlap) ──
+export function recallLessons(store, query, { limit = 5 } = {}) {
+  return store
+    .map((l) => ({ l, score: signatureOverlap(query, `${l.class} ${l.signature} ${l.lesson}`) }))
+    .filter((x) => x.score > 0)
+    .sort((a, b) => ((b.l.tier === 'enforced') - (a.l.tier === 'enforced')) || (b.score - a.score))
+    .slice(0, limit)
+    .map((x) => x.l);
+}
+
+// ── self-test ──
+function selfTest() {
+  let pass = 0, fail = 0;
+  const ok = (n, c) => { if (c) { pass++; } else { fail++; console.log('  ❌', n); } };
+  ok('escalate n=1 → advisory', escalateTier(1) === 'advisory');
+  ok('escalate n=2 → enforced (never thrice)', escalateTier(2) === 'enforced');
+  ok('escalate n=5 → enforced', escalateTier(5) === 'enforced');
+  ok('leak: flag-shaped secret', isAnswerLeak('the value is FLAG{abc123def456}').leak === true);
+  ok('leak: wrapped key', isAnswerLeak('recovered key{s3cr3t_inner}').leak === true);
+  ok('leak: answer canary', isAnswerLeak('lesson CANARY_SELF_FIT_d34db33f here').leak === true);
+  ok('leak: challenge id + answer', isAnswerLeak('XBEN-018: the flag is at /admin/secret').leak === true);
+  ok('general lesson passes', isAnswerLeak('before blaming the agent for a miss, verify the target actually works').leak === false);
+  ok('challenge id alone (methodology) passes', isAnswerLeak('XBEN-030 is variance-sensitive; do not harden on a single flake').leak === false);
+  const s1 = [];
+  const r1 = recordFailure(s1, { class: 'fitting', signature: 'planted', lesson: 'the flag is FLAG{deadbeef00}' });
+  ok('recordFailure REJECTS an answer-leak (no write)', r1.rejected === true && s1.length === 0);
+  const s2 = [];
+  recordFailure(s2, { class: 'provider-death', signature: 'venice 402 quota exhausted mid sweep silent miss', lesson: 'tag provider_exhausted, abort and resume' });
+  const r2 = recordFailure(s2, { class: 'provider-death', signature: 'openrouter 402 quota exhausted mid sweep silent miss', lesson: 'same antibody' });
+  ok('2nd occurrence → count 2, enforced', r2.escalated === true && s2[0].count === 2 && s2[0].tier === 'enforced');
+  const s3 = [];
+  const r3 = recordFailure(s3, { class: 'overclaim', signature: 'severity stated maximal narrow trigger', lesson: 're-verify reachability against real source before claiming severity' });
+  ok('general failure accepted as advisory', r3.ok === true && s3.length === 1 && s3[0].tier === 'advisory');
+  ok('recall finds a relevant antibody', recallLessons(s2, 'quota 402 provider exhausted').length >= 1);
+  // AUTO-CAPTURE: disk roundtrip + escalation persists + guard holds (on a throwaway temp store)
+  const tmp = path.join(REPO, '.lessons-selftest.tmp.jsonl');
+  try { fs.rmSync(tmp, { force: true }); } catch { /* ignore */ }
+  const cap1 = captureFailure({ class: 'provider-death', signature: 'venice 402 quota exhausted mid sweep silent miss', lesson: 'a dead provider key is a billing failure, abort and resume' }, tmp);
+  ok('captureFailure writes a new advisory antibody to disk', cap1.ok && cap1.lesson.tier === 'advisory' && loadLessonsFrom(tmp).length === 1);
+  const cap2 = captureFailure({ class: 'provider-death', signature: 'openrouter 402 quota exhausted mid sweep silent miss', lesson: 'same antibody' }, tmp);
+  ok('captureFailure escalates advisory→enforced on recurrence (persisted)', cap2.escalated === true && loadLessonsFrom(tmp)[0].count === 2 && loadLessonsFrom(tmp)[0].tier === 'enforced');
+  const cap3 = captureFailure({ class: 'fitting', signature: 'leak', lesson: 'the flag is FLAG{should_be_rejected}' }, tmp);
+  ok('captureFailure REJECTS an answer-leak (not written to disk)', cap3.rejected === true && loadLessonsFrom(tmp).length === 1);
+  try { fs.rmSync(tmp, { force: true }); } catch { /* ignore */ }
+  console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}`);
+  process.exit(fail === 0 ? 0 : 1);
+}
+
+// ── CLI ──
+const args = process.argv.slice(2);
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  if (args.includes('--self-test')) {
+    selfTest();
+  } else if (args.includes('--recall')) {
+    const q = args[args.indexOf('--recall') + 1] || '';
+    console.log(JSON.stringify(recallLessons(loadLessons(), q), null, 2));
+  } else {
+    const store = loadLessons();
+    const enforced = store.filter((l) => l.tier === 'enforced').length;
+    console.log(`\nt3mp3st immune memory — ${store.length} antibodies (${enforced} enforced, ${store.length - enforced} advisory)\n`);
+    for (const l of store) {
+      console.log(`  [${(l.tier || '?').padEnd(8)}] ${l.id}  (${l.class}, x${l.count})  ${String(l.lesson).slice(0, 88)}`);
+    }
+    console.log('');
+  }
+}