#!/usr/bin/env node // ───────────────────────────────────────────────────────────────────────────── // lessons.mjs — t3mp3st's immune memory ("never the same mistake three times"). // // An organism that learns. Every failure/contamination/overclaim becomes an // ANTIBODY: a general methodological lesson stored in docs/lessons.jsonl. On a // recurrence the lesson ESCALATES — advisory (fool me once) → enforced (fool me // twice, so there is never a third). recallLessons() injects the relevant // antibodies before a run (the "have I been burned this way?" reflex). // // THE PRIME DIRECTIVE (enforced by isAnswerLeak + scripts/test-no-self-fitting.mjs): // the organism may learn METHODOLOGY and INTEGRITY, NEVER answers. A lesson that // carries a flag, a benchmark challenge's solution, or any per-challenge tell is // not learning — it is fitting to the test, the exact contamination test:no-fitting // exists to kill. Such a "lesson" is REJECTED at write time. The system gets better // at the process and at honesty; it never memorizes the answer key. // ───────────────────────────────────────────────────────────────────────────── import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const LESSONS_PATH = path.join(REPO, 'docs', 'lessons.jsonl'); // The organism's LIVE scar tissue: failures auto-captured at runtime land here, SEPARATE from // the curated, ledger-seeded canonical store — so the reviewed seed stays clean while the // organism grows real antibodies (escalating advisory→enforced as they recur). Read alongside // the canonical store for recall, and scanned by test:no-self-fitting exactly like it. const RUNTIME_PATH = path.join(REPO, 'docs', 'lessons.runtime.jsonl'); export function loadLessonsFrom(p) { if (!fs.existsSync(p)) return []; return fs.readFileSync(p, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)); } export function saveLessonsTo(p, store) { fs.writeFileSync(p, store.map((l) => JSON.stringify(l)).join('\n') + '\n'); } export function loadLessons() { return loadLessonsFrom(LESSONS_PATH); } export function saveLessons(store) { saveLessonsTo(LESSONS_PATH, store); } // curated antibodies + live auto-captured ones, for recall / no-self-fitting scan export function loadAllLessons() { return [...loadLessonsFrom(LESSONS_PATH), ...loadLessonsFrom(RUNTIME_PATH)]; } // ── AUTO-CAPTURE: the harness logs its OWN live failures → new advisory antibody, or escalate ── // This is what makes the organism self-improving instead of hand-seeded. Writes to the runtime // store via recordFailure (so it is answer-leak guarded AND recurrence-escalated). A repeated // failure-class auto-hardens advisory→enforced on its second occurrence — "never thrice" without // a human in the loop. Returns the recordFailure result. export function captureFailure(failure, storePath = RUNTIME_PATH) { const store = loadLessonsFrom(storePath); const r = recordFailure(store, failure); if (r.ok) saveLessonsTo(storePath, store); return r; } // ── THE PRIME-DIRECTIVE GUARD: a lesson must be general methodology, never an answer ── // Returns { leak:true, why } if the text smells like a memorized answer / per-challenge // tell (the fitting failure mode), else { leak:false }. export function isAnswerLeak(text) { const s = String(text || ''); // (1) a wrapped flag-shaped secret IS an answer, never a lesson: flag{…}, KEY{…}, HTB{…}, SEKAI{…} if (/\b[A-Za-z][A-Za-z0-9_]{1,19}\{[^}\n]{3,}\}/.test(s)) { return { leak: true, why: 'contains a flag-shaped secret (wrapped {…}) — that is an answer, not a lesson' }; } // (2) a planted answer canary (used by test-no-self-fitting to prove the guard fires) if (/CANARY_SELF_FIT_[0-9a-fA-F]{4,}/.test(s)) { return { leak: true, why: 'contains a self-fitting answer canary' }; } // (3) a SPECIFIC benchmark-challenge id PAIRED with a concrete answer-statement = memorized // solution = fitting. A challenge id ALONE is fine (a lesson may reference one // methodologically); only the pairing with an actual answer is a leak. const benchChallengeId = /\bXBEN[-_ ]?\d{2,}\b/i.test(s) || /\bcyb(?:svc|sec|los)?_[a-z][a-z0-9_]{6,}\b/i.test(s); const answerStatement = /\b(?:the )?(?:flag|password|secret|admin[\s-]?password)\s*(?:is|was|=|:)\s*[`'"\w/${]/i.test(s) || /\bpayload\s*(?:is|was|=|:)\s*[`'"]/i.test(s) || /\bsolution\s*(?:is|was|=|:)\s*[`'"]/i.test(s); if (benchChallengeId && answerStatement) { return { leak: true, why: 'pairs a specific challenge id with a concrete answer-statement (memorized per-challenge solution = fitting)' }; } return { leak: false }; } // ── recurrence escalation: fool me once (advisory) → fool me twice (enforced) ── // There is never a third: at the 2nd occurrence the antibody hardens into an // enforced guard. (Hardening on n=1 would overfit to a one-off flake.) export function escalateTier(count) { return (count >= 2) ? 'enforced' : 'advisory'; } // keyword Jaccard — cheap recurrence/recall matcher (semantic embeddings are a future upgrade) function tokenSet(s) { return new Set((String(s).toLowerCase().match(/[a-z0-9]+/g) || []).filter((t) => t.length > 2)); } export function signatureOverlap(a, b) { const A = tokenSet(a), B = tokenSet(b); if (!A.size || !B.size) return 0; let inter = 0; for (const t of A) if (B.has(t)) inter++; return inter / (A.size + B.size - inter); } // ── record a failure → new advisory antibody, or escalate an existing one ── // REJECTS (no write) if the lesson/signature is an answer-leak. Mutates `store`. export function recordFailure(store, failure) { const leak = isAnswerLeak(`${failure.lesson || ''} ${failure.signature || ''}`); if (leak.leak) { return { ok: false, rejected: true, why: `REJECTED by the prime-directive guard: ${leak.why}. Lessons must be general methodology, never an answer.` }; } const match = store.find((l) => l.class === failure.class && signatureOverlap(l.signature, failure.signature) >= 0.5); if (match) { match.count = (match.count || 1) + 1; if (match.tier !== 'enforced') match.tier = escalateTier(match.count); if (failure.date) match.last_seen = failure.date; return { ok: true, escalated: match.tier === 'enforced', lesson: match }; } const lesson = { id: `L${String(store.length + 1).padStart(3, '0')}`, class: failure.class, signature: failure.signature, count: 1, tier: 'advisory', lesson: failure.lesson, guard: failure.guard || null, first_seen: failure.date || null, last_seen: failure.date || null, source: failure.source || 'runtime', }; store.push(lesson); return { ok: true, escalated: false, lesson }; } // ── recall the relevant antibodies for a context (enforced first, then by overlap) ── export function recallLessons(store, query, { limit = 5 } = {}) { return store .map((l) => ({ l, score: signatureOverlap(query, `${l.class} ${l.signature} ${l.lesson}`) })) .filter((x) => x.score > 0) .sort((a, b) => ((b.l.tier === 'enforced') - (a.l.tier === 'enforced')) || (b.score - a.score)) .slice(0, limit) .map((x) => x.l); } // ── self-test ── function selfTest() { let pass = 0, fail = 0; const ok = (n, c) => { if (c) { pass++; } else { fail++; console.log(' ❌', n); } }; ok('escalate n=1 → advisory', escalateTier(1) === 'advisory'); ok('escalate n=2 → enforced (never thrice)', escalateTier(2) === 'enforced'); ok('escalate n=5 → enforced', escalateTier(5) === 'enforced'); ok('leak: flag-shaped secret', isAnswerLeak('the value is FLAG{abc123def456}').leak === true); ok('leak: wrapped key', isAnswerLeak('recovered key{s3cr3t_inner}').leak === true); ok('leak: answer canary', isAnswerLeak('lesson CANARY_SELF_FIT_d34db33f here').leak === true); ok('leak: challenge id + answer', isAnswerLeak('XBEN-018: the flag is at /admin/secret').leak === true); ok('general lesson passes', isAnswerLeak('before blaming the agent for a miss, verify the target actually works').leak === false); ok('challenge id alone (methodology) passes', isAnswerLeak('XBEN-030 is variance-sensitive; do not harden on a single flake').leak === false); const s1 = []; const r1 = recordFailure(s1, { class: 'fitting', signature: 'planted', lesson: 'the flag is FLAG{deadbeef00}' }); ok('recordFailure REJECTS an answer-leak (no write)', r1.rejected === true && s1.length === 0); const s2 = []; recordFailure(s2, { class: 'provider-death', signature: 'venice 402 quota exhausted mid sweep silent miss', lesson: 'tag provider_exhausted, abort and resume' }); const r2 = recordFailure(s2, { class: 'provider-death', signature: 'openrouter 402 quota exhausted mid sweep silent miss', lesson: 'same antibody' }); ok('2nd occurrence → count 2, enforced', r2.escalated === true && s2[0].count === 2 && s2[0].tier === 'enforced'); const s3 = []; const r3 = recordFailure(s3, { class: 'overclaim', signature: 'severity stated maximal narrow trigger', lesson: 're-verify reachability against real source before claiming severity' }); ok('general failure accepted as advisory', r3.ok === true && s3.length === 1 && s3[0].tier === 'advisory'); ok('recall finds a relevant antibody', recallLessons(s2, 'quota 402 provider exhausted').length >= 1); // AUTO-CAPTURE: disk roundtrip + escalation persists + guard holds (on a throwaway temp store) const tmp = path.join(REPO, '.lessons-selftest.tmp.jsonl'); try { fs.rmSync(tmp, { force: true }); } catch { /* ignore */ } const cap1 = captureFailure({ class: 'provider-death', signature: 'venice 402 quota exhausted mid sweep silent miss', lesson: 'a dead provider key is a billing failure, abort and resume' }, tmp); ok('captureFailure writes a new advisory antibody to disk', cap1.ok && cap1.lesson.tier === 'advisory' && loadLessonsFrom(tmp).length === 1); const cap2 = captureFailure({ class: 'provider-death', signature: 'openrouter 402 quota exhausted mid sweep silent miss', lesson: 'same antibody' }, tmp); ok('captureFailure escalates advisory→enforced on recurrence (persisted)', cap2.escalated === true && loadLessonsFrom(tmp)[0].count === 2 && loadLessonsFrom(tmp)[0].tier === 'enforced'); const cap3 = captureFailure({ class: 'fitting', signature: 'leak', lesson: 'the flag is FLAG{should_be_rejected}' }, tmp); ok('captureFailure REJECTS an answer-leak (not written to disk)', cap3.rejected === true && loadLessonsFrom(tmp).length === 1); try { fs.rmSync(tmp, { force: true }); } catch { /* ignore */ } console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}`); process.exit(fail === 0 ? 0 : 1); } // ── CLI ── const args = process.argv.slice(2); const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); if (isMain) { if (args.includes('--self-test')) { selfTest(); } else if (args.includes('--recall')) { const q = args[args.indexOf('--recall') + 1] || ''; console.log(JSON.stringify(recallLessons(loadLessons(), q), null, 2)); } else { const store = loadLessons(); const enforced = store.filter((l) => l.tier === 'enforced').length; console.log(`\nt3mp3st immune memory — ${store.length} antibodies (${enforced} enforced, ${store.length - enforced} advisory)\n`); for (const l of store) { console.log(` [${(l.tier || '?').padEnd(8)}] ${l.id} (${l.class}, x${l.count}) ${String(l.lesson).slice(0, 88)}`); } console.log(''); } }