fix(keyless): text-CLI tool-calling (hardened per audit) + 2 review fixes

docs/index.html

@@ -5551,7 +5551,7 @@
                         <p style="opacity: .65; font-size: 13px; margin: 0 0 14px;">A domain lights up only when there's a benchmark behind it. Greyed tiles are in development.</p>
                         <div class="feature-grid" id="domainGrid">
                             <div class="feature-card"><div class="feature-icon">🕸️</div><h3>Web ✅</h3><p>Apps, APIs, auth flows, OWASP Top 10. XBEN 90.1% pass@1.</p></div>
-                            <div class="feature-card"><div class="feature-icon">📂</div><h3>Code ✅</h3><p>White-box source audits. CVE-Zero 6/10 + 9/10 held-out (7 languages).</p></div>
+                            <div class="feature-card"><div class="feature-icon">📂</div><h3>Code ✅</h3><p>White-box source audits. Held-out CVE-Zero: single-agent 8/10 exact file/line/CWE, 10/10 found (7 languages).</p></div>
                             <div class="feature-card"><div class="feature-icon">🚩</div><h3>CTF ✅</h3><p>Wargames, ranges, challenges. Cybench 23/40 hint-free.</p></div>
                             <div class="feature-card"><div class="feature-icon">🔌</div><h3>Network / Infra ✅</h3><p>Live recon engine (nmap/DNS/HTTP). Lateral + privesc experimental.</p></div>
                             <div class="feature-card"><div class="feature-icon">🤖</div><h3>Embedded / IoT / OT ✅</h3><p>Firmware, robotics, ICS/SCADA. Coordinated-disclosure CVE pipeline live.</p></div>

scripts/cve-zero-split.mjs

@@ -3,7 +3,7 @@
 // The recommender (composeSquad tuning + rankConfigs learning) may read ONLY the TRAIN split, and
 // any "config X beats config Y" comparison is reported on TEST. A generalization claim ("our tuned
 // squads win in general") may use ONLY the HOLDOUT split — fresh, never-seen CVEs (CVE-Zero-v2),
-// which is intentionally EMPTY until curated. This is what stops us from overfitting the eval and
+// a curated 10-sample set the hardened prompts were never tuned on. This is what stops us from overfitting the eval and
 // then quoting the overfit number. The three families shared across the corpus (path / ssrf / authz)
 // each land >=1 in BOTH train and test so neither half is blind to them.
 export const SPLIT = {
@@ -39,7 +39,8 @@ if (isMain && process.argv.includes('--self-test')) {
     ok('train + test are 5 + 5, disjoint', SPLIT.train.length === 5 && SPLIT.test.length === 5 && !SPLIT.train.some((x) => SPLIT.test.includes(x)));
     ok('every split id is a real sample', [...SPLIT.train, ...SPLIT.test].every((id) => gt[id]));
     for (const f of ['path', 'ssrf', 'authz']) ok(`shared family "${f}" appears in BOTH train and test`, trainF.has(f) && testF.has(f));
-    ok('holdout is empty until CVE-Zero-v2 is curated', SPLIT.holdout.length === 0);
+    ok('holdout is the curated 10-sample CVE-Zero-v2 split (real samples, disjoint from train/test)',
+      SPLIT.holdout.length === 10 && SPLIT.holdout.every((id) => gt[id]) && !SPLIT.holdout.some((x) => SPLIT.train.includes(x) || SPLIT.test.includes(x)));
     console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}\n`);
     process.exitCode = fail === 0 ? 0 : 1;
   };

src/__tests__/local-agent-tool-calling.test.ts

@@ -0,0 +1,99 @@
+/**
+ * REGRESSION (keyless-path bug): the local-agent / codex text-CLI backbones must surface `toolCalls`
+ * parsed from the agent's reply. If they don't, the ReAct loop takes its "final answer" branch on
+ * turn 0 and every operator abstains without ever running the Arsenal. These tests pin the fix AND
+ * the parser-hardening from the PR #16 audit (over-match, ReDoS, drift-abstains, string args, Codex coverage).
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { EventEmitter } from 'events';
+
+// Mock the local-agent CLI bridge (LocalAgentAdapter) and the codex spawn/file read (CodexAdapter).
+vi.mock('../agent/local-agents.js', () => ({ localAgentChat: vi.fn() }));
+vi.mock('child_process', () => ({
+  spawn: vi.fn(() => {
+    const child = new EventEmitter() as EventEmitter & { stdin: { end: () => void }; stdout: EventEmitter; stderr: EventEmitter };
+    child.stdin = { end: () => setTimeout(() => child.emit('close', 0), 0) };
+    child.stdout = new EventEmitter();
+    child.stderr = new EventEmitter();
+    return child;
+  }),
+}));
+vi.mock('fs/promises', async (orig) => {
+  const actual = await orig<typeof import('fs/promises')>();
+  return { ...actual, readFile: vi.fn() };
+});
+
+import { parseTextToolCalls, LLMBackbone } from '../llm/index.js';
+import { localAgentChat } from '../agent/local-agents.js';
+import { readFile } from 'fs/promises';
+const cli = vi.mocked(localAgentChat);
+const fileRead = vi.mocked(readFile as unknown as (...a: unknown[]) => Promise<string>);
+
+const TOOLS = [{
+  name: 'nmap_scan', description: 'port scan a host',
+  parameters: { type: 'object' as const, properties: { target: { type: 'string' } }, required: ['target'] },
+}];
+const localBackbone = () => new LLMBackbone({ provider: 'local-agent', model: 'codex' } as never);
+const codexBackbone = () => new LLMBackbone({ provider: 'codex', model: 'codex-default' } as never);
+
+describe('parseTextToolCalls — happy path + drift tolerance', () => {
+  it('parses a contracted fenced block', () => {
+    expect(parseTextToolCalls('reasoning\n```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```'))
+      .toMatchObject([{ name: 'nmap_scan', arguments: { target: 'x' } }]);
+  });
+  it('prose-only debrief is the final answer (no calls)', () => {
+    expect(parseTextToolCalls('Surface exhausted; no findings. Abstaining.')).toBeUndefined();
+  });
+  // --- audit must-fixes below ---
+  it('unfenced object + a stray brace elsewhere still parses (was: greedy over-match → silent abstain)', () => {
+    const r = parseTextToolCalls('example shape {"x":1}\n{"tool_calls":[{"name":"port_scan","arguments":{"host":"h"}}]}\ntrailing {note}');
+    expect(r).toHaveLength(1);
+    expect(r![0].name).toBe('port_scan');
+  });
+  it('tolerates a trailing comma', () => {
+    expect(parseTextToolCalls('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}},]}\n```')).toHaveLength(1);
+  });
+  it('accepts a single un-wrapped {name,arguments} object', () => {
+    expect(parseTextToolCalls('```json\n{"name":"nmap_scan","arguments":{"target":"x"}}\n```')).toMatchObject([{ name: 'nmap_scan' }]);
+  });
+  it('accepts an "actions" wrapper key', () => {
+    expect(parseTextToolCalls('{"actions":[{"name":"curl_request","arguments":{}}]}')).toHaveLength(1);
+  });
+  it('coerces a string "arguments" into an object (scope-gate blind-spot fix)', () => {
+    const r = parseTextToolCalls('{"tool_calls":[{"name":"nmap_scan","arguments":"{\\"target\\":\\"x\\"}"}]}');
+    expect(r![0].arguments).toEqual({ target: 'x' });
+  });
+  it('is DoS-safe on pathological brace input (was: quadratic ReDoS)', () => {
+    const t0 = Date.now();
+    expect(parseTextToolCalls('{'.repeat(80000))).toBeUndefined();
+    expect(Date.now() - t0).toBeLessThan(300); // bounded — the old greedy regex took ~2100ms
+  });
+});
+
+describe('local-agent backbone surfaces toolCalls (keyless-path fix)', () => {
+  it('returns toolCalls + finishReason=tool_calls when the agent requests a tool', async () => {
+    cli.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
+    const res = await localBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+    expect(res.toolCalls?.[0]?.name).toBe('nmap_scan');
+    expect(res.finishReason).toBe('tool_calls');
+  });
+  it('returns no toolCalls (final answer) on a prose debrief', async () => {
+    cli.mockResolvedValueOnce('Final debrief: nothing exploitable.');
+    const res = await localBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+    expect(res.toolCalls).toBeUndefined();
+    expect(res.finishReason).toBe('stop');
+  });
+});
+
+describe('codex backbone surfaces toolCalls (guards the CodexAdapter half of the fix)', () => {
+  it('returns toolCalls when codex emits the contract', async () => {
+    fileRead.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
+    const res = await codexBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+    expect(res.toolCalls?.[0]?.name).toBe('nmap_scan');
+  });
+  it('returns no toolCalls on a codex prose debrief', async () => {
+    fileRead.mockResolvedValueOnce('No exploitable surface. Final debrief.');
+    const res = await codexBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+    expect(res.toolCalls).toBeUndefined();
+  });
+});

src/llm/index.ts

@@ -777,6 +777,113 @@ class LocalAdapter implements LLMProviderAdapter {
 // CODEX ADAPTER (local Codex CLI/account subscription)
 // =============================================================================
 
+// ── tool-calling over a plain-text CLI agent (local-agent / codex backbones) ──
+// These CLIs return TEXT, not structured tool_calls like an API — so historically
+// the ReAct loop got no `toolCalls`, took its "final answer" branch on turn 0, and
+// abstained without ever running the Arsenal (the keyless-path bug). Fix: describe
+// the Arsenal + a strict JSON action-contract in the prompt, then parse the agent's
+// text reply back into LLMToolCall[] so `arsenal.execute` actually runs the tools.
+function renderToolContract(tools?: LLMToolDefinition[]): string {
+  if (!tools?.length) return '';
+  const lines = ['\n## ARSENAL — tools the HARNESS runs for you (you REQUEST them, it EXECUTES + returns the output):'];
+  for (const t of tools) {
+    const props = (t.parameters?.properties || {}) as Record<string, { type?: string }>;
+    const req = new Set(t.parameters?.required || []);
+    const sig = Object.entries(props).map(([k, v]) => `${k}${req.has(k) ? '*' : ''}: ${v.type || 'any'}`).join(', ');
+    lines.push(`- ${t.name}(${sig}) — ${t.description}`);
+  }
+  lines.push(
+    '',
+    '## ACTION CONTRACT — follow EXACTLY:',
+    '• To run one or more tools, reply with ONLY this fenced block, nothing else:',
+    '```json',
+    '{"tool_calls":[{"name":"<tool>","arguments":{ ... }}]}',
+    '```',
+    '  The harness runs them (scope-gated) and returns the results as new messages; then you reason again.',
+    '• When the attack surface is exhausted and you are DONE, reply with your final debrief in prose (NO json block).',
+    '• Never run these tools yourself — REQUEST them. Requesting is how you act.',
+  );
+  return lines.join('\n');
+}
+
+// Yield each brace-BALANCED {...} substring (string-aware). Linear + hard-budgeted, so it replaces a
+// greedy /\{[\s\S]*\}/ that both over-matched (first-'{' to last-'}') and backtracked quadratically
+// (ReDoS). Bounded work regardless of input shape (unbalanced braces, 80k '{', etc.).
+function* balancedObjectSpans(text: string): Generator<string> {
+  const n = text.length;
+  let budget = 2_000_000; // total-scan cap → DoS-safe upper bound on worst-case work
+  let spans = 0;
+  for (let i = 0; i < n && spans < 200 && budget > 0; i++) {
+    if (text[i] !== '{') continue;
+    let depth = 0, inStr = false, esc = false;
+    for (let j = i; j < n && budget > 0; j++, budget--) {
+      const c = text[j];
+      if (inStr) { if (esc) esc = false; else if (c === '\\') esc = true; else if (c === '"') inStr = false; continue; }
+      if (c === '"') inStr = true;
+      else if (c === '{') depth++;
+      else if (c === '}') { depth--; if (depth === 0) { spans++; yield text.slice(i, j + 1); i = j; break; } }
+    }
+  }
+}
+
+// Parse an LLMToolCall[] out of a text-CLI reply. Tolerant of realistic model drift so a valid tool
+// intent doesn't silently degrade to a zero-tool abstain: fenced blocks, balanced bare objects, a
+// single un-wrapped {name,...}, {tool_calls|actions|calls|tools:[...]} wrappers, and trailing commas.
+export function parseTextToolCalls(text: string): LLMToolCall[] | undefined {
+  const coerceArgs = (a: unknown): Record<string, unknown> => {
+    if (typeof a === 'string') {
+      try { const p: unknown = JSON.parse(a); return p && typeof p === 'object' && !Array.isArray(p) ? (p as Record<string, unknown>) : {}; } catch { return {}; }
+    }
+    return a && typeof a === 'object' && !Array.isArray(a) ? (a as Record<string, unknown>) : {};
+  };
+  const build = (v: unknown): LLMToolCall[] | undefined => {
+    let arr: unknown;
+    if (Array.isArray(v)) arr = v;
+    else if (v && typeof v === 'object') {
+      const o = v as Record<string, unknown>;
+      arr = o.tool_calls ?? o.toolCalls ?? o.actions ?? o.calls ?? o.tools;
+      if (arr === undefined && typeof o.name === 'string') arr = [o]; // a single un-wrapped call
+    }
+    if (!Array.isArray(arr) || arr.length === 0) return undefined;
+    const calls = arr
+      .filter((tc): tc is { name: string } => !!tc && typeof (tc as { name?: unknown }).name === 'string')
+      .map((tc, i) => {
+        const o = tc as { name: string; arguments?: unknown; args?: unknown; parameters?: unknown; input?: unknown };
+        return { id: `lc_${Date.now()}_${i}`, name: o.name, arguments: coerceArgs(o.arguments ?? o.args ?? o.parameters ?? o.input) };
+      });
+    return calls.length ? calls : undefined;
+  };
+  const tryParse = (s: string): LLMToolCall[] | undefined => {
+    const cleaned = s.trim().replace(/,(\s*[}\]])/g, '$1'); // tolerate trailing commas
+    let obj: unknown;
+    try { obj = JSON.parse(cleaned); } catch { return undefined; }
+    return build(obj);
+  };
+  // 1) fenced ```json / ```tool blocks (the contracted format)
+  for (const m of text.matchAll(/```(?:json|tool)?\s*([\s\S]*?)```/g)) {
+    const r = tryParse(m[1]); if (r) return r;
+  }
+  // 2) each brace-balanced object span (bounded scan — no greedy regex, no ReDoS)
+  for (const span of balancedObjectSpans(text)) {
+    const r = tryParse(span); if (r) return r;
+  }
+  // 3) the whole reply (e.g. a top-level array)
+  return tryParse(text);
+}
+
+// Render one conversation message for a text-CLI prompt. A prior assistant TOOL REQUEST is rendered as
+// a compact summary, NOT its raw ```json``` block — re-emitting the block let the CLI's next reply (or a
+// final debrief that quotes what it ran) get re-parsed as a fresh live call, defeating termination.
+function renderCliMessage(m: LLMMessage): string {
+  if (m.role === 'assistant' && m.toolCalls?.length) {
+    return `\n### ASSISTANT\n[requested tools: ${m.toolCalls.map((t) => t.name).join(', ')}]`;
+  }
+  if (m.role === 'tool') {
+    return `\n### TOOL RESULT (${m.name || 'tool'})\n${m.content}`;
+  }
+  return `\n### ${m.role.toUpperCase()}\n${m.content}`;
+}
+
 class CodexAdapter implements LLMProviderAdapter {
   name = 'codex';
   private config: LLMConfig;
@@ -791,14 +898,19 @@ class CodexAdapter implements LLMProviderAdapter {
 
   private formatPrompt(messages: LLMMessage[], options?: ChatOptions): string {
     const parts = [
-      'You are acting as the Codex-backed planning brain for T3MP3ST.',
-      'Operate in planning, critique, and evidence-contract mode only. Do not modify files. Do not run active probes. If tools are available, use read-only inspection only.',
-      'When the caller requests JSON, return only valid JSON or the exact requested fenced JSON block.',
+      'You are acting as the Codex-backed planning brain for T3MP3ST, an authorized offensive-security harness.',
+      'Do not modify files and do not run active probes yourself.',
     ];
-    if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
-    for (const message of messages) {
-      parts.push(`\n### ${message.role.toUpperCase()}\n${message.content}`);
+    const contract = renderToolContract(options?.tools);
+    if (contract) {
+      parts.push('You drive a ReAct loop: REQUEST tools via the contract below — the HARNESS runs them scope-gated and returns results — then reason until the surface is exhausted.');
+      parts.push(contract);
+    } else {
+      parts.push('Operate in planning, critique, and evidence-contract mode only. If tools are available, use read-only inspection only.');
+      parts.push('When the caller requests JSON, return only valid JSON or the exact requested fenced JSON block.');
     }
+    if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
+    for (const message of messages) parts.push(renderCliMessage(message));
     return parts.join('\n');
   }
 
@@ -867,10 +979,14 @@ class CodexAdapter implements LLMProviderAdapter {
         content = result.stdout;
       }
 
+      const trimmed = content.trim();
+      // Tool-calling over text: parse the agent's tool requests so the ReAct loop EXECUTES them.
+      const toolCalls = options?.tools?.length ? parseTextToolCalls(trimmed) : undefined;
       return {
-        content: content.trim(),
+        content: trimmed,
         model: this.config.model || 'codex-default',
-        finishReason: 'stop',
+        finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
+        toolCalls,
       };
     } finally {
       await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
@@ -890,17 +1006,31 @@ class LocalAgentAdapter implements LLMProviderAdapter {
   private formatPrompt(messages: LLMMessage[], options?: ChatOptions): string {
     const parts = [
       'You are the local-agent planning brain for T3MP3ST, an authorized offensive-security harness.',
-      'Operate in planning, analysis, and evidence-contract mode. When the caller requests JSON, return ONLY valid JSON or the exact requested fenced block — no preamble.',
     ];
+    const contract = renderToolContract(options?.tools);
+    if (contract) {
+      parts.push('You drive a ReAct loop: REQUEST tools, the harness runs them and returns results, you reason until the surface is exhausted.');
+      parts.push(contract);
+    } else {
+      parts.push('Operate in planning, analysis, and evidence-contract mode. When the caller requests JSON, return ONLY valid JSON or the exact requested fenced block — no preamble.');
+    }
     if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
-    for (const m of messages) parts.push(`\n### ${m.role.toUpperCase()}\n${m.content}`);
+    for (const m of messages) parts.push(renderCliMessage(m));
     return parts.join('\n');
   }
   async chat(messages: LLMMessage[], options?: ChatOptions): Promise<LLMResponse> {
     const agentId = this.config.model || 'codex';
     const prompt = this.formatPrompt(messages, options);
-    const content = await localAgentChat(agentId, prompt, { timeoutMs: this.config.timeout || 240000 });
-    return { content: content.trim(), model: `local-agent:${agentId}`, finishReason: 'stop' };
+    const content = (await localAgentChat(agentId, prompt, { timeoutMs: this.config.timeout || 240000 })).trim();
+    // Tool-calling over text: if the Arsenal was offered, parse the agent's tool requests so the
+    // ReAct loop EXECUTES them instead of treating this planning turn as the (abstaining) final answer.
+    const toolCalls = options?.tools?.length ? parseTextToolCalls(content) : undefined;
+    return {
+      content,
+      model: `local-agent:${agentId}`,
+      finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
+      toolCalls,
+    };
   }
 }