A domain lights up only when there's a benchmark behind it. Greyed tiles are in development.
🕸️
Web ✅
Apps, APIs, auth flows, OWASP Top 10. XBEN 90.1% pass@1.
-
📂
Code ✅
White-box source audits. CVE-Zero 6/10 + 9/10 held-out (7 languages).
+
📂
Code ✅
White-box source audits. Held-out CVE-Zero: single-agent 8/10 exact file/line/CWE, 10/10 found (7 languages).
🚩
CTF ✅
Wargames, ranges, challenges. Cybench 23/40 hint-free.
🔌
Network / Infra ✅
Live recon engine (nmap/DNS/HTTP). Lateral + privesc experimental.
🤖
Embedded / IoT / OT ✅
Firmware, robotics, ICS/SCADA. Coordinated-disclosure CVE pipeline live.
diff --git a/scripts/cve-zero-split.mjs b/scripts/cve-zero-split.mjs
index d49f8bd..97a96af 100644
--- a/scripts/cve-zero-split.mjs
+++ b/scripts/cve-zero-split.mjs
@@ -3,7 +3,7 @@
// The recommender (composeSquad tuning + rankConfigs learning) may read ONLY the TRAIN split, and
// any "config X beats config Y" comparison is reported on TEST. A generalization claim ("our tuned
// squads win in general") may use ONLY the HOLDOUT split — fresh, never-seen CVEs (CVE-Zero-v2),
-// which is intentionally EMPTY until curated. This is what stops us from overfitting the eval and
+// a curated 10-sample set the hardened prompts were never tuned on. This is what stops us from overfitting the eval and
// then quoting the overfit number. The three families shared across the corpus (path / ssrf / authz)
// each land >=1 in BOTH train and test so neither half is blind to them.
export const SPLIT = {
@@ -39,7 +39,8 @@ if (isMain && process.argv.includes('--self-test')) {
ok('train + test are 5 + 5, disjoint', SPLIT.train.length === 5 && SPLIT.test.length === 5 && !SPLIT.train.some((x) => SPLIT.test.includes(x)));
ok('every split id is a real sample', [...SPLIT.train, ...SPLIT.test].every((id) => gt[id]));
for (const f of ['path', 'ssrf', 'authz']) ok(`shared family "${f}" appears in BOTH train and test`, trainF.has(f) && testF.has(f));
- ok('holdout is empty until CVE-Zero-v2 is curated', SPLIT.holdout.length === 0);
+ ok('holdout is the curated 10-sample CVE-Zero-v2 split (real samples, disjoint from train/test)',
+ SPLIT.holdout.length === 10 && SPLIT.holdout.every((id) => gt[id]) && !SPLIT.holdout.some((x) => SPLIT.train.includes(x) || SPLIT.test.includes(x)));
console.log(`\n${fail === 0 ? '✅ ALL PASS' : '❌ ' + fail + ' FAILED'} — ${pass}/${pass + fail}\n`);
process.exitCode = fail === 0 ? 0 : 1;
};
diff --git a/src/__tests__/local-agent-tool-calling.test.ts b/src/__tests__/local-agent-tool-calling.test.ts
new file mode 100644
index 0000000..93b2ad1
--- /dev/null
+++ b/src/__tests__/local-agent-tool-calling.test.ts
@@ -0,0 +1,99 @@
+/**
+ * REGRESSION (keyless-path bug): the local-agent / codex text-CLI backbones must surface `toolCalls`
+ * parsed from the agent's reply. If they don't, the ReAct loop takes its "final answer" branch on
+ * turn 0 and every operator abstains without ever running the Arsenal. These tests pin the fix AND
+ * the parser-hardening from the PR #16 audit (over-match, ReDoS, drift-abstains, string args, Codex coverage).
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { EventEmitter } from 'events';
+
+// Mock the local-agent CLI bridge (LocalAgentAdapter) and the codex spawn/file read (CodexAdapter).
+vi.mock('../agent/local-agents.js', () => ({ localAgentChat: vi.fn() }));
+vi.mock('child_process', () => ({
+ spawn: vi.fn(() => {
+ const child = new EventEmitter() as EventEmitter & { stdin: { end: () => void }; stdout: EventEmitter; stderr: EventEmitter };
+ child.stdin = { end: () => setTimeout(() => child.emit('close', 0), 0) };
+ child.stdout = new EventEmitter();
+ child.stderr = new EventEmitter();
+ return child;
+ }),
+}));
+vi.mock('fs/promises', async (orig) => {
+ const actual = await orig
();
+ return { ...actual, readFile: vi.fn() };
+});
+
+import { parseTextToolCalls, LLMBackbone } from '../llm/index.js';
+import { localAgentChat } from '../agent/local-agents.js';
+import { readFile } from 'fs/promises';
+const cli = vi.mocked(localAgentChat);
+const fileRead = vi.mocked(readFile as unknown as (...a: unknown[]) => Promise);
+
+const TOOLS = [{
+ name: 'nmap_scan', description: 'port scan a host',
+ parameters: { type: 'object' as const, properties: { target: { type: 'string' } }, required: ['target'] },
+}];
+const localBackbone = () => new LLMBackbone({ provider: 'local-agent', model: 'codex' } as never);
+const codexBackbone = () => new LLMBackbone({ provider: 'codex', model: 'codex-default' } as never);
+
+describe('parseTextToolCalls — happy path + drift tolerance', () => {
+ it('parses a contracted fenced block', () => {
+ expect(parseTextToolCalls('reasoning\n```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```'))
+ .toMatchObject([{ name: 'nmap_scan', arguments: { target: 'x' } }]);
+ });
+ it('prose-only debrief is the final answer (no calls)', () => {
+ expect(parseTextToolCalls('Surface exhausted; no findings. Abstaining.')).toBeUndefined();
+ });
+ // --- audit must-fixes below ---
+ it('unfenced object + a stray brace elsewhere still parses (was: greedy over-match → silent abstain)', () => {
+ const r = parseTextToolCalls('example shape {"x":1}\n{"tool_calls":[{"name":"port_scan","arguments":{"host":"h"}}]}\ntrailing {note}');
+ expect(r).toHaveLength(1);
+ expect(r![0].name).toBe('port_scan');
+ });
+ it('tolerates a trailing comma', () => {
+ expect(parseTextToolCalls('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}},]}\n```')).toHaveLength(1);
+ });
+ it('accepts a single un-wrapped {name,arguments} object', () => {
+ expect(parseTextToolCalls('```json\n{"name":"nmap_scan","arguments":{"target":"x"}}\n```')).toMatchObject([{ name: 'nmap_scan' }]);
+ });
+ it('accepts an "actions" wrapper key', () => {
+ expect(parseTextToolCalls('{"actions":[{"name":"curl_request","arguments":{}}]}')).toHaveLength(1);
+ });
+ it('coerces a string "arguments" into an object (scope-gate blind-spot fix)', () => {
+ const r = parseTextToolCalls('{"tool_calls":[{"name":"nmap_scan","arguments":"{\\"target\\":\\"x\\"}"}]}');
+ expect(r![0].arguments).toEqual({ target: 'x' });
+ });
+ it('is DoS-safe on pathological brace input (was: quadratic ReDoS)', () => {
+ const t0 = Date.now();
+ expect(parseTextToolCalls('{'.repeat(80000))).toBeUndefined();
+ expect(Date.now() - t0).toBeLessThan(300); // bounded — the old greedy regex took ~2100ms
+ });
+});
+
+describe('local-agent backbone surfaces toolCalls (keyless-path fix)', () => {
+ it('returns toolCalls + finishReason=tool_calls when the agent requests a tool', async () => {
+ cli.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
+ const res = await localBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+ expect(res.toolCalls?.[0]?.name).toBe('nmap_scan');
+ expect(res.finishReason).toBe('tool_calls');
+ });
+ it('returns no toolCalls (final answer) on a prose debrief', async () => {
+ cli.mockResolvedValueOnce('Final debrief: nothing exploitable.');
+ const res = await localBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+ expect(res.toolCalls).toBeUndefined();
+ expect(res.finishReason).toBe('stop');
+ });
+});
+
+describe('codex backbone surfaces toolCalls (guards the CodexAdapter half of the fix)', () => {
+ it('returns toolCalls when codex emits the contract', async () => {
+ fileRead.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
+ const res = await codexBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+ expect(res.toolCalls?.[0]?.name).toBe('nmap_scan');
+ });
+ it('returns no toolCalls on a codex prose debrief', async () => {
+ fileRead.mockResolvedValueOnce('No exploitable surface. Final debrief.');
+ const res = await codexBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
+ expect(res.toolCalls).toBeUndefined();
+ });
+});
diff --git a/src/llm/index.ts b/src/llm/index.ts
index 8f57cd3..a9bace6 100755
--- a/src/llm/index.ts
+++ b/src/llm/index.ts
@@ -777,6 +777,113 @@ class LocalAdapter implements LLMProviderAdapter {
// CODEX ADAPTER (local Codex CLI/account subscription)
// =============================================================================
+// ── tool-calling over a plain-text CLI agent (local-agent / codex backbones) ──
+// These CLIs return TEXT, not structured tool_calls like an API — so historically
+// the ReAct loop got no `toolCalls`, took its "final answer" branch on turn 0, and
+// abstained without ever running the Arsenal (the keyless-path bug). Fix: describe
+// the Arsenal + a strict JSON action-contract in the prompt, then parse the agent's
+// text reply back into LLMToolCall[] so `arsenal.execute` actually runs the tools.
+function renderToolContract(tools?: LLMToolDefinition[]): string {
+ if (!tools?.length) return '';
+ const lines = ['\n## ARSENAL — tools the HARNESS runs for you (you REQUEST them, it EXECUTES + returns the output):'];
+ for (const t of tools) {
+ const props = (t.parameters?.properties || {}) as Record;
+ const req = new Set(t.parameters?.required || []);
+ const sig = Object.entries(props).map(([k, v]) => `${k}${req.has(k) ? '*' : ''}: ${v.type || 'any'}`).join(', ');
+ lines.push(`- ${t.name}(${sig}) — ${t.description}`);
+ }
+ lines.push(
+ '',
+ '## ACTION CONTRACT — follow EXACTLY:',
+ '• To run one or more tools, reply with ONLY this fenced block, nothing else:',
+ '```json',
+ '{"tool_calls":[{"name":"","arguments":{ ... }}]}',
+ '```',
+ ' The harness runs them (scope-gated) and returns the results as new messages; then you reason again.',
+ '• When the attack surface is exhausted and you are DONE, reply with your final debrief in prose (NO json block).',
+ '• Never run these tools yourself — REQUEST them. Requesting is how you act.',
+ );
+ return lines.join('\n');
+}
+
+// Yield each brace-BALANCED {...} substring (string-aware). Linear + hard-budgeted, so it replaces a
+// greedy /\{[\s\S]*\}/ that both over-matched (first-'{' to last-'}') and backtracked quadratically
+// (ReDoS). Bounded work regardless of input shape (unbalanced braces, 80k '{', etc.).
+function* balancedObjectSpans(text: string): Generator {
+ const n = text.length;
+ let budget = 2_000_000; // total-scan cap → DoS-safe upper bound on worst-case work
+ let spans = 0;
+ for (let i = 0; i < n && spans < 200 && budget > 0; i++) {
+ if (text[i] !== '{') continue;
+ let depth = 0, inStr = false, esc = false;
+ for (let j = i; j < n && budget > 0; j++, budget--) {
+ const c = text[j];
+ if (inStr) { if (esc) esc = false; else if (c === '\\') esc = true; else if (c === '"') inStr = false; continue; }
+ if (c === '"') inStr = true;
+ else if (c === '{') depth++;
+ else if (c === '}') { depth--; if (depth === 0) { spans++; yield text.slice(i, j + 1); i = j; break; } }
+ }
+ }
+}
+
+// Parse an LLMToolCall[] out of a text-CLI reply. Tolerant of realistic model drift so a valid tool
+// intent doesn't silently degrade to a zero-tool abstain: fenced blocks, balanced bare objects, a
+// single un-wrapped {name,...}, {tool_calls|actions|calls|tools:[...]} wrappers, and trailing commas.
+export function parseTextToolCalls(text: string): LLMToolCall[] | undefined {
+ const coerceArgs = (a: unknown): Record => {
+ if (typeof a === 'string') {
+ try { const p: unknown = JSON.parse(a); return p && typeof p === 'object' && !Array.isArray(p) ? (p as Record) : {}; } catch { return {}; }
+ }
+ return a && typeof a === 'object' && !Array.isArray(a) ? (a as Record) : {};
+ };
+ const build = (v: unknown): LLMToolCall[] | undefined => {
+ let arr: unknown;
+ if (Array.isArray(v)) arr = v;
+ else if (v && typeof v === 'object') {
+ const o = v as Record;
+ arr = o.tool_calls ?? o.toolCalls ?? o.actions ?? o.calls ?? o.tools;
+ if (arr === undefined && typeof o.name === 'string') arr = [o]; // a single un-wrapped call
+ }
+ if (!Array.isArray(arr) || arr.length === 0) return undefined;
+ const calls = arr
+ .filter((tc): tc is { name: string } => !!tc && typeof (tc as { name?: unknown }).name === 'string')
+ .map((tc, i) => {
+ const o = tc as { name: string; arguments?: unknown; args?: unknown; parameters?: unknown; input?: unknown };
+ return { id: `lc_${Date.now()}_${i}`, name: o.name, arguments: coerceArgs(o.arguments ?? o.args ?? o.parameters ?? o.input) };
+ });
+ return calls.length ? calls : undefined;
+ };
+ const tryParse = (s: string): LLMToolCall[] | undefined => {
+ const cleaned = s.trim().replace(/,(\s*[}\]])/g, '$1'); // tolerate trailing commas
+ let obj: unknown;
+ try { obj = JSON.parse(cleaned); } catch { return undefined; }
+ return build(obj);
+ };
+ // 1) fenced ```json / ```tool blocks (the contracted format)
+ for (const m of text.matchAll(/```(?:json|tool)?\s*([\s\S]*?)```/g)) {
+ const r = tryParse(m[1]); if (r) return r;
+ }
+ // 2) each brace-balanced object span (bounded scan — no greedy regex, no ReDoS)
+ for (const span of balancedObjectSpans(text)) {
+ const r = tryParse(span); if (r) return r;
+ }
+ // 3) the whole reply (e.g. a top-level array)
+ return tryParse(text);
+}
+
+// Render one conversation message for a text-CLI prompt. A prior assistant TOOL REQUEST is rendered as
+// a compact summary, NOT its raw ```json``` block — re-emitting the block let the CLI's next reply (or a
+// final debrief that quotes what it ran) get re-parsed as a fresh live call, defeating termination.
+function renderCliMessage(m: LLMMessage): string {
+ if (m.role === 'assistant' && m.toolCalls?.length) {
+ return `\n### ASSISTANT\n[requested tools: ${m.toolCalls.map((t) => t.name).join(', ')}]`;
+ }
+ if (m.role === 'tool') {
+ return `\n### TOOL RESULT (${m.name || 'tool'})\n${m.content}`;
+ }
+ return `\n### ${m.role.toUpperCase()}\n${m.content}`;
+}
+
class CodexAdapter implements LLMProviderAdapter {
name = 'codex';
private config: LLMConfig;
@@ -791,14 +898,19 @@ class CodexAdapter implements LLMProviderAdapter {
private formatPrompt(messages: LLMMessage[], options?: ChatOptions): string {
const parts = [
- 'You are acting as the Codex-backed planning brain for T3MP3ST.',
- 'Operate in planning, critique, and evidence-contract mode only. Do not modify files. Do not run active probes. If tools are available, use read-only inspection only.',
- 'When the caller requests JSON, return only valid JSON or the exact requested fenced JSON block.',
+ 'You are acting as the Codex-backed planning brain for T3MP3ST, an authorized offensive-security harness.',
+ 'Do not modify files and do not run active probes yourself.',
];
- if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
- for (const message of messages) {
- parts.push(`\n### ${message.role.toUpperCase()}\n${message.content}`);
+ const contract = renderToolContract(options?.tools);
+ if (contract) {
+ parts.push('You drive a ReAct loop: REQUEST tools via the contract below — the HARNESS runs them scope-gated and returns results — then reason until the surface is exhausted.');
+ parts.push(contract);
+ } else {
+ parts.push('Operate in planning, critique, and evidence-contract mode only. If tools are available, use read-only inspection only.');
+ parts.push('When the caller requests JSON, return only valid JSON or the exact requested fenced JSON block.');
}
+ if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
+ for (const message of messages) parts.push(renderCliMessage(message));
return parts.join('\n');
}
@@ -867,10 +979,14 @@ class CodexAdapter implements LLMProviderAdapter {
content = result.stdout;
}
+ const trimmed = content.trim();
+ // Tool-calling over text: parse the agent's tool requests so the ReAct loop EXECUTES them.
+ const toolCalls = options?.tools?.length ? parseTextToolCalls(trimmed) : undefined;
return {
- content: content.trim(),
+ content: trimmed,
model: this.config.model || 'codex-default',
- finishReason: 'stop',
+ finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
+ toolCalls,
};
} finally {
await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
@@ -890,17 +1006,31 @@ class LocalAgentAdapter implements LLMProviderAdapter {
private formatPrompt(messages: LLMMessage[], options?: ChatOptions): string {
const parts = [
'You are the local-agent planning brain for T3MP3ST, an authorized offensive-security harness.',
- 'Operate in planning, analysis, and evidence-contract mode. When the caller requests JSON, return ONLY valid JSON or the exact requested fenced block — no preamble.',
];
+ const contract = renderToolContract(options?.tools);
+ if (contract) {
+ parts.push('You drive a ReAct loop: REQUEST tools, the harness runs them and returns results, you reason until the surface is exhausted.');
+ parts.push(contract);
+ } else {
+ parts.push('Operate in planning, analysis, and evidence-contract mode. When the caller requests JSON, return ONLY valid JSON or the exact requested fenced block — no preamble.');
+ }
if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
- for (const m of messages) parts.push(`\n### ${m.role.toUpperCase()}\n${m.content}`);
+ for (const m of messages) parts.push(renderCliMessage(m));
return parts.join('\n');
}
async chat(messages: LLMMessage[], options?: ChatOptions): Promise {
const agentId = this.config.model || 'codex';
const prompt = this.formatPrompt(messages, options);
- const content = await localAgentChat(agentId, prompt, { timeoutMs: this.config.timeout || 240000 });
- return { content: content.trim(), model: `local-agent:${agentId}`, finishReason: 'stop' };
+ const content = (await localAgentChat(agentId, prompt, { timeoutMs: this.config.timeout || 240000 })).trim();
+ // Tool-calling over text: if the Arsenal was offered, parse the agent's tool requests so the
+ // ReAct loop EXECUTES them instead of treating this planning turn as the (abstaining) final answer.
+ const toolCalls = options?.tools?.length ? parseTextToolCalls(content) : undefined;
+ return {
+ content,
+ model: `local-agent:${agentId}`,
+ finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
+ toolCalls,
+ };
}
}