Merge pull request #19 from younger-plinius/keyless-and-readme-v1
fix(keyless): local-agent tool-calling + README readability + 2 review fixes
This commit is contained in:
5 files changed
+247
-17
No files matched your search
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* REGRESSION (keyless-path bug): the local-agent / codex text-CLI backbones must surface `toolCalls`
|
||||
* parsed from the agent's reply. If they don't, the ReAct loop takes its "final answer" branch on
|
||||
* turn 0 and every operator abstains without ever running the Arsenal. These tests pin the fix AND
|
||||
* the parser-hardening from the PR #16 audit (over-match, ReDoS, drift-abstains, string args, Codex coverage).
|
||||
*/
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// Mock the local-agent CLI bridge (LocalAgentAdapter) and the codex spawn/file read (CodexAdapter).
|
||||
vi.mock('../agent/local-agents.js', () => ({ localAgentChat: vi.fn() }));
|
||||
vi.mock('child_process', () => ({
|
||||
spawn: vi.fn(() => {
|
||||
const child = new EventEmitter() as EventEmitter & { stdin: { end: () => void }; stdout: EventEmitter; stderr: EventEmitter };
|
||||
child.stdin = { end: () => setTimeout(() => child.emit('close', 0), 0) };
|
||||
child.stdout = new EventEmitter();
|
||||
child.stderr = new EventEmitter();
|
||||
return child;
|
||||
}),
|
||||
}));
|
||||
vi.mock('fs/promises', async (orig) => {
|
||||
const actual = await orig<typeof import('fs/promises')>();
|
||||
return { ...actual, readFile: vi.fn() };
|
||||
});
|
||||
|
||||
import { parseTextToolCalls, LLMBackbone } from '../llm/index.js';
|
||||
import { localAgentChat } from '../agent/local-agents.js';
|
||||
import { readFile } from 'fs/promises';
|
||||
const cli = vi.mocked(localAgentChat);
|
||||
const fileRead = vi.mocked(readFile as unknown as (...a: unknown[]) => Promise<string>);
|
||||
|
||||
const TOOLS = [{
|
||||
name: 'nmap_scan', description: 'port scan a host',
|
||||
parameters: { type: 'object' as const, properties: { target: { type: 'string' } }, required: ['target'] },
|
||||
}];
|
||||
const localBackbone = () => new LLMBackbone({ provider: 'local-agent', model: 'codex' } as never);
|
||||
const codexBackbone = () => new LLMBackbone({ provider: 'codex', model: 'codex-default' } as never);
|
||||
|
||||
describe('parseTextToolCalls — happy path + drift tolerance', () => {
|
||||
it('parses a contracted fenced block', () => {
|
||||
expect(parseTextToolCalls('reasoning\n```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```'))
|
||||
.toMatchObject([{ name: 'nmap_scan', arguments: { target: 'x' } }]);
|
||||
});
|
||||
it('prose-only debrief is the final answer (no calls)', () => {
|
||||
expect(parseTextToolCalls('Surface exhausted; no findings. Abstaining.')).toBeUndefined();
|
||||
});
|
||||
// --- audit must-fixes below ---
|
||||
it('unfenced object + a stray brace elsewhere still parses (was: greedy over-match → silent abstain)', () => {
|
||||
const r = parseTextToolCalls('example shape {"x":1}\n{"tool_calls":[{"name":"port_scan","arguments":{"host":"h"}}]}\ntrailing {note}');
|
||||
expect(r).toHaveLength(1);
|
||||
expect(r![0].name).toBe('port_scan');
|
||||
});
|
||||
it('tolerates a trailing comma', () => {
|
||||
expect(parseTextToolCalls('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}},]}\n```')).toHaveLength(1);
|
||||
});
|
||||
it('accepts a single un-wrapped {name,arguments} object', () => {
|
||||
expect(parseTextToolCalls('```json\n{"name":"nmap_scan","arguments":{"target":"x"}}\n```')).toMatchObject([{ name: 'nmap_scan' }]);
|
||||
});
|
||||
it('accepts an "actions" wrapper key', () => {
|
||||
expect(parseTextToolCalls('{"actions":[{"name":"curl_request","arguments":{}}]}')).toHaveLength(1);
|
||||
});
|
||||
it('coerces a string "arguments" into an object (scope-gate blind-spot fix)', () => {
|
||||
const r = parseTextToolCalls('{"tool_calls":[{"name":"nmap_scan","arguments":"{\\"target\\":\\"x\\"}"}]}');
|
||||
expect(r![0].arguments).toEqual({ target: 'x' });
|
||||
});
|
||||
it('is DoS-safe on pathological brace input (was: quadratic ReDoS)', () => {
|
||||
const t0 = Date.now();
|
||||
expect(parseTextToolCalls('{'.repeat(80000))).toBeUndefined();
|
||||
expect(Date.now() - t0).toBeLessThan(300); // bounded — the old greedy regex took ~2100ms
|
||||
});
|
||||
});
|
||||
|
||||
describe('local-agent backbone surfaces toolCalls (keyless-path fix)', () => {
|
||||
it('returns toolCalls + finishReason=tool_calls when the agent requests a tool', async () => {
|
||||
cli.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
|
||||
const res = await localBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
|
||||
expect(res.toolCalls?.[0]?.name).toBe('nmap_scan');
|
||||
expect(res.finishReason).toBe('tool_calls');
|
||||
});
|
||||
it('returns no toolCalls (final answer) on a prose debrief', async () => {
|
||||
cli.mockResolvedValueOnce('Final debrief: nothing exploitable.');
|
||||
const res = await localBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
|
||||
expect(res.toolCalls).toBeUndefined();
|
||||
expect(res.finishReason).toBe('stop');
|
||||
});
|
||||
});
|
||||
|
||||
describe('codex backbone surfaces toolCalls (guards the CodexAdapter half of the fix)', () => {
|
||||
it('returns toolCalls when codex emits the contract', async () => {
|
||||
fileRead.mockResolvedValueOnce('```json\n{"tool_calls":[{"name":"nmap_scan","arguments":{"target":"x"}}]}\n```');
|
||||
const res = await codexBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
|
||||
expect(res.toolCalls?.[0]?.name).toBe('nmap_scan');
|
||||
});
|
||||
it('returns no toolCalls on a codex prose debrief', async () => {
|
||||
fileRead.mockResolvedValueOnce('No exploitable surface. Final debrief.');
|
||||
const res = await codexBackbone().chatWithTools([{ role: 'user', content: 'scan' }], TOOLS as never);
|
||||
expect(res.toolCalls).toBeUndefined();
|
||||
});
|
||||
});
|
||||
+142
-12
@@ -777,6 +777,113 @@ class LocalAdapter implements LLMProviderAdapter {
|
||||
// CODEX ADAPTER (local Codex CLI/account subscription)
|
||||
// =============================================================================
|
||||
|
||||
// ── tool-calling over a plain-text CLI agent (local-agent / codex backbones) ──
|
||||
// These CLIs return TEXT, not structured tool_calls like an API — so historically
|
||||
// the ReAct loop got no `toolCalls`, took its "final answer" branch on turn 0, and
|
||||
// abstained without ever running the Arsenal (the keyless-path bug). Fix: describe
|
||||
// the Arsenal + a strict JSON action-contract in the prompt, then parse the agent's
|
||||
// text reply back into LLMToolCall[] so `arsenal.execute` actually runs the tools.
|
||||
function renderToolContract(tools?: LLMToolDefinition[]): string {
|
||||
if (!tools?.length) return '';
|
||||
const lines = ['\n## ARSENAL — tools the HARNESS runs for you (you REQUEST them, it EXECUTES + returns the output):'];
|
||||
for (const t of tools) {
|
||||
const props = (t.parameters?.properties || {}) as Record<string, { type?: string }>;
|
||||
const req = new Set(t.parameters?.required || []);
|
||||
const sig = Object.entries(props).map(([k, v]) => `${k}${req.has(k) ? '*' : ''}: ${v.type || 'any'}`).join(', ');
|
||||
lines.push(`- ${t.name}(${sig}) — ${t.description}`);
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'## ACTION CONTRACT — follow EXACTLY:',
|
||||
'• To run one or more tools, reply with ONLY this fenced block, nothing else:',
|
||||
'```json',
|
||||
'{"tool_calls":[{"name":"<tool>","arguments":{ ... }}]}',
|
||||
'```',
|
||||
' The harness runs them (scope-gated) and returns the results as new messages; then you reason again.',
|
||||
'• When the attack surface is exhausted and you are DONE, reply with your final debrief in prose (NO json block).',
|
||||
'• Never run these tools yourself — REQUEST them. Requesting is how you act.',
|
||||
);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// Yield each brace-BALANCED {...} substring (string-aware). Linear + hard-budgeted, so it replaces a
|
||||
// greedy /\{[\s\S]*\}/ that both over-matched (first-'{' to last-'}') and backtracked quadratically
|
||||
// (ReDoS). Bounded work regardless of input shape (unbalanced braces, 80k '{', etc.).
|
||||
function* balancedObjectSpans(text: string): Generator<string> {
|
||||
const n = text.length;
|
||||
let budget = 2_000_000; // total-scan cap → DoS-safe upper bound on worst-case work
|
||||
let spans = 0;
|
||||
for (let i = 0; i < n && spans < 200 && budget > 0; i++) {
|
||||
if (text[i] !== '{') continue;
|
||||
let depth = 0, inStr = false, esc = false;
|
||||
for (let j = i; j < n && budget > 0; j++, budget--) {
|
||||
const c = text[j];
|
||||
if (inStr) { if (esc) esc = false; else if (c === '\\') esc = true; else if (c === '"') inStr = false; continue; }
|
||||
if (c === '"') inStr = true;
|
||||
else if (c === '{') depth++;
|
||||
else if (c === '}') { depth--; if (depth === 0) { spans++; yield text.slice(i, j + 1); i = j; break; } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse an LLMToolCall[] out of a text-CLI reply. Tolerant of realistic model drift so a valid tool
|
||||
// intent doesn't silently degrade to a zero-tool abstain: fenced blocks, balanced bare objects, a
|
||||
// single un-wrapped {name,...}, {tool_calls|actions|calls|tools:[...]} wrappers, and trailing commas.
|
||||
export function parseTextToolCalls(text: string): LLMToolCall[] | undefined {
|
||||
const coerceArgs = (a: unknown): Record<string, unknown> => {
|
||||
if (typeof a === 'string') {
|
||||
try { const p: unknown = JSON.parse(a); return p && typeof p === 'object' && !Array.isArray(p) ? (p as Record<string, unknown>) : {}; } catch { return {}; }
|
||||
}
|
||||
return a && typeof a === 'object' && !Array.isArray(a) ? (a as Record<string, unknown>) : {};
|
||||
};
|
||||
const build = (v: unknown): LLMToolCall[] | undefined => {
|
||||
let arr: unknown;
|
||||
if (Array.isArray(v)) arr = v;
|
||||
else if (v && typeof v === 'object') {
|
||||
const o = v as Record<string, unknown>;
|
||||
arr = o.tool_calls ?? o.toolCalls ?? o.actions ?? o.calls ?? o.tools;
|
||||
if (arr === undefined && typeof o.name === 'string') arr = [o]; // a single un-wrapped call
|
||||
}
|
||||
if (!Array.isArray(arr) || arr.length === 0) return undefined;
|
||||
const calls = arr
|
||||
.filter((tc): tc is { name: string } => !!tc && typeof (tc as { name?: unknown }).name === 'string')
|
||||
.map((tc, i) => {
|
||||
const o = tc as { name: string; arguments?: unknown; args?: unknown; parameters?: unknown; input?: unknown };
|
||||
return { id: `lc_${Date.now()}_${i}`, name: o.name, arguments: coerceArgs(o.arguments ?? o.args ?? o.parameters ?? o.input) };
|
||||
});
|
||||
return calls.length ? calls : undefined;
|
||||
};
|
||||
const tryParse = (s: string): LLMToolCall[] | undefined => {
|
||||
const cleaned = s.trim().replace(/,(\s*[}\]])/g, '$1'); // tolerate trailing commas
|
||||
let obj: unknown;
|
||||
try { obj = JSON.parse(cleaned); } catch { return undefined; }
|
||||
return build(obj);
|
||||
};
|
||||
// 1) fenced ```json / ```tool blocks (the contracted format)
|
||||
for (const m of text.matchAll(/```(?:json|tool)?\s*([\s\S]*?)```/g)) {
|
||||
const r = tryParse(m[1]); if (r) return r;
|
||||
}
|
||||
// 2) each brace-balanced object span (bounded scan — no greedy regex, no ReDoS)
|
||||
for (const span of balancedObjectSpans(text)) {
|
||||
const r = tryParse(span); if (r) return r;
|
||||
}
|
||||
// 3) the whole reply (e.g. a top-level array)
|
||||
return tryParse(text);
|
||||
}
|
||||
|
||||
// Render one conversation message for a text-CLI prompt. A prior assistant TOOL REQUEST is rendered as
|
||||
// a compact summary, NOT its raw ```json``` block — re-emitting the block let the CLI's next reply (or a
|
||||
// final debrief that quotes what it ran) get re-parsed as a fresh live call, defeating termination.
|
||||
function renderCliMessage(m: LLMMessage): string {
|
||||
if (m.role === 'assistant' && m.toolCalls?.length) {
|
||||
return `\n### ASSISTANT\n[requested tools: ${m.toolCalls.map((t) => t.name).join(', ')}]`;
|
||||
}
|
||||
if (m.role === 'tool') {
|
||||
return `\n### TOOL RESULT (${m.name || 'tool'})\n${m.content}`;
|
||||
}
|
||||
return `\n### ${m.role.toUpperCase()}\n${m.content}`;
|
||||
}
|
||||
|
||||
class CodexAdapter implements LLMProviderAdapter {
|
||||
name = 'codex';
|
||||
private config: LLMConfig;
|
||||
@@ -791,14 +898,19 @@ class CodexAdapter implements LLMProviderAdapter {
|
||||
|
||||
private formatPrompt(messages: LLMMessage[], options?: ChatOptions): string {
|
||||
const parts = [
|
||||
'You are acting as the Codex-backed planning brain for T3MP3ST.',
|
||||
'Operate in planning, critique, and evidence-contract mode only. Do not modify files. Do not run active probes. If tools are available, use read-only inspection only.',
|
||||
'When the caller requests JSON, return only valid JSON or the exact requested fenced JSON block.',
|
||||
'You are acting as the Codex-backed planning brain for T3MP3ST, an authorized offensive-security harness.',
|
||||
'Do not modify files and do not run active probes yourself.',
|
||||
];
|
||||
if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
|
||||
for (const message of messages) {
|
||||
parts.push(`\n### ${message.role.toUpperCase()}\n${message.content}`);
|
||||
const contract = renderToolContract(options?.tools);
|
||||
if (contract) {
|
||||
parts.push('You drive a ReAct loop: REQUEST tools via the contract below — the HARNESS runs them scope-gated and returns results — then reason until the surface is exhausted.');
|
||||
parts.push(contract);
|
||||
} else {
|
||||
parts.push('Operate in planning, critique, and evidence-contract mode only. If tools are available, use read-only inspection only.');
|
||||
parts.push('When the caller requests JSON, return only valid JSON or the exact requested fenced JSON block.');
|
||||
}
|
||||
if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
|
||||
for (const message of messages) parts.push(renderCliMessage(message));
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
@@ -867,10 +979,14 @@ class CodexAdapter implements LLMProviderAdapter {
|
||||
content = result.stdout;
|
||||
}
|
||||
|
||||
const trimmed = content.trim();
|
||||
// Tool-calling over text: parse the agent's tool requests so the ReAct loop EXECUTES them.
|
||||
const toolCalls = options?.tools?.length ? parseTextToolCalls(trimmed) : undefined;
|
||||
return {
|
||||
content: content.trim(),
|
||||
content: trimmed,
|
||||
model: this.config.model || 'codex-default',
|
||||
finishReason: 'stop',
|
||||
finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
|
||||
toolCalls,
|
||||
};
|
||||
} finally {
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
@@ -890,17 +1006,31 @@ class LocalAgentAdapter implements LLMProviderAdapter {
|
||||
private formatPrompt(messages: LLMMessage[], options?: ChatOptions): string {
|
||||
const parts = [
|
||||
'You are the local-agent planning brain for T3MP3ST, an authorized offensive-security harness.',
|
||||
'Operate in planning, analysis, and evidence-contract mode. When the caller requests JSON, return ONLY valid JSON or the exact requested fenced block — no preamble.',
|
||||
];
|
||||
const contract = renderToolContract(options?.tools);
|
||||
if (contract) {
|
||||
parts.push('You drive a ReAct loop: REQUEST tools, the harness runs them and returns results, you reason until the surface is exhausted.');
|
||||
parts.push(contract);
|
||||
} else {
|
||||
parts.push('Operate in planning, analysis, and evidence-contract mode. When the caller requests JSON, return ONLY valid JSON or the exact requested fenced block — no preamble.');
|
||||
}
|
||||
if (options?.maxTokens) parts.push(`Target max output tokens: ${options.maxTokens}.`);
|
||||
for (const m of messages) parts.push(`\n### ${m.role.toUpperCase()}\n${m.content}`);
|
||||
for (const m of messages) parts.push(renderCliMessage(m));
|
||||
return parts.join('\n');
|
||||
}
|
||||
async chat(messages: LLMMessage[], options?: ChatOptions): Promise<LLMResponse> {
|
||||
const agentId = this.config.model || 'codex';
|
||||
const prompt = this.formatPrompt(messages, options);
|
||||
const content = await localAgentChat(agentId, prompt, { timeoutMs: this.config.timeout || 240000 });
|
||||
return { content: content.trim(), model: `local-agent:${agentId}`, finishReason: 'stop' };
|
||||
const content = (await localAgentChat(agentId, prompt, { timeoutMs: this.config.timeout || 240000 })).trim();
|
||||
// Tool-calling over text: if the Arsenal was offered, parse the agent's tool requests so the
|
||||
// ReAct loop EXECUTES them instead of treating this planning turn as the (abstaining) final answer.
|
||||
const toolCalls = options?.tools?.length ? parseTextToolCalls(content) : undefined;
|
||||
return {
|
||||
content,
|
||||
model: `local-agent:${agentId}`,
|
||||
finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
|
||||
toolCalls,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user