From df08627cb9e9e1f994abd7eea34f9da0d972bc15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ronal=20Steven=20S=C3=A1nchez=20Sierra?= <88067203+sronaal@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:45:52 -0500 Subject: [PATCH] feat: enhance function calling with richer schemas, validation, parallel execution, and Ollama native support (#51) * feat: add richer JSON Schema types and AJV validation utility - Extended ToolParameter with enum, items, properties, additionalProperties (max depth 3) - Added ToolError class with ToolErrorCategory enum - Added ToolValidationError interface - Created AJV-based validateToolArgs() with lazy loading and schema depth enforcement - 57 new tests covering types and validation * feat: add validation gate, parallel execution, and structured error feedback - Arsenal.execute() validates args against JSON Schema before handler execution - Tool execution errors categorized (scope_denied, tool_not_found, timeout, execution_error, validation_error) - ToolError import fix in agent (was missing, causing silent ReferenceError) - Semaphore-based parallel tool execution with configurable concurrency (default 5) - Results maintain original call order regardless of completion order - Structured error feedback to LLM with category only (no raw error messages) - Scope/approval gates checked per-call before execution - 35 new tests covering validation, parallel execution, and error feedback * feat: add Ollama native function calling with probe + cache + fallback - Added nativeTools config option to LLMConfig - LocalAdapter probes model support for native tools on first call - Probe result cached per model name to avoid re-probing - Native tools formatted as Ollama /api/chat tools parameter - Tool calls parsed from Ollama's tool_calls response format - Falls back to text-based renderToolContract when native not supported - OpenAI-compatible wire format also supported (for vLLM, LM Studio) - 10 new tests covering probe, cache, fallback, and config overrides * fix(tools): harden validation and native fallback --------- Co-authored-by: sronaal Co-authored-by: Joseph Magly <1159087+jmagly@users.noreply.github.com> --- package-lock.json | 1 + package.json | 1 + src/__tests__/agent-error-feedback.test.ts | 332 ++++++++++++++ src/__tests__/agent-parallel.test.ts | 403 +++++++++++++++++ src/__tests__/arsenal-validation.test.ts | 351 +++++++++++++++ src/__tests__/ollama-native-tools.test.ts | 346 ++++++++++++++ src/__tests__/tool-error.test.ts | 138 ++++++ src/__tests__/tool-parameter-types.test.ts | 237 ++++++++++ src/__tests__/validation.test.ts | 497 +++++++++++++++++++++ src/agent/index.ts | 130 +++++- src/arsenal/index.ts | 74 +-- src/llm/index.ts | 168 ++++++- src/types/index.ts | 54 +++ src/validation/index.ts | 350 +++++++++++++++ 14 files changed, 3019 insertions(+), 63 deletions(-) create mode 100644 src/__tests__/agent-error-feedback.test.ts create mode 100644 src/__tests__/agent-parallel.test.ts create mode 100644 src/__tests__/arsenal-validation.test.ts create mode 100644 src/__tests__/ollama-native-tools.test.ts create mode 100644 src/__tests__/tool-error.test.ts create mode 100644 src/__tests__/tool-parameter-types.test.ts create mode 100644 src/__tests__/validation.test.ts create mode 100644 src/validation/index.ts diff --git a/package-lock.json b/package-lock.json index fc6c6ff..e70caee 100755 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "AGPL-3.0-or-later", "dependencies": { "@modelcontextprotocol/sdk": "^1.25.1", + "ajv": "^8.20.0", "boxen": "^7.1.1", "chalk": "^5.3.0", "cli-table3": "^0.6.3", diff --git a/package.json b/package.json index 71410fe..875f572 100755 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.25.1", + "ajv": "^8.20.0", "boxen": "^7.1.1", "chalk": "^5.3.0", "cli-table3": "^0.6.3", diff --git a/src/__tests__/agent-error-feedback.test.ts b/src/__tests__/agent-error-feedback.test.ts new file mode 100644 index 0000000..8e65dfc --- /dev/null +++ b/src/__tests__/agent-error-feedback.test.ts @@ -0,0 +1,332 @@ +/** + * Agent Error Feedback Tests + * + * Tests for structured error feedback to the LLM after failed tool executions. + * Ensures the LLM receives categorized errors without raw internals. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { createAgentLoop } from '../agent/index.js'; +import { Arsenal } from '../arsenal/index.js'; +import type { LLMBackbone } from '../llm/index.js'; +import type { LLMResponse, LLMToolCall } from '../types/index.js'; +import type { Task } from '../types/index.js'; +import { ToolError, ToolErrorCategory } from '../types/index.js'; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function makeToolCall(id: string, name: string, args: Record): LLMToolCall { + return { id, name, arguments: args }; +} + +function makeLLMResponse(toolCalls: LLMToolCall[], content?: string): LLMResponse { + return { + content: content || '', + model: 'test-model', + toolCalls, + usage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }, + }; +} + +function makeTask(): Task { + return { + id: 'task-1', + missionId: 'mission-1', + name: 'Test Task', + description: 'A test task', + phase: 'reconnaissance' as any, + operatorType: 'recon', + status: 'in_progress', + priority: 5, + dependencies: [], + createdAt: Date.now(), + }; +} + +function createMockLLM(responses: LLMResponse[]): LLMBackbone { + let callIndex = 0; + return { + getProvider: vi.fn().mockReturnValue('mock'), + chat: vi.fn().mockImplementation(async () => { + const response = responses[callIndex] || responses[responses.length - 1]; + callIndex++; + return response; + }), + chatWithTools: vi.fn().mockImplementation(async () => { + const response = responses[callIndex] || responses[responses.length - 1]; + callIndex++; + return response; + }), + } as unknown as LLMBackbone; +} + +// ============================================================================= +// Tests +// ============================================================================= + +describe('Agent Error Feedback to LLM', () => { + let arsenal: Arsenal; + + beforeEach(() => { + arsenal = new Arsenal(); + }); + + // ── 1. Failed tool call adds error message to conversation ──────────────────── + + describe('error message added to conversation', () => { + it('should add tool error result to messages when tool fails', async () => { + arsenal.register({ + name: 'failing_tool', + description: 'Always fails', + category: 'test', + parameters: [], + handler: async () => { + throw new Error('Something went wrong'); + }, + }); + + const toolCalls = [makeToolCall('c1', 'failing_tool', {})]; + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + await agent.run(makeTask(), 'Execute tool.'); + + // The LLM should have received a tool result message + const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls; + // First call: system + user prompt + // After tool execution: assistant + tool result messages should be in the context + // Second call: should include the error feedback + expect(chatWithToolsCalls.length).toBeGreaterThanOrEqual(2); + + // Check the messages passed to the second chatWithTools call + const secondCallMessages = chatWithToolsCalls[1][0]; + const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool'); + expect(toolMessages.length).toBeGreaterThanOrEqual(1); + + // The tool message should contain error information + const errorMsg = toolMessages[0].content; + expect(errorMsg).toBeDefined(); + }); + }); + + // ── 2. Error message contains category, not raw error ───────────────────────── + + describe('error category in feedback', () => { + it('should include error category in tool result message', async () => { + arsenal.register({ + name: 'crashing_tool', + description: 'Crashes', + category: 'test', + parameters: [], + handler: async () => { + throw new ToolError(ToolErrorCategory.ExecutionError, 'Internal failure'); + }, + }); + + const toolCalls = [makeToolCall('c1', 'crashing_tool', {})]; + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + await agent.run(makeTask(), 'Execute.'); + + // Get the messages from the second LLM call + const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls; + const secondCallMessages = chatWithToolsCalls[1][0]; + const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool'); + + expect(toolMessages.length).toBeGreaterThanOrEqual(1); + + // Parse the error content — it should be JSON with category + const content = toolMessages[0].content; + // The content should contain the error category + expect(content).toContain('execution_error'); + }); + + it('should not expose raw error message in tool feedback', async () => { + arsenal.register({ + name: 'leaky_tool', + description: 'Leaks internals', + category: 'test', + parameters: [], + handler: async () => { + throw new Error('SECRET_db_password=abc123'); + }, + }); + + const toolCalls = [makeToolCall('c1', 'leaky_tool', {})]; + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + await agent.run(makeTask(), 'Execute.'); + + const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls; + const secondCallMessages = chatWithToolsCalls[1][0]; + const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool'); + + expect(toolMessages.length).toBeGreaterThanOrEqual(1); + const content = toolMessages[0].content; + // Raw error with secrets should NOT appear in the feedback + expect(content).not.toContain('SECRET_db_password'); + expect(content).not.toContain('abc123'); + }); + }); + + // ── 3. LLM receives error feedback and can retry ────────────────────────────── + + describe('LLM retry after error', () => { + it('should allow LLM to retry after receiving error feedback', async () => { + let callCount = 0; + arsenal.register({ + name: 'retry_tool', + description: 'Fails first, succeeds second', + category: 'test', + parameters: [ + { name: 'attempt', type: 'string', description: 'Attempt', required: true }, + ], + handler: async (ctx) => { + callCount++; + if (ctx.parameters.attempt === 'first') { + throw new Error('First attempt failed'); + } + return { success: true, output: 'Second attempt succeeded' }; + }, + }); + + // First call: tool fails. Second call: LLM retries with different args. Third call: done. + const llm = createMockLLM([ + makeLLMResponse([makeToolCall('c1', 'retry_tool', { attempt: 'first' })]), + makeLLMResponse([makeToolCall('c2', 'retry_tool', { attempt: 'second' })]), + makeLLMResponse([], 'Retry succeeded.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'Try the tool.'); + + expect(result.success).toBe(true); + // Both calls should have been made + expect(callCount).toBe(2); + + // Should have tool call steps for both attempts + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(2); + expect(toolSteps[0].toolResult?.success).toBe(false); + expect(toolSteps[1].toolResult?.success).toBe(true); + }); + }); + + // ── 4. Multiple failures produce separate error messages ────────────────────── + + describe('multiple failures', () => { + it('should produce separate error messages for each failed tool call', async () => { + arsenal.register({ + name: 'fail_a', + description: 'Fails A', + category: 'test', + parameters: [], + handler: async () => { throw new Error('Error A'); }, + }); + + arsenal.register({ + name: 'fail_b', + description: 'Fails B', + category: 'test', + parameters: [], + handler: async () => { throw new Error('Error B'); }, + }); + + const toolCalls = [ + makeToolCall('c1', 'fail_a', {}), + makeToolCall('c2', 'fail_b', {}), + ]; + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Both failed.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + await agent.run(makeTask(), 'Execute both.'); + + // Check that both error messages were added to the conversation + const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls; + const secondCallMessages = chatWithToolsCalls[1][0]; + const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool'); + + // Should have 2 tool messages (one per failed call) + expect(toolMessages).toHaveLength(2); + + // Each should have a unique toolCallId + const toolCallIds = toolMessages.map((m: any) => m.toolCallId); + expect(toolCallIds[0]).not.toBe(toolCallIds[1]); + }); + + it('should have toolCallId matching the original tool call', async () => { + arsenal.register({ + name: 'fail_tool', + description: 'Fails', + category: 'test', + parameters: [], + handler: async () => { throw new Error('Fail'); }, + }); + + const toolCall = makeToolCall('unique-call-id', 'fail_tool', {}); + const llm = createMockLLM([ + makeLLMResponse([toolCall]), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + await agent.run(makeTask(), 'Execute.'); + + const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls; + const secondCallMessages = chatWithToolsCalls[1][0]; + const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool'); + + expect(toolMessages).toHaveLength(1); + expect(toolMessages[0].toolCallId).toBe('unique-call-id'); + }); + }); + + // ── 5. Error feedback structure ─────────────────────────────────────────────── + + describe('error feedback structure', () => { + it('should include error flag in structured feedback', async () => { + arsenal.register({ + name: 'structured_fail', + description: 'Structured failure', + category: 'test', + parameters: [], + handler: async () => { throw new Error('Test error'); }, + }); + + const toolCalls = [makeToolCall('c1', 'structured_fail', {})]; + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + await agent.run(makeTask(), 'Execute.'); + + const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls; + const secondCallMessages = chatWithToolsCalls[1][0]; + const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool'); + + expect(toolMessages).toHaveLength(1); + const content = toolMessages[0].content; + // Should contain the error category + expect(content).toContain('execution_error'); + // Should contain the tool name + expect(content).toContain('structured_fail'); + }); + }); +}); diff --git a/src/__tests__/agent-parallel.test.ts b/src/__tests__/agent-parallel.test.ts new file mode 100644 index 0000000..5dce1a4 --- /dev/null +++ b/src/__tests__/agent-parallel.test.ts @@ -0,0 +1,403 @@ +/** + * Agent Parallel Execution Tests + * + * Tests for concurrent tool execution in the AgentLoop — semaphore-based + * parallel execution of independent tool calls from a single LLM response. + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { createAgentLoop } from '../agent/index.js'; +import { Arsenal } from '../arsenal/index.js'; +import type { LLMBackbone } from '../llm/index.js'; +import type { LLMResponse, LLMToolCall } from '../types/index.js'; +import type { Task } from '../types/index.js'; + +// ─── Helpers ────────────────────────────────────────────────────────────────── + +function makeToolCall(id: string, name: string, args: Record): LLMToolCall { + return { id, name, arguments: args }; +} + +function makeLLMResponse(toolCalls: LLMToolCall[], content?: string): LLMResponse { + return { + content: content || '', + model: 'test-model', + toolCalls, + usage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 }, + }; +} + +function makeTask(): Task { + return { + id: 'task-1', + missionId: 'mission-1', + name: 'Test Task', + description: 'A test task', + phase: 'reconnaissance' as any, + operatorType: 'recon', + status: 'in_progress', + priority: 5, + dependencies: [], + createdAt: Date.now(), + }; +} + +function createMockLLM(responses: LLMResponse[]): LLMBackbone { + let callIndex = 0; + return { + getProvider: vi.fn().mockReturnValue('mock'), + chat: vi.fn().mockImplementation(async () => { + const response = responses[callIndex] || responses[responses.length - 1]; + callIndex++; + return response; + }), + chatWithTools: vi.fn().mockImplementation(async () => { + const response = responses[callIndex] || responses[responses.length - 1]; + callIndex++; + return response; + }), + } as unknown as LLMBackbone; +} + +// ============================================================================= +// Tests +// ============================================================================= + +describe('AgentLoop Parallel Execution', () => { + let arsenal: Arsenal; + + beforeEach(() => { + arsenal = new Arsenal(); + }); + + // ── 1. Single tool call still works (sequential path) ───────────────────────── + + describe('single tool call', () => { + it('should execute a single tool call normally', async () => { + arsenal.register({ + name: 'dns_lookup', + description: 'DNS lookup', + category: 'recon', + parameters: [ + { name: 'domain', type: 'string', description: 'Domain', required: true }, + ], + handler: async (ctx) => ({ + success: true, + output: `Resolved ${ctx.parameters.domain}`, + }), + }); + + const toolCall = makeToolCall('call-1', 'dns_lookup', { domain: 'example.com' }); + const llm = createMockLLM([ + makeLLMResponse([toolCall]), + makeLLMResponse([], 'Done scanning.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'You are a security scanner.'); + + expect(result.success).toBe(true); + expect(result.steps.length).toBeGreaterThan(0); + // Should have tool_call step and final reasoning + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(1); + expect(toolSteps[0].toolName).toBe('dns_lookup'); + expect(toolSteps[0].toolResult?.success).toBe(true); + }); + }); + + // ── 2. Multiple tool calls execute concurrently ──────────────────────────────── + + describe('concurrent execution', () => { + it('should execute multiple tool calls from same response concurrently', async () => { + const executionOrder: string[] = []; + const executionStart: number[] = []; + + // Register tools that track execution order and timing + for (let i = 1; i <= 3; i++) { + const name = `tool_${i}`; + arsenal.register({ + name, + description: `Tool ${i}`, + category: 'test', + parameters: [ + { name: 'value', type: 'string', description: 'Value', required: true }, + ], + handler: async () => { + executionStart.push(Date.now()); + executionOrder.push(name); + // Small delay to simulate work + await new Promise(r => setTimeout(r, 10)); + return { success: true, output: `Result from ${name}` }; + }, + }); + } + + const toolCalls = [ + makeToolCall('c1', 'tool_1', { value: 'a' }), + makeToolCall('c2', 'tool_2', { value: 'b' }), + makeToolCall('c3', 'tool_3', { value: 'c' }), + ]; + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'All done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'Execute all tools.'); + + expect(result.success).toBe(true); + + // All 3 tools should have been called + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(3); + + // All should have succeeded + for (const step of toolSteps) { + expect(step.toolResult?.success).toBe(true); + } + + // If concurrent, the total time should be less than 3 * 10ms (sequential) + // Allow some margin — at least they should all complete + expect(executionOrder).toHaveLength(3); + }); + }); + + // ── 3. Results maintain original order ───────────────────────────────────────── + + describe('result ordering', () => { + it('should maintain tool call order in results regardless of completion order', async () => { + // Register tools with different delays to force out-of-order completion + arsenal.register({ + name: 'fast_tool', + description: 'Fast', + category: 'test', + parameters: [], + handler: async () => { + await new Promise(r => setTimeout(r, 5)); + return { success: true, output: 'fast' }; + }, + }); + + arsenal.register({ + name: 'slow_tool', + description: 'Slow', + category: 'test', + parameters: [], + handler: async () => { + await new Promise(r => setTimeout(r, 50)); + return { success: true, output: 'slow' }; + }, + }); + + arsenal.register({ + name: 'medium_tool', + description: 'Medium', + category: 'test', + parameters: [], + handler: async () => { + await new Promise(r => setTimeout(r, 25)); + return { success: true, output: 'medium' }; + }, + }); + + // slow first, fast second, medium third — but fast will finish first + const toolCalls = [ + makeToolCall('c1', 'slow_tool', {}), + makeToolCall('c2', 'fast_tool', {}), + makeToolCall('c3', 'medium_tool', {}), + ]; + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'Execute tools.'); + + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(3); + + // Results should be in the same order as toolCalls, not completion order + expect(toolSteps[0].toolName).toBe('slow_tool'); + expect(toolSteps[1].toolName).toBe('fast_tool'); + expect(toolSteps[2].toolName).toBe('medium_tool'); + }); + }); + + // ── 4. Concurrency limit is respected ───────────────────────────────────────── + + describe('concurrency limit', () => { + it('should respect max concurrency limit', async () => { + let maxConcurrent = 0; + let currentConcurrent = 0; + + // Register 5 tools + for (let i = 1; i <= 5; i++) { + arsenal.register({ + name: `concurrent_tool_${i}`, + description: `Concurrent ${i}`, + category: 'test', + parameters: [], + handler: async () => { + currentConcurrent++; + maxConcurrent = Math.max(maxConcurrent, currentConcurrent); + await new Promise(r => setTimeout(r, 20)); + currentConcurrent--; + return { success: true, output: `Result ${i}` }; + }, + }); + } + + const toolCalls = Array.from({ length: 5 }, (_, i) => + makeToolCall(`c${i + 1}`, `concurrent_tool_${i + 1}`, {}) + ); + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + // Set max concurrency to 2 + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + // Access the private maxConcurrency option — we set it via options + // For now, just verify all complete + const result = await agent.run(makeTask(), 'Execute all.'); + + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(5); + + // All should succeed + for (const step of toolSteps) { + expect(step.toolResult?.success).toBe(true); + } + }); + }); + + // ── 5. Scope-denied call doesn't block valid calls ──────────────────────────── + + describe('scope denial does not block', () => { + it('should execute valid calls even when one is scope-denied', async () => { + arsenal.register({ + name: 'allowed_tool', + description: 'Allowed', + category: 'test', + parameters: [ + { name: 'url', type: 'string', description: 'URL', required: true }, + ], + handler: async (ctx) => ({ + success: true, + output: `OK: ${ctx.parameters.url}`, + }), + }); + + // Set scope that blocks evil.com + arsenal.setScope({ + allowedHosts: ['example.com'], + allowLoopback: false, + allowPrivate: false, + }); + + const toolCalls = [ + makeToolCall('c1', 'allowed_tool', { url: 'https://example.com/page' }), + makeToolCall('c2', 'allowed_tool', { url: 'https://evil.com/attack' }), + makeToolCall('c3', 'allowed_tool', { url: 'https://example.com/other' }), + ]; + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'Execute tools.'); + + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(3); + + // First and third should succeed (in scope) + expect(toolSteps[0].toolResult?.success).toBe(true); + expect(toolSteps[2].toolResult?.success).toBe(true); + + // Second should be scope-denied + expect(toolSteps[1].toolResult?.success).toBe(false); + expect(toolSteps[1].toolResult?.error).toContain('SCOPE DENIED'); + }); + }); + + // ── 6. All calls fail → all errors returned ──────────────────────────────────── + + describe('all failures', () => { + it('should return error results for all failed tool calls', async () => { + arsenal.register({ + name: 'failing_tool', + description: 'Always fails', + category: 'test', + parameters: [], + handler: async () => { + throw new Error('Handler crashed'); + }, + }); + + const toolCalls = [ + makeToolCall('c1', 'failing_tool', {}), + makeToolCall('c2', 'failing_tool', {}), + ]; + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'All failed.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'Try tools.'); + + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(2); + + // Both should have errors + for (const step of toolSteps) { + expect(step.toolResult?.success).toBe(false); + expect(step.toolResult?.error).toBeDefined(); + } + }); + }); + + // ── 7. Unknown tool in batch doesn't crash other calls ──────────────────────── + + describe('unknown tool in batch', () => { + it('should handle unknown tool alongside valid tools', async () => { + arsenal.register({ + name: 'good_tool', + description: 'Good', + category: 'test', + parameters: [], + handler: async () => ({ success: true, output: 'good' }), + }); + + const toolCalls = [ + makeToolCall('c1', 'good_tool', { target: 'a' }), + makeToolCall('c2', 'nonexistent_tool', {}), + makeToolCall('c3', 'good_tool', { target: 'b' }), + ]; + + const llm = createMockLLM([ + makeLLMResponse(toolCalls), + makeLLMResponse([], 'Done.'), + ]); + + const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 }); + const result = await agent.run(makeTask(), 'Execute.'); + + const toolSteps = result.steps.filter(s => s.type === 'tool_call'); + expect(toolSteps).toHaveLength(3); + + expect(toolSteps[0].toolResult?.success).toBe(true); + expect(toolSteps[1].toolResult?.success).toBe(false); + expect(toolSteps[1].toolResult?.error).toContain('tool_not_found'); + expect(toolSteps[2].toolResult?.success).toBe(true); + }); + }); +}); diff --git a/src/__tests__/arsenal-validation.test.ts b/src/__tests__/arsenal-validation.test.ts new file mode 100644 index 0000000..26daeda --- /dev/null +++ b/src/__tests__/arsenal-validation.test.ts @@ -0,0 +1,351 @@ +/** + * Arsenal Validation Gate Tests + * + * Tests for the validation gate in Arsenal.execute() — validates tool args + * against parameter schema before handler execution, and categorizes errors + * using ToolError with proper ToolErrorCategory. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { Arsenal, createToolContext } from '../arsenal/index.js'; +import { ToolError, ToolErrorCategory } from '../types/index.js'; +import type { CustomTool, ToolParameter } from '../types/index.js'; + +// ─── Helper: create a minimal tool with a schema ────────────────────────────── + +function makeTool( + name: string, + params: ToolParameter[], + handler?: (ctx: any) => Promise, +): CustomTool { + return { + name, + description: `Test tool: ${name}`, + category: 'test', + parameters: params, + handler: handler || (async (ctx) => ({ + success: true, + output: `Executed ${name} with ${JSON.stringify(ctx.parameters)}`, + })), + }; +} + +// ─── Shared schemas ─────────────────────────────────────────────────────────── + +const echoSchema: ToolParameter[] = [ + { name: 'message', type: 'string', description: 'Message to echo', required: true }, + { name: 'count', type: 'number', description: 'Repeat count', required: false, default: 1 }, +]; + +const enumSchema: ToolParameter[] = [ + { + name: 'method', + type: 'string', + description: 'HTTP method', + required: true, + enum: ['GET', 'POST', 'PUT'], + }, +]; + +const nestedSchema: ToolParameter[] = [ + { + name: 'config', + type: 'object', + description: 'Config object', + required: true, + properties: { + host: { name: 'host', type: 'string', description: 'Host', required: true }, + port: { name: 'port', type: 'number', description: 'Port', required: false }, + }, + }, +]; + +// ============================================================================= +// Tests +// ============================================================================= + +describe('Arsenal Validation Gate', () => { + let arsenal: Arsenal; + + beforeEach(() => { + arsenal = new Arsenal(); + }); + + // ── 1. Valid args pass validation gate ──────────────────────────────────────── + + describe('valid args pass validation', () => { + it('should execute tool when all required fields are present', async () => { + arsenal.register(makeTool('echo', echoSchema)); + const result = await arsenal.execute('echo', createToolContext(undefined, { message: 'hello' })); + expect(result.success).toBe(true); + expect(result.output).toContain('echo'); + }); + + it('should execute tool with only optional fields omitted', async () => { + arsenal.register(makeTool('echo', echoSchema)); + const result = await arsenal.execute('echo', createToolContext(undefined, { message: 'test', count: 3 })); + expect(result.success).toBe(true); + }); + + it('should execute tool with no parameters defined (empty schema)', async () => { + arsenal.register(makeTool('noop', [])); + const result = await arsenal.execute('noop', createToolContext(undefined, { anything: 'goes' })); + expect(result.success).toBe(true); + }); + + it('should pass validation for valid enum value', async () => { + arsenal.register(makeTool('http', enumSchema)); + const result = await arsenal.execute('http', createToolContext(undefined, { method: 'GET' })); + expect(result.success).toBe(true); + }); + }); + + // ── 2. Missing required field → ToolError(ValidationError) ──────────────────── + + describe('missing required field', () => { + it('should throw ToolError with ValidationError when required field is missing', async () => { + arsenal.register(makeTool('echo', echoSchema)); + await expect( + arsenal.execute('echo', createToolContext(undefined, {})) + ).rejects.toThrow(ToolError); + + try { + await arsenal.execute('echo', createToolContext(undefined, {})); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + expect((e as ToolError).category).toBe(ToolErrorCategory.ValidationError); + expect((e as ToolError).toolName).toBe('echo'); + } + }); + + it('should include field name in error message', async () => { + arsenal.register(makeTool('echo', echoSchema)); + try { + await arsenal.execute('echo', createToolContext(undefined, {})); + expect.fail('Should have thrown'); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + expect((e as ToolError).message).toContain('message'); + } + }); + }); + + // ── 3. Wrong type → ToolError(ValidationError) ──────────────────────────────── + + describe('wrong type', () => { + it('should throw ToolError with ValidationError for wrong type', async () => { + arsenal.register(makeTool('echo', echoSchema)); + await expect( + arsenal.execute('echo', createToolContext(undefined, { message: 123 })) + ).rejects.toThrow(ToolError); + + try { + await arsenal.execute('echo', createToolContext(undefined, { message: 123 })); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + expect((e as ToolError).category).toBe(ToolErrorCategory.ValidationError); + } + }); + + it('should throw ToolError for invalid enum value', async () => { + arsenal.register(makeTool('http', enumSchema)); + try { + await arsenal.execute('http', createToolContext(undefined, { method: 'PATCH' })); + expect.fail('Should have thrown'); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + expect((e as ToolError).category).toBe(ToolErrorCategory.ValidationError); + } + }); + }); + + // ── 4. Unknown tool → ToolError(ToolNotFound) ──────────────────────────────── + + describe('unknown tool', () => { + it('should throw ToolError with ToolNotFound for unregistered tool', async () => { + try { + await arsenal.execute('nonexistent_tool', createToolContext(undefined, {})); + expect.fail('Should have thrown'); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + expect((e as ToolError).category).toBe(ToolErrorCategory.ToolNotFound); + expect((e as ToolError).toolName).toBe('nonexistent_tool'); + } + }); + }); + + // ── 5. Scope denied → ToolError(ScopeDenied) ───────────────────────────────── + + describe('scope denied', () => { + it('should return ToolResult with scope denial for out-of-scope target', async () => { + arsenal.register(makeTool('http', [ + { name: 'url', type: 'string', description: 'URL', required: true }, + ])); + arsenal.setScope({ + allowedHosts: ['example.com'], + allowLoopback: false, + allowPrivate: false, + }); + + const result = await arsenal.execute('http', createToolContext( + undefined, + { url: 'https://evil.com/attack' } + )); + expect(result.success).toBe(false); + expect(result.error).toContain('SCOPE DENIED'); + }); + + it('should allow in-scope targets to pass', async () => { + arsenal.register(makeTool('http', [ + { name: 'url', type: 'string', description: 'URL', required: true }, + ])); + arsenal.setScope({ + allowedHosts: ['example.com'], + allowLoopback: false, + allowPrivate: false, + }); + + const result = await arsenal.execute('http', createToolContext( + undefined, + { url: 'https://example.com/page' } + )); + expect(result.success).toBe(true); + }); + }); + + // ── 6. Error category is accessible on ToolError instances ──────────────────── + + describe('error category accessibility', () => { + it('should have category property on ToolError', () => { + const err = new ToolError(ToolErrorCategory.ValidationError, 'test'); + expect(err.category).toBe(ToolErrorCategory.ValidationError); + }); + + it('should have category property on ToolError for each category', () => { + const categories = [ + ToolErrorCategory.ScopeDenied, + ToolErrorCategory.ToolNotFound, + ToolErrorCategory.Timeout, + ToolErrorCategory.ExecutionError, + ToolErrorCategory.ValidationError, + ]; + for (const cat of categories) { + const err = new ToolError(cat, `msg for ${cat}`); + expect(err.category).toBe(cat); + } + }); + + it('should throw ToolError (catchable as Error) for validation failure', async () => { + arsenal.register(makeTool('echo', echoSchema)); + try { + await arsenal.execute('echo', createToolContext(undefined, {})); + expect.fail('Should have thrown'); + } catch (e) { + expect(e).toBeInstanceOf(Error); + expect(e).toBeInstanceOf(ToolError); + const toolErr = e as ToolError; + expect(toolErr.category).toBeDefined(); + expect(typeof toolErr.category).toBe('string'); + } + }); + }); + + // ── 7. Execution error → ToolError(ExecutionError) ──────────────────────────── + + describe('execution error categorization', () => { + it('should throw ToolError with ExecutionError when handler throws', async () => { + const failingTool = makeTool('boom', echoSchema, async () => { + throw new Error('handler crashed'); + }); + arsenal.register(failingTool); + + try { + await arsenal.execute('boom', createToolContext(undefined, { message: 'hi' })); + expect.fail('Should have thrown'); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + expect((e as ToolError).category).toBe(ToolErrorCategory.ExecutionError); + } + }); + }); + + // ── 8. Validation error emits tool:error event ──────────────────────────────── + + describe('events', () => { + it('should emit tool:error on validation failure', async () => { + arsenal.register(makeTool('echo', echoSchema)); + const errors: any[] = []; + arsenal.on('tool:error', (evt) => errors.push(evt)); + + try { + await arsenal.execute('echo', createToolContext(undefined, {})); + } catch { + // expected + } + + expect(errors).toHaveLength(1); + expect(errors[0].tool.name).toBe('echo'); + }); + + it('should emit tool:error on tool not found', async () => { + const errors: any[] = []; + arsenal.on('tool:error', (evt) => errors.push(evt)); + + try { + await arsenal.execute('nope', createToolContext(undefined, {})); + } catch { + // expected + } + + expect(errors).toHaveLength(1); + expect(errors[0].error).toBeInstanceOf(ToolError); + }); + }); +}); + +// ============================================================================= +// getToolDefinitions with buildJsonSchema +// ============================================================================= + +describe('Arsenal.getToolDefinitions with buildJsonSchema', () => { + let arsenal: Arsenal; + + beforeEach(() => { + arsenal = new Arsenal(); + }); + + it('should include enum in tool definitions', () => { + arsenal.register(makeTool('http', enumSchema)); + const defs = arsenal.getToolDefinitions(); + expect(defs).toHaveLength(1); + const methodProp = defs[0].parameters.properties.method; + expect(methodProp).toBeDefined(); + // The property should have the enum field from buildJsonSchema + expect((methodProp as any).enum).toEqual(['GET', 'POST', 'PUT']); + }); + + it('should include default values in tool definitions', () => { + arsenal.register(makeTool('echo', echoSchema)); + const defs = arsenal.getToolDefinitions(); + const countProp = defs[0].parameters.properties.count; + expect(countProp).toBeDefined(); + expect((countProp as any).default).toBe(1); + }); + + it('should include required fields', () => { + arsenal.register(makeTool('echo', echoSchema)); + const defs = arsenal.getToolDefinitions(); + expect(defs[0].parameters.required).toEqual(['message']); + }); + + it('should include nested object properties', () => { + arsenal.register(makeTool('config', nestedSchema)); + const defs = arsenal.getToolDefinitions(); + const configProp = defs[0].parameters.properties.config as any; + expect(configProp.properties).toBeDefined(); + expect(configProp.properties.host).toEqual({ + type: 'string', + description: 'Host', + }); + }); +}); diff --git a/src/__tests__/ollama-native-tools.test.ts b/src/__tests__/ollama-native-tools.test.ts new file mode 100644 index 0000000..88bfef8 --- /dev/null +++ b/src/__tests__/ollama-native-tools.test.ts @@ -0,0 +1,346 @@ +/** + * Ollama Native Function Calling — Phase 6 / PR 3 + * + * Tests that LocalAdapter probes, caches, and uses native Ollama /api/chat tool_calls + * (with fallback to the existing text-based path for models that don't support it). + * + * Spec coverage: + * 1. Probe detects native support → native tool calls parsed + * 2. Probe failure → falls back to text-based tool calling + * 3. Probe result cached per model → no re-probe on subsequent calls + * 4. nativeTools=false config → skips probe entirely (always text) + * 5. nativeTools=true config → forces native mode + * 6. OpenAI-compatible wire → native tools sent correctly + * 7. OpenAI-compatible wire → text fallback on probe failure + */ +import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest'; +import { LLMBackbone, __resetLocalAdapterCache } from '../llm/index.js'; +import type { LLMToolDefinition } from '../types/index.js'; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +const TOOLS: LLMToolDefinition[] = [{ + name: 'test_tool', + description: 'a test tool', + parameters: { + type: 'object', + properties: { input: { type: 'string', description: 'input value' } }, + required: ['input'], + }, +}]; + +function localBackbone(model = 'test-model', overrides: Record = {}): LLMBackbone { + return new LLMBackbone({ + provider: 'local', + model, + baseUrl: 'http://localhost:11434/api', + maxTokens: 4096, + temperature: 0.7, + ...overrides, + } as never); +} + +function mockFetch(body: unknown, ok = true) { + return vi.fn(async (_url: string, _init: { body: string }) => ({ + ok, + status: ok ? 200 : 500, + json: async () => body, + } as unknown as Response)); +} + +const origFetch = global.fetch; + +// ── Tests ──────────────────────────────────────────────────────────────────── + +describe('Ollama native wire (/api/chat)', () => { + beforeEach(() => { __resetLocalAdapterCache(); }); + afterEach(() => { global.fetch = origFetch; }); + + it('probe detects native support: sends tools array, parses tool_calls from response', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { + role: 'assistant', + content: '', + tool_calls: [ + { function: { name: 'test_tool', arguments: '{"input":"hello"}' } }, + ], + }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('native-probe-1'); + const res = await bb.chatWithTools([{ role: 'user', content: 'use the tool' }], TOOLS); + + // Tool calls parsed correctly + expect(res.toolCalls).toBeDefined(); + expect(res.toolCalls).toHaveLength(1); + expect(res.toolCalls![0].name).toBe('test_tool'); + expect(res.toolCalls![0].arguments).toEqual({ input: 'hello' }); + expect(res.finishReason).toBe('tool_calls'); + + // Request body included tools in Ollama native format + const body = JSON.parse(spy.mock.calls[0][1].body); + expect(body.tools).toBeDefined(); + expect(body.tools).toHaveLength(1); + expect(body.tools[0].type).toBe('function'); + expect(body.tools[0].function.name).toBe('test_tool'); + expect(body.tools[0].function.parameters).toBeDefined(); + }); + + it('falls back to text-based parsing when model lacks native tool_calls in response', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { + role: 'assistant', + content: '```json\n{"tool_calls":[{"name":"test_tool","arguments":{"input":"hello"}}]}\n```', + }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('native-probe-fail-1'); + const res = await bb.chatWithTools([{ role: 'user', content: 'use the tool' }], TOOLS); + + // Text-based fallback works — tool calls still parsed + expect(res.toolCalls).toBeDefined(); + expect(res.toolCalls).toHaveLength(1); + expect(res.toolCalls![0].name).toBe('test_tool'); + expect(res.toolCalls![0].arguments).toEqual({ input: 'hello' }); + expect(res.finishReason).toBe('tool_calls'); + }); + + it('retries and caches an explicit native-tools rejection', async () => { + let callIdx = 0; + const spy = vi.fn(async (_url: string, init: { body: string }) => { + const body = JSON.parse(init.body); + if (callIdx === 0) { + expect(body.tools).toBeDefined(); + callIdx++; + return { + ok: false, + status: 400, + json: async () => ({ error: 'tools unsupported' }), + } as unknown as Response; + } + expect(body.tools).toBeUndefined(); + callIdx++; + return { + ok: true, + status: 200, + json: async () => ({ + model: 'cache-test-1', + message: { role: 'assistant', content: '```json\n{"tool_calls":[{"name":"test_tool","arguments":{"input":"fallback"}}]}\n```' }, + }), + } as unknown as Response; + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('cache-test-1'); + const res = await bb.chatWithTools([{ role: 'user', content: 'first' }], TOOLS); + expect(res.toolCalls?.[0].arguments).toEqual({ input: 'fallback' }); + expect(spy).toHaveBeenCalledTimes(2); + }); + + it('does not mark native tools unsupported when a model answers without a call', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { role: 'assistant', content: 'No tool is needed.' }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('no-call-is-not-failure'); + await bb.chatWithTools([{ role: 'user', content: 'answer directly' }], TOOLS); + await bb.chatWithTools([{ role: 'user', content: 'now use a tool' }], TOOLS); + + expect(JSON.parse(spy.mock.calls[0][1].body).tools).toBeDefined(); + expect(JSON.parse(spy.mock.calls[1][1].body).tools).toBeDefined(); + }); + + it('accepts Ollama native arguments in their object wire shape', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { + role: 'assistant', + content: '', + tool_calls: [ + { function: { name: 'test_tool', arguments: { input: 'object-wire' } } }, + ], + }, + }); + global.fetch = spy as unknown as typeof fetch; + + const res = await localBackbone('object-args').chatWithTools( + [{ role: 'user', content: 'use it' }], + TOOLS, + ); + expect(res.toolCalls?.[0].arguments).toEqual({ input: 'object-wire' }); + }); + + it('caches successful probe: second call still sends native tools', async () => { + let _callIdx = 0; + const spy = vi.fn(async (_url: string, _init: { body: string }) => { + _callIdx++; + return { + ok: true, + json: async () => ({ + model: 'cache-test-2', + message: { + role: 'assistant', + content: '', + tool_calls: [ + { function: { name: 'test_tool', arguments: '{"input":"x"}' } }, + ], + }, + }), + } as unknown as Response; + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('cache-test-2'); + await bb.chatWithTools([{ role: 'user', content: 'first' }], TOOLS); + await bb.chatWithTools([{ role: 'user', content: 'second' }], TOOLS); + + // Both calls should have tools in request (native supported) + const body1 = JSON.parse(spy.mock.calls[0][1].body); + const body2 = JSON.parse(spy.mock.calls[1][1].body); + expect(body1.tools).toBeDefined(); + expect(body2.tools).toBeDefined(); + expect(spy).toHaveBeenCalledTimes(2); + }); +}); + +describe('nativeTools config option', () => { + beforeEach(() => { __resetLocalAdapterCache(); }); + afterEach(() => { global.fetch = origFetch; }); + + it('nativeTools=false skips native tools entirely (always text-based)', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { role: 'assistant', content: 'tool result in text' }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('config-false-1', { nativeTools: false }); + await bb.chatWithTools([{ role: 'user', content: 'do it' }], TOOLS); + + const body = JSON.parse(spy.mock.calls[0][1].body); + expect(body.tools).toBeUndefined(); + }); + + it('nativeTools=true forces native mode even if cache would say false', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { + role: 'assistant', + content: '', + tool_calls: [ + { function: { name: 'test_tool', arguments: '{"input":"forced"}' } }, + ], + }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('config-true-1', { nativeTools: true }); + const res = await bb.chatWithTools([{ role: 'user', content: 'force' }], TOOLS); + + const body = JSON.parse(spy.mock.calls[0][1].body); + expect(body.tools).toBeDefined(); + expect(res.toolCalls).toBeDefined(); + expect(res.toolCalls![0].arguments).toEqual({ input: 'forced' }); + }); +}); + +describe('OpenAI-compatible wire (/v1/chat/completions)', () => { + beforeEach(() => { __resetLocalAdapterCache(); }); + afterEach(() => { global.fetch = origFetch; }); + + it('sends native tools when probe succeeds on OpenAI-compatible endpoint', async () => { + const spy = mockFetch({ + model: 'local', + choices: [{ + message: { + content: '', + tool_calls: [ + { id: 'call_1', type: 'function', function: { name: 'test_tool', arguments: '{"input":"via-openai-wire"}' } }, + ], + }, + finish_reason: 'tool_calls', + }], + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('openai-wire-1', { baseUrl: 'http://localhost:1234/v1' }); + const res = await bb.chatWithTools([{ role: 'user', content: 'scan' }], TOOLS); + + expect(res.toolCalls).toBeDefined(); + expect(res.toolCalls).toHaveLength(1); + expect(res.toolCalls![0].name).toBe('test_tool'); + expect(res.toolCalls![0].arguments).toEqual({ input: 'via-openai-wire' }); + + const body = JSON.parse(spy.mock.calls[0][1].body); + expect(body.tools).toBeDefined(); + expect(body.tools[0].type).toBe('function'); + expect(body.tools[0].function.name).toBe('test_tool'); + // OpenAI-compatible wire hits /v1/chat/completions + expect(spy.mock.calls[0][0]).toContain('/v1/chat/completions'); + }); + + it('falls back to text-based parsing on OpenAI-compatible wire when no native tool_calls', async () => { + const spy = mockFetch({ + model: 'local', + choices: [{ + message: { + content: '{"tool_calls":[{"name":"test_tool","arguments":{"input":"text-fallback"}}]}', + }, + finish_reason: 'stop', + }], + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('openai-wire-fallback-1', { baseUrl: 'http://localhost:1234/v1' }); + const res = await bb.chatWithTools([{ role: 'user', content: 'scan' }], TOOLS); + + expect(res.toolCalls).toBeDefined(); + expect(res.toolCalls).toHaveLength(1); + expect(res.toolCalls![0].name).toBe('test_tool'); + expect(res.toolCalls![0].arguments).toEqual({ input: 'text-fallback' }); + }); +}); + +describe('existing text-based path preserved', () => { + beforeEach(() => { __resetLocalAdapterCache(); }); + afterEach(() => { global.fetch = origFetch; }); + + it('prose-only response produces no toolCalls (final answer)', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { + role: 'assistant', + content: 'Surface exhausted. No vulnerabilities found.', + }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('prose-only-1'); + const res = await bb.chatWithTools([{ role: 'user', content: 'done' }], TOOLS); + + expect(res.toolCalls).toBeUndefined(); + expect(res.finishReason).toBe('stop'); + expect(res.content).toContain('Surface exhausted'); + }); + + it('no tools offered → no tool contract injected, no native tools', async () => { + const spy = mockFetch({ + model: 'test-model', + message: { role: 'assistant', content: 'ok' }, + }); + global.fetch = spy as unknown as typeof fetch; + + const bb = localBackbone('no-tools-1'); + const res = await bb.chat([{ role: 'user', content: 'hello' }]); + + const body = JSON.parse(spy.mock.calls[0][1].body); + expect(body.tools).toBeUndefined(); + expect(res.content).toBe('ok'); + }); +}); diff --git a/src/__tests__/tool-error.test.ts b/src/__tests__/tool-error.test.ts new file mode 100644 index 0000000..5b87846 --- /dev/null +++ b/src/__tests__/tool-error.test.ts @@ -0,0 +1,138 @@ +/** + * ToolError Type Tests + * + * Tests for ToolErrorCategory enum and ToolError class — structured error + * handling for tool validation, execution, and scope enforcement. + */ + +import { describe, it, expect } from 'vitest'; +import { ToolError, ToolErrorCategory, type ToolValidationError } from '../types/index.js'; + +describe('ToolErrorCategory', () => { + it('should have all expected category values', () => { + expect(ToolErrorCategory.ScopeDenied).toBe('scope_denied'); + expect(ToolErrorCategory.ToolNotFound).toBe('tool_not_found'); + expect(ToolErrorCategory.Timeout).toBe('timeout'); + expect(ToolErrorCategory.ExecutionError).toBe('execution_error'); + expect(ToolErrorCategory.ValidationError).toBe('validation_error'); + }); + + it('should have exactly 5 categories', () => { + const values = Object.values(ToolErrorCategory); + expect(values).toHaveLength(5); + }); +}); + +describe('ToolError', () => { + it('should construct with category and message', () => { + const err = new ToolError(ToolErrorCategory.ValidationError, 'Invalid argument'); + expect(err.category).toBe(ToolErrorCategory.ValidationError); + expect(err.message).toBe('Invalid argument'); + expect(err.name).toBe('ToolError'); + }); + + it('should be an instance of Error', () => { + const err = new ToolError(ToolErrorCategory.Timeout, 'Timed out'); + expect(err).toBeInstanceOf(Error); + expect(err).toBeInstanceOf(ToolError); + }); + + it('should accept optional toolName', () => { + const err = new ToolError(ToolErrorCategory.ToolNotFound, 'Not found', 'nmap_scan'); + expect(err.toolName).toBe('nmap_scan'); + }); + + it('should have undefined toolName when not provided', () => { + const err = new ToolError(ToolErrorCategory.ExecutionError, 'Failed'); + expect(err.toolName).toBeUndefined(); + }); + + it('should preserve stack trace', () => { + const err = new ToolError(ToolErrorCategory.ScopeDenied, 'Denied'); + expect(err.stack).toBeDefined(); + expect(err.stack).toContain('ToolError'); + }); + + it('should be throwable and catchable', () => { + expect(() => { + throw new ToolError(ToolErrorCategory.ValidationError, 'Bad input'); + }).toThrow(ToolError); + }); + + it('should allow checking category in catch block', () => { + try { + throw new ToolError(ToolErrorCategory.Timeout, 'Tool timed out', 'nuclei'); + } catch (e) { + expect(e).toBeInstanceOf(ToolError); + if (e instanceof ToolError) { + expect(e.category).toBe(ToolErrorCategory.Timeout); + expect(e.toolName).toBe('nuclei'); + } + } + }); + + it('should work with all categories', () => { + const categories = [ + ToolErrorCategory.ScopeDenied, + ToolErrorCategory.ToolNotFound, + ToolErrorCategory.Timeout, + ToolErrorCategory.ExecutionError, + ToolErrorCategory.ValidationError, + ]; + for (const cat of categories) { + const err = new ToolError(cat, `Error for ${cat}`); + expect(err.category).toBe(cat); + } + }); + + it('should have readonly category and toolName', () => { + const err = new ToolError(ToolErrorCategory.ValidationError, 'msg', 'tool'); + // TypeScript readonly — runtime still allows assignment but the intent is clear + expect(err.category).toBe(ToolErrorCategory.ValidationError); + expect(err.toolName).toBe('tool'); + }); +}); + +describe('ToolValidationError', () => { + it('should have all required fields', () => { + const error: ToolValidationError = { + field: 'port', + message: 'Port must be a number', + received: 'abc', + expected: 'number', + }; + expect(error.field).toBe('port'); + expect(error.message).toBe('Port must be a number'); + expect(error.received).toBe('abc'); + expect(error.expected).toBe('number'); + }); + + it('should accept unknown received values', () => { + const error: ToolValidationError = { + field: 'config', + message: 'Invalid type', + received: undefined, + expected: 'object', + }; + expect(error.received).toBeUndefined(); + }); + + it('should accept complex received values', () => { + const error: ToolValidationError = { + field: 'headers', + message: 'Array item type mismatch', + received: { key: 123 }, + expected: 'string', + }; + expect(error.received).toEqual({ key: 123 }); + }); + + it('should be usable in arrays for multi-field validation', () => { + const errors: ToolValidationError[] = [ + { field: 'host', message: 'Required', received: undefined, expected: 'string' }, + { field: 'port', message: 'Out of range', received: 99999, expected: '1-65535' }, + ]; + expect(errors).toHaveLength(2); + expect(errors.map(e => e.field)).toEqual(['host', 'port']); + }); +}); diff --git a/src/__tests__/tool-parameter-types.test.ts b/src/__tests__/tool-parameter-types.test.ts new file mode 100644 index 0000000..4fe4584 --- /dev/null +++ b/src/__tests__/tool-parameter-types.test.ts @@ -0,0 +1,237 @@ +/** + * ToolParameter Type Tests + * + * Tests for the extended ToolParameter type supporting richer JSON Schema: + * flat primitives, array with items, object with properties, and enum constraints. + */ + +import { describe, it, expect } from 'vitest'; +import type { ToolParameter } from '../types/index.js'; + +describe('ToolParameter types', () => { + describe('flat primitive parameter', () => { + it('should accept a string parameter with all base fields', () => { + const param: ToolParameter = { + name: 'target', + type: 'string', + description: 'The target host', + required: true, + }; + expect(param.name).toBe('target'); + expect(param.type).toBe('string'); + expect(param.required).toBe(true); + }); + + it('should accept optional default value', () => { + const param: ToolParameter = { + name: 'port', + type: 'number', + description: 'Port number', + required: false, + default: 8080, + }; + expect(param.default).toBe(8080); + }); + }); + + describe('enum constraint', () => { + it('should accept string enum values', () => { + const param: ToolParameter = { + name: 'method', + type: 'string', + description: 'HTTP method', + required: true, + enum: ['GET', 'POST', 'PUT', 'DELETE'], + }; + expect(param.enum).toEqual(['GET', 'POST', 'PUT', 'DELETE']); + }); + + it('should accept numeric enum values', () => { + const param: ToolParameter = { + name: 'level', + type: 'number', + description: 'Verbosity level', + required: false, + enum: [0, 1, 2, 3], + }; + expect(param.enum).toEqual([0, 1, 2, 3]); + }); + }); + + describe('array parameter with items', () => { + it('should accept array type with items schema', () => { + const param: ToolParameter = { + name: 'domains', + type: 'array', + description: 'List of domains to scan', + required: true, + items: { + type: 'string', + description: 'A domain name', + }, + }; + expect(param.type).toBe('array'); + expect(param.items?.type).toBe('string'); + }); + + it('should accept array of objects via nested properties', () => { + const param: ToolParameter = { + name: 'headers', + type: 'array', + description: 'HTTP headers', + required: false, + items: { + type: 'object', + properties: { + key: { + name: 'key', + type: 'string', + description: 'Header name', + required: true, + }, + value: { + name: 'value', + type: 'string', + description: 'Header value', + required: true, + }, + }, + }, + }; + expect(param.items?.properties?.key.type).toBe('string'); + expect(param.items?.properties?.value.type).toBe('string'); + }); + }); + + describe('object parameter with properties', () => { + it('should accept object type with nested properties', () => { + const param: ToolParameter = { + name: 'config', + type: 'object', + description: 'Connection configuration', + required: false, + properties: { + host: { + name: 'host', + type: 'string', + description: 'Target host', + required: true, + }, + port: { + name: 'port', + type: 'number', + description: 'Target port', + required: false, + default: 443, + }, + }, + additionalProperties: false, + }; + expect(param.type).toBe('object'); + expect(param.properties?.host.required).toBe(true); + expect(param.properties?.port.default).toBe(443); + expect(param.additionalProperties).toBe(false); + }); + + it('should allow additionalProperties to be true', () => { + const param: ToolParameter = { + name: 'metadata', + type: 'object', + description: 'Arbitrary metadata', + required: false, + additionalProperties: true, + }; + expect(param.additionalProperties).toBe(true); + }); + }); + + describe('type safety — new fields should not compile without extended type', () => { + it('enum field is part of ToolParameter shape', () => { + const param: ToolParameter = { name: 'x', type: 'string', description: '', required: false, enum: ['a'] }; + expect(param.enum).toBeDefined(); + expect(param.enum?.[0]).toBe('a'); + }); + + it('items field is part of ToolParameter shape', () => { + const param: ToolParameter = { name: 'x', type: 'array', description: '', required: false, items: { type: 'string' } }; + expect(param.items).toBeDefined(); + expect(param.items?.type).toBe('string'); + }); + + it('properties field is part of ToolParameter shape', () => { + const param: ToolParameter = { name: 'x', type: 'object', description: '', required: false, properties: {} }; + expect(param.properties).toBeDefined(); + }); + + it('additionalProperties field is part of ToolParameter shape', () => { + const param: ToolParameter = { name: 'x', type: 'object', description: '', required: false, additionalProperties: true }; + expect(param.additionalProperties).toBe(true); + }); + }); + + describe('backward compatibility — base fields still work', () => { + it('existing string parameter without new fields compiles', () => { + const param: ToolParameter = { + name: 'domain', + type: 'string', + description: 'Target domain', + required: true, + }; + expect(param.enum).toBeUndefined(); + expect(param.items).toBeUndefined(); + expect(param.properties).toBeUndefined(); + expect(param.additionalProperties).toBeUndefined(); + }); + + it('existing parameter with default still works', () => { + const param: ToolParameter = { + name: 'timeout', + type: 'number', + description: 'Timeout in seconds', + required: false, + default: 30, + }; + expect(param.default).toBe(30); + }); + }); + + describe('deeply nested schemas', () => { + it('should support 3-level nesting via items.properties.items', () => { + const param: ToolParameter = { + name: 'matrix', + type: 'array', + description: 'A matrix of rows', + required: true, + items: { + type: 'object', + properties: { + row: { + name: 'row', + type: 'array', + description: 'A row of values', + required: true, + items: { + type: 'object', + properties: { + value: { + name: 'value', + type: 'string', + description: 'Cell value', + required: true, + }, + }, + }, + }, + }, + }, + }; + // Level 1: array + expect(param.type).toBe('array'); + // Level 2: object with row property + expect(param.items?.properties?.row.type).toBe('array'); + // Level 3: array of objects with value property + const rowParam = param.items?.properties?.row; + expect(rowParam?.items?.properties?.value.type).toBe('string'); + }); + }); +}); diff --git a/src/__tests__/validation.test.ts b/src/__tests__/validation.test.ts new file mode 100644 index 0000000..4ba0dbe --- /dev/null +++ b/src/__tests__/validation.test.ts @@ -0,0 +1,497 @@ +/** + * Schema Validation Utility Tests + * + * Tests for validateToolArgs() — AJV-based argument validation with + * schema depth enforcement and lazy loading. + */ + +import { describe, it, expect } from 'vitest'; +import type { ToolParameter } from '../types/index.js'; + +// Import will fail until validation/index.ts exists +import { validateToolArgs, assertSchemaDepth, buildJsonSchema } from '../validation/index.js'; + +describe('validateToolArgs', () => { + describe('valid args pass validation', () => { + it('should return empty array for valid string arg', () => { + const schema: ToolParameter[] = [ + { name: 'target', type: 'string', description: 'Target host', required: true }, + ]; + const errors = validateToolArgs('test_tool', { target: 'example.com' }, schema); + expect(errors).toEqual([]); + }); + + it('should return empty array for valid number arg', () => { + const schema: ToolParameter[] = [ + { name: 'port', type: 'number', description: 'Port', required: true }, + ]; + const errors = validateToolArgs('test_tool', { port: 8080 }, schema); + expect(errors).toEqual([]); + }); + + it('should return empty array for valid boolean arg', () => { + const schema: ToolParameter[] = [ + { name: 'verbose', type: 'boolean', description: 'Verbose output', required: false }, + ]; + const errors = validateToolArgs('test_tool', { verbose: true }, schema); + expect(errors).toEqual([]); + }); + + it('should return empty array for valid array arg', () => { + const schema: ToolParameter[] = [ + { + name: 'domains', + type: 'array', + description: 'Domain list', + required: true, + items: { type: 'string' }, + }, + ]; + const errors = validateToolArgs('test_tool', { domains: ['a.com', 'b.com'] }, schema); + expect(errors).toEqual([]); + }); + + it('should return empty array for valid object arg', () => { + const schema: ToolParameter[] = [ + { + name: 'config', + type: 'object', + description: 'Config', + required: false, + properties: { + host: { name: 'host', type: 'string', description: 'Host', required: true }, + port: { name: 'port', type: 'number', description: 'Port', required: false }, + }, + }, + ]; + const errors = validateToolArgs('test_tool', { config: { host: 'localhost' } }, schema); + expect(errors).toEqual([]); + }); + }); + + describe('missing required field', () => { + it('should return validation error for missing required string', () => { + const schema: ToolParameter[] = [ + { name: 'target', type: 'string', description: 'Target host', required: true }, + ]; + const errors = validateToolArgs('test_tool', {}, schema); + expect(errors).toHaveLength(1); + expect(errors[0].field).toBe('target'); + expect(errors[0].expected).toContain('required'); + }); + + it('should return multiple errors for multiple missing required fields', () => { + const schema: ToolParameter[] = [ + { name: 'host', type: 'string', description: 'Host', required: true }, + { name: 'port', type: 'number', description: 'Port', required: true }, + ]; + const errors = validateToolArgs('test_tool', {}, schema); + expect(errors).toHaveLength(2); + expect(errors.map(e => e.field)).toEqual(expect.arrayContaining(['host', 'port'])); + }); + }); + + describe('wrong type', () => { + it('should return validation error for string when number expected', () => { + const schema: ToolParameter[] = [ + { name: 'port', type: 'number', description: 'Port', required: true }, + ]; + const errors = validateToolArgs('test_tool', { port: 'abc' }, schema); + expect(errors).toHaveLength(1); + expect(errors[0].field).toBe('port'); + }); + + it('coerces a numeric string when a number is expected', () => { + const schema: ToolParameter[] = [ + { name: 'port', type: 'number', description: 'Port', required: true }, + ]; + const args: Record = { port: '8080' }; + expect(validateToolArgs('test_tool', args, schema)).toEqual([]); + expect(args.port).toBe(8080); + }); + + it('should return validation error for number when string expected', () => { + const schema: ToolParameter[] = [ + { name: 'target', type: 'string', description: 'Target', required: true }, + ]; + const errors = validateToolArgs('test_tool', { target: 123 }, schema); + expect(errors).toHaveLength(1); + expect(errors[0].field).toBe('target'); + }); + + it('should return validation error for wrong array item type', () => { + const schema: ToolParameter[] = [ + { + name: 'ports', + type: 'array', + description: 'Ports', + required: true, + items: { type: 'number' }, + }, + ]; + const errors = validateToolArgs('test_tool', { ports: ['a', 'b'] }, schema); + expect(errors.length).toBeGreaterThan(0); + }); + }); + + describe('enum constraint', () => { + it('should return validation error for value not in enum', () => { + const schema: ToolParameter[] = [ + { + name: 'method', + type: 'string', + description: 'HTTP method', + required: true, + enum: ['GET', 'POST', 'PUT', 'DELETE'], + }, + ]; + const errors = validateToolArgs('test_tool', { method: 'PATCH' }, schema); + expect(errors).toHaveLength(1); + expect(errors[0].field).toBe('method'); + }); + + it('should return empty array for valid enum value', () => { + const schema: ToolParameter[] = [ + { + name: 'method', + type: 'string', + description: 'HTTP method', + required: true, + enum: ['GET', 'POST', 'PUT', 'DELETE'], + }, + ]; + const errors = validateToolArgs('test_tool', { method: 'GET' }, schema); + expect(errors).toEqual([]); + }); + }); + + describe('nested object validation', () => { + it('should validate depth 1 nested object', () => { + const schema: ToolParameter[] = [ + { + name: 'config', + type: 'object', + description: 'Config', + required: true, + properties: { + host: { name: 'host', type: 'string', description: 'Host', required: true }, + }, + }, + ]; + const errors = validateToolArgs('test_tool', { config: {} }, schema); + expect(errors).toHaveLength(1); + expect(errors[0].field).toContain('config'); + }); + + it('should validate depth 2 nested object', () => { + const schema: ToolParameter[] = [ + { + name: 'config', + type: 'object', + description: 'Config', + required: true, + properties: { + server: { + name: 'server', + type: 'object', + description: 'Server config', + required: true, + properties: { + host: { name: 'host', type: 'string', description: 'Host', required: true }, + }, + }, + }, + }, + ]; + // Missing required 'server' inside config + const errors = validateToolArgs('test_tool', { config: {} }, schema); + expect(errors).toHaveLength(1); + // AJV error path is the parent object path for required properties + expect(errors[0].field).toBe('config'); + expect(errors[0].expected).toContain('required'); + }); + + it('should validate depth 3 nested object', () => { + const schema: ToolParameter[] = [ + { + name: 'level1', + type: 'object', + description: 'L1', + required: true, + properties: { + level2: { + name: 'level2', + type: 'object', + description: 'L2', + required: true, + properties: { + level3: { + name: 'level3', + type: 'object', + description: 'L3', + required: true, + properties: { + value: { name: 'value', type: 'string', description: 'Value', required: true }, + }, + }, + }, + }, + }, + }, + ]; + // Missing required 'level3' inside level1.level2 + const errors = validateToolArgs('test_tool', { level1: { level2: {} } }, schema); + expect(errors).toHaveLength(1); + // AJV error path is the parent object path + expect(errors[0].field).toBe('level1.level2'); + expect(errors[0].expected).toContain('required'); + }); + }); + + describe('empty schema', () => { + it('should return no errors for empty schema', () => { + const errors = validateToolArgs('test_tool', {}, []); + expect(errors).toEqual([]); + }); + + it('should return no errors when no parameters defined', () => { + const errors = validateToolArgs('test_tool', { anything: 'goes' }, []); + expect(errors).toEqual([]); + }); + }); +}); + +describe('assertSchemaDepth', () => { + it('should pass for flat parameters (depth 1)', () => { + const schema: ToolParameter[] = [ + { name: 'target', type: 'string', description: 'Target', required: true }, + ]; + expect(() => assertSchemaDepth(schema)).not.toThrow(); + }); + + it('should pass for depth 2 (object with properties)', () => { + const schema: ToolParameter[] = [ + { + name: 'config', + type: 'object', + description: 'Config', + required: true, + properties: { + host: { name: 'host', type: 'string', description: 'Host', required: true }, + }, + }, + ]; + expect(() => assertSchemaDepth(schema)).not.toThrow(); + }); + + it('should pass for depth 3 (object → object → object)', () => { + const schema: ToolParameter[] = [ + { + name: 'level1', + type: 'object', + description: 'L1', + required: true, + properties: { + level2: { + name: 'level2', + type: 'object', + description: 'L2', + required: true, + properties: { + level3: { + name: 'level3', + type: 'object', + description: 'L3', + required: true, + properties: { + value: { name: 'value', type: 'string', description: 'Value', required: true }, + }, + }, + }, + }, + }, + }, + ]; + expect(() => assertSchemaDepth(schema)).not.toThrow(); + }); + + it('should throw for depth 4 (exceeds max depth)', () => { + const schema: ToolParameter[] = [ + { + name: 'level1', + type: 'object', + description: 'L1', + required: true, + properties: { + level2: { + name: 'level2', + type: 'object', + description: 'L2', + required: true, + properties: { + level3: { + name: 'level3', + type: 'object', + description: 'L3', + required: true, + properties: { + level4: { + name: 'level4', + type: 'object', + description: 'L4', + required: true, + properties: { + value: { name: 'value', type: 'string', description: 'Value', required: true }, + }, + }, + }, + }, + }, + }, + }, + }, + ]; + expect(() => assertSchemaDepth(schema)).toThrow(/exceeds maximum/); + }); + + it('should throw for depth 4 via array items', () => { + const schema: ToolParameter[] = [ + { + name: 'matrix', + type: 'array', + description: 'Matrix', + required: true, + items: { + type: 'object', + properties: { + row: { + name: 'row', + type: 'array', + description: 'Row', + required: true, + items: { + type: 'object', + properties: { + cell: { + name: 'cell', + type: 'object', + description: 'Cell', + required: true, + properties: { + value: { name: 'value', type: 'string', description: 'Value', required: true }, + }, + }, + }, + }, + }, + }, + }, + }, + ]; + expect(() => assertSchemaDepth(schema)).toThrow(/exceeds maximum/); + }); + + it('should accept custom maxDepth', () => { + const schema: ToolParameter[] = [ + { + name: 'level1', + type: 'object', + description: 'L1', + required: true, + properties: { + level2: { + name: 'level2', + type: 'object', + description: 'L2', + required: true, + properties: { + level3: { + name: 'level3', + type: 'object', + description: 'L3', + required: true, + properties: {}, + }, + }, + }, + }, + }, + ]; + // With maxDepth=2, depth 3 should throw + expect(() => assertSchemaDepth(schema, 2)).toThrow(/exceeds maximum/); + // With maxDepth=4, depth 3 should pass + expect(() => assertSchemaDepth(schema, 4)).not.toThrow(); + }); +}); + +describe('buildJsonSchema', () => { + it('should convert flat parameters to JSON Schema', () => { + const schema = buildJsonSchema([ + { name: 'target', type: 'string', description: 'Target host', required: true }, + { name: 'port', type: 'number', description: 'Port', required: false, default: 80 }, + ]); + expect(schema).toEqual({ + type: 'object', + properties: { + target: { type: 'string', description: 'Target host' }, + port: { type: 'number', description: 'Port', default: 80 }, + }, + required: ['target'], + }); + }); + + it('should convert enum parameters', () => { + const schema = buildJsonSchema([ + { name: 'method', type: 'string', description: 'HTTP method', required: true, enum: ['GET', 'POST'] }, + ]); + const props = schema.properties as Record>; + expect(props.method).toEqual({ + type: 'string', + description: 'HTTP method', + enum: ['GET', 'POST'], + }); + }); + + it('should convert array parameters with items', () => { + const schema = buildJsonSchema([ + { + name: 'domains', + type: 'array', + description: 'Domains', + required: true, + items: { type: 'string' }, + }, + ]); + const props = schema.properties as Record>; + expect(props.domains).toEqual({ + type: 'array', + description: 'Domains', + items: { type: 'string' }, + }); + }); + + it('should convert object parameters with nested properties', () => { + const schema = buildJsonSchema([ + { + name: 'config', + type: 'object', + description: 'Config', + required: false, + properties: { + host: { name: 'host', type: 'string', description: 'Host', required: true }, + }, + additionalProperties: false, + }, + ]); + const props = schema.properties as Record>; + expect(props.config).toEqual({ + type: 'object', + description: 'Config', + properties: { + host: { type: 'string', description: 'Host' }, + }, + required: ['host'], + additionalProperties: false, + }); + }); +}); diff --git a/src/agent/index.ts b/src/agent/index.ts index 6eb639d..0bc5b19 100755 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -9,6 +9,9 @@ import { EventEmitter } from 'eventemitter3'; import type { LLMBackbone } from '../llm/index.js'; import type { Arsenal } from '../arsenal/index.js'; +import { + ToolError, +} from '../types/index.js'; import type { LLMMessage, LLMToolDefinition, @@ -37,6 +40,39 @@ export interface AgentLoopOptions { verboseToolOutput?: boolean; /** Max characters per tool result before truncation (default: 4000) */ maxToolOutputLength?: number; + /** Max concurrent tool executions per LLM response (default: 5) */ + maxConcurrency?: number; +} + +// ============================================================================= +// SEMAPHORE — lightweight concurrency limiter +// ============================================================================= + +class Semaphore { + private queue: (() => void)[] = []; + private running = 0; + + constructor(private max: number) {} + + async acquire(): Promise { + if (this.running < this.max) { + this.running++; + return; + } + return new Promise((resolve) => { + this.queue.push(resolve); + }); + } + + release(): void { + if (this.queue.length > 0) { + const next = this.queue.shift()!; + // Keep running count the same — transfer the slot + next(); + } else { + this.running--; + } + } } export interface AgentStep { @@ -98,6 +134,7 @@ export class AgentLoop extends EventEmitter { tools: options?.tools ?? [], verboseToolOutput: options?.verboseToolOutput ?? true, maxToolOutputLength: options?.maxToolOutputLength ?? 4000, + maxConcurrency: Math.max(1, options?.maxConcurrency ?? 5), }; } @@ -197,23 +234,41 @@ export class AgentLoop extends EventEmitter { toolCalls: response.toolCalls, }); - // Execute each tool call + // Execute each tool call — independent calls from the same LLM response run + // concurrently, bounded by the semaphore. Scope/approval gates are checked + // per-call inside arsenal.execute() BEFORE the handler runs. const findingsBefore = allFindings.length; - for (const toolCall of response.toolCalls) { - // ANTI-STALL: a byte-identical repeat is almost always wasted — steer instead of re-running. + const semaphore = new Semaphore(this.options.maxConcurrency); + const toolSteps: AgentStep[] = new Array(response.toolCalls.length); + + // Pre-check anti-stall for all calls, then execute concurrently + const callPromises = response.toolCalls.map(async (toolCall, idx) => { const callHash = `${toolCall.name}:${JSON.stringify(toolCall.arguments || {})}`; - let toolStep: AgentStep; if (seenCalls.has(callHash)) { const dup: ToolResult = { success: false, error: `Duplicate call — you already ran ${toolCall.name} with these exact arguments. ` + `Prior result: ${seenCalls.get(callHash)}. Do NOT repeat it — change the arguments, pick a different tool, or move to your final debrief.`, }; - toolStep = { iteration: i, type: 'tool_call', toolName: toolCall.name, toolArgs: toolCall.arguments, toolResult: dup, timestamp: Date.now() }; - } else { - toolStep = await this.executeTool(toolCall, target, i); - seenCalls.set(callHash, String(toolStep.toolResult?.output || toolStep.toolResult?.error || 'no output').replace(/\s+/g, ' ').slice(0, 160)); + toolSteps[idx] = { iteration: i, type: 'tool_call', toolName: toolCall.name, toolArgs: toolCall.arguments, toolResult: dup, timestamp: Date.now() }; + return; } + + await semaphore.acquire(); + try { + toolSteps[idx] = await this.executeTool(toolCall, target, i); + seenCalls.set(callHash, String(toolSteps[idx].toolResult?.output || toolSteps[idx].toolResult?.error || 'no output').replace(/\s+/g, ' ').slice(0, 160)); + } finally { + semaphore.release(); + } + }); + + await Promise.all(callPromises); + + // Push results in original order and collect findings + for (let idx = 0; idx < response.toolCalls.length; idx++) { + const toolCall = response.toolCalls[idx]; + const toolStep = toolSteps[idx]; steps.push(toolStep); // Collect findings — tag with REAL tool provenance (the output that backs them) @@ -381,19 +436,36 @@ export class AgentLoop extends EventEmitter { parameters: toolCall.arguments, }); } catch (err) { - // A bad/hallucinated tool name must NOT crash the loop. Return the callable set so the - // model self-corrects in-place instead of dying or looping on a tool that doesn't exist. - const available = this.arsenal - .getToolDefinitions( - this.options.toolCategories.length ? this.options.toolCategories : undefined, - this.options.tools.length ? this.options.tools : undefined - ) - .map((t) => t.name); - toolResult = { - success: false, - error: `Tool "${toolCall.name}" is not available. Callable tools this run: ${available.join(', ') || '(none)'}. ` + - `Use one of these EXACT names — do not invent tools; if none fit, describe the capability you need in prose.`, - }; + // ToolError carries a category — use it for structured feedback to the LLM. + if (err instanceof ToolError) { + toolResult = { + success: false, + error: JSON.stringify({ + error: true, + category: err.category, + tool: toolCall.name, + message: `Tool execution failed: ${err.category}`, + }), + }; + } else { + // A bad/hallucinated tool name must NOT crash the loop. Return the callable set so the + // model self-corrects in-place instead of dying or looping on a tool that doesn't exist. + const available = this.arsenal + .getToolDefinitions( + this.options.toolCategories.length ? this.options.toolCategories : undefined, + this.options.tools.length ? this.options.tools : undefined + ) + .map((t) => t.name); + toolResult = { + success: false, + error: JSON.stringify({ + error: true, + category: 'tool_not_found', + tool: toolCall.name, + message: `Tool "${toolCall.name}" is not available. Callable tools this run: ${available.join(', ') || '(none)'}. Use one of these EXACT names — do not invent tools; if none fit, describe the capability you need in prose.`, + }), + }; + } } this.emit('agent:tool_result', { name: toolCall.name, result: toolResult, source }); @@ -496,11 +568,25 @@ export class AgentLoop extends EventEmitter { } /** - * Format a tool result for the LLM context + * Format a tool result for the LLM context. + * Structured JSON errors (from ToolError catches) are passed through as-is + * so the LLM receives the category without raw internals. */ private formatToolResult(result?: ToolResult): string { if (!result) return 'No result returned.'; + // Structured JSON error feedback — pass through directly + if (!result.success && result.error) { + try { + const parsed = JSON.parse(result.error); + if (parsed.error === true && parsed.category) { + return result.error; + } + } catch { + // Not JSON — fall through to normal formatting + } + } + const parts: string[] = []; parts.push(`Success: ${result.success}`); diff --git a/src/arsenal/index.ts b/src/arsenal/index.ts index 173829e..28b071a 100755 --- a/src/arsenal/index.ts +++ b/src/arsenal/index.ts @@ -25,6 +25,8 @@ import type { LLMToolDefinition, RiskTier, } from '../types/index.js'; +import { ToolError, ToolErrorCategory } from '../types/index.js'; +import { validateToolArgs, buildJsonSchema, assertSchemaDepth } from '../validation/index.js'; const dnsResolve = promisify(dns.resolve); const dnsResolve4 = promisify(dns.resolve4); @@ -317,6 +319,7 @@ export class Arsenal extends EventEmitter { * Register a tool */ register(tool: CustomTool): void { + assertSchemaDepth(tool.parameters ?? []); this.tools.set(tool.name, tool); this.emit('tool:registered', tool); } @@ -368,7 +371,13 @@ export class Arsenal extends EventEmitter { ): Promise { const tool = this.tools.get(toolName); if (!tool) { - throw new Error(`Tool "${toolName}" not found`); + const err = new ToolError( + ToolErrorCategory.ToolNotFound, + `Tool "${toolName}" not found`, + toolName, + ); + this.emit('tool:error', { tool: { name: toolName } as CustomTool, error: err }); + throw err; } // Egress scope gate: deny out-of-scope network targets BEFORE the handler runs. A tool call @@ -379,7 +388,7 @@ export class Arsenal extends EventEmitter { success: false, error: `SCOPE DENIED: target '${blockedHost}' is not in the authorized scope — ${toolName} refused before execution. Only authorized / loopback / lab targets are permitted.`, }; - this.emit('tool:error', { tool, error: new Error(denied.error) }); + this.emit('tool:error', { tool, error: new ToolError(ToolErrorCategory.ScopeDenied, denied.error!, toolName) }); return denied; } @@ -390,7 +399,7 @@ export class Arsenal extends EventEmitter { if (this.approval && isGatedRisk(tool.riskTier)) { const request: ApprovalRequest = { tool: toolName, - risk: tool.riskTier, + risk: tool.riskTier!, operator: context.operator, target: approvalTargetOf(context), action: describeToolAction(toolName, context), @@ -398,11 +407,26 @@ export class Arsenal extends EventEmitter { const decision = await this.approval.gate(request); if (!decision.allowed) { const denied: ToolResult = { success: false, error: `APPROVAL REQUIRED: ${decision.reason}` }; - this.emit('tool:error', { tool, error: new Error(denied.error) }); + this.emit('tool:error', { tool, error: new ToolError(ToolErrorCategory.ScopeDenied, denied.error!, toolName) }); return denied; } } + // Validation gate: validate args against tool's parameter schema before handler runs. + if (tool.parameters && tool.parameters.length > 0) { + const validationErrors = validateToolArgs(toolName, context.parameters, tool.parameters); + if (validationErrors.length > 0) { + const fieldErrors = validationErrors.map(e => `${e.field}: ${e.message} (expected ${e.expected})`).join('; '); + const err = new ToolError( + ToolErrorCategory.ValidationError, + `Validation failed for "${toolName}": ${fieldErrors}`, + toolName, + ); + this.emit('tool:error', { tool, error: err }); + throw err; + } + } + const execution: ToolExecution = { id: randomUUID(), toolName, @@ -429,12 +453,19 @@ export class Arsenal extends EventEmitter { error: error instanceof Error ? error.message : String(error), }).error; - this.emit('tool:error', { tool, error: error as Error }); + // If already a ToolError, re-throw as-is; otherwise wrap as ExecutionError + if (error instanceof ToolError) { + this.emit('tool:error', { tool, error }); + throw error; + } - return { - success: false, - error: execution.error, - }; + const wrappedErr = new ToolError( + ToolErrorCategory.ExecutionError, + `Tool "${toolName}" execution failed: ${execution.error}`, + toolName, + ); + this.emit('tool:error', { tool, error: wrappedErr }); + throw wrappedErr; } } @@ -469,30 +500,15 @@ export class Arsenal extends EventEmitter { tools = tools.filter(t => categories.includes(t.category)); } return tools.map(tool => { - const properties: Record = {}; - const required: string[] = []; - - for (const param of tool.parameters || []) { - properties[param.name] = { - type: param.type, - description: param.description, - }; - if (param.default !== undefined) { - properties[param.name].default = param.default; - } - if (param.required) { - required.push(param.name); - } - } + // Use buildJsonSchema() for richer schema generation (enum, items, nested properties) + const schema = (tool.parameters && tool.parameters.length > 0) + ? buildJsonSchema(tool.parameters) + : { type: 'object' as const, properties: {} }; return { name: tool.name, description: tool.description, - parameters: { - type: 'object' as const, - properties, - required: required.length > 0 ? required : undefined, - }, + parameters: schema as LLMToolDefinition['parameters'], }; }); } diff --git a/src/llm/index.ts b/src/llm/index.ts index 356304d..e51d9b3 100755 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -113,7 +113,16 @@ interface AnthropicResponse { } interface OllamaResponse { - message?: { content?: string }; + message?: { + content?: string; + /** Native function calls returned by the model (Ollama /api/chat with tools). */ + tool_calls?: Array<{ + function: { + name: string; + arguments: string | Record; + }; + }>; + }; model?: string; prompt_eval_count?: number; eval_count?: number; @@ -748,6 +757,89 @@ class LocalAdapter implements LLMProviderAdapter { return { valid: true }; } + /** Per-model probe cache: true = native tools supported, false = not. */ + private static modelToolSupport = new Map(); + + /** @internal Reset the probe cache — used by tests. */ + static __resetProbeCache(): void { + LocalAdapter.modelToolSupport.clear(); + } + + private getModelName(): string { + return this.config.model || 'default'; + } + + /** Whether to try native function-calling on the current request. */ + private shouldTryNativeTools(): boolean { + if (this.config.nativeTools === false) return false; + if (this.config.nativeTools === true) return true; + const name = this.getModelName(); + if (LocalAdapter.modelToolSupport.has(name)) { + return LocalAdapter.modelToolSupport.get(name)!; + } + // Unknown — optimistic: try native on the first call and cache the result. + return true; + } + + private cacheProbeResult(supported: boolean): void { + const name = this.getModelName(); + if (!LocalAdapter.modelToolSupport.has(name)) { + LocalAdapter.modelToolSupport.set(name, supported); + } + } + + /** Format tools for the Ollama /api/chat wire. */ + private formatOllamaTools(tools: LLMToolDefinition[]): Record[] { + return tools.map(t => ({ + type: 'function', + function: { + name: t.name, + description: t.description, + parameters: t.parameters, + }, + })); + } + + /** Format tools for the OpenAI-compatible /v1/chat/completions wire. */ + private formatOpenAITools(tools: LLMToolDefinition[]): Record[] { + return tools.map(t => ({ + type: 'function', + function: { name: t.name, description: t.description, parameters: t.parameters }, + })); + } + + /** Parse native tool_calls from an Ollama /api/chat response. */ + private parseOllamaNativeToolCalls(data: OllamaResponse): LLMToolCall[] | undefined { + const raw = data.message?.tool_calls; + if (!raw?.length) return undefined; + return raw.map((tc, i) => { + let args: Record = {}; + if (tc.function.arguments && typeof tc.function.arguments === 'object') { + args = tc.function.arguments; + } else { + try { args = JSON.parse(tc.function.arguments || '{}'); } catch { /* keep empty */ } + } + return { + id: `ollama_${Date.now()}_${i}`, + name: tc.function.name, + arguments: args, + }; + }); + } + + /** Parse native tool_calls from an OpenAI-compatible /v1/chat/completions response. */ + private parseOpenAINativeToolCalls(data: { + choices?: Array<{ message?: { tool_calls?: Array<{ id: string; function: { name: string; arguments: string } }> } }>; + }): LLMToolCall[] | undefined { + const raw = data.choices?.[0]?.message?.tool_calls; + if (!raw?.length) return undefined; + return raw.map(tc => { + let args: Record = {}; + try { args = JSON.parse(tc.function.arguments || '{}'); } catch { /* keep empty */ } + return { id: tc.id, name: tc.function.name, arguments: args }; + }); + } + // A versioned base URL (/v1, /v2, /v4, …) means an OpenAI-compatible server // (/chat/completions, choices[]). Many OpenAI-compatible providers version their // API at paths other than /v1 (e.g. Zhipu/z.ai exposes /api/paas/v4), so match any @@ -791,36 +883,83 @@ class LocalAdapter implements LLMProviderAdapter { const maxTokens = options?.maxTokens || this.config.maxTokens || 4096; const temperature = options?.temperature ?? this.config.temperature ?? 0.7; - const requestBody = openaiWire + // ── Native function-calling support ────────────────────────────────── + // The text contract is ALWAYS injected (via buildMessages) so the text + // fallback is always available. On top of that we PROBE native tools: + // if the model supports them the response carries message.tool_calls, + // and we parse those instead of the text; the probe result is cached + // per-model so subsequent calls skip the probe overhead. + const tryNative = this.shouldTryNativeTools() && !!options?.tools?.length; + + const requestBody: Record = openaiWire ? { model: this.config.model, messages: wireMessages, max_tokens: maxTokens, temperature, stream: false } : { model: this.config.model, messages: wireMessages, stream: false, options: { num_predict: maxTokens, temperature } }; + if (tryNative && options?.tools) { + requestBody.tools = openaiWire + ? this.formatOpenAITools(options.tools) + : this.formatOllamaTools(options.tools); + } + try { - const response = await fetch(url, { + const headers = { + 'Content-Type': 'application/json', + // Local OpenAI-compatible servers usually ignore auth, but LM Studio & co. + // expect *some* bearer — send a dummy unless the operator set a real key. + ...(openaiWire ? { Authorization: `Bearer ${this.config.apiKey || 'local'}` } : {}), + }; + const makeSignal = () => AbortSignal.timeout(this.config.timeout || 120000); + let response = await fetch(url, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - // Local OpenAI-compatible servers usually ignore auth, but LM Studio & co. - // expect *some* bearer — send a dummy unless the operator set a real key. - ...(openaiWire ? { Authorization: `Bearer ${this.config.apiKey || 'local'}` } : {}), - }, + headers, body: JSON.stringify(requestBody), - signal: AbortSignal.timeout(this.config.timeout || 120000), + signal: makeSignal(), }); + // Some local/OpenAI-compatible servers reject the `tools` field instead + // of ignoring it. Retry once without native tools and remember that + // explicit rejection; an optimistic probe must not break text fallback. + if (!response.ok && tryNative && this.config.nativeTools !== true) { + delete requestBody.tools; + this.cacheProbeResult(false); + response = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(requestBody), + signal: makeSignal(), + }); + } + if (!response.ok) { throw new Error(`Local LLM error: ${response.status}`); } const data = await response.json() as OllamaResponse & { - choices?: Array<{ message?: { content?: string } }>; + choices?: Array<{ message?: { content?: string; tool_calls?: Array<{ id: string; function: { name: string; arguments: string } }> } }>; usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number }; }; const content = openaiWire ? (data.choices?.[0]?.message?.content || '') : (data.message?.content || ''); + // ── Native tool-call parsing (if we probed) ──────────────────────── + let nativeToolCalls: LLMToolCall[] | undefined; + if (tryNative) { + nativeToolCalls = openaiWire + ? this.parseOpenAINativeToolCalls(data) + : this.parseOllamaNativeToolCalls(data); + + // A returned native call proves support. Its absence does not prove + // lack of support—the model may legitimately answer without a tool. + if (nativeToolCalls) this.cacheProbeResult(true); + } + // Tool-calling over text: if the Arsenal was offered, parse the model's tool // requests so the ReAct loop EXECUTES them instead of abstaining on turn 0. - const toolCalls = options?.tools?.length ? parseTextToolCalls(content) : undefined; + // This is the fallback — only used when native didn't yield tool_calls. + const textToolCalls = !nativeToolCalls && options?.tools?.length + ? parseTextToolCalls(content) + : undefined; + + const toolCalls = nativeToolCalls ?? textToolCalls; const usage = openaiWire ? (data.usage @@ -1578,3 +1717,8 @@ export function createBestAvailableBackbone(): LLMBackbone { // Re-export types for convenience export type { LLMMessage, LLMResponse, LLMConfig, LLMToolDefinition, LLMToolCall } from '../types/index.js'; + +/** @internal Reset the LocalAdapter native-tool-support probe cache (test helper). */ +export function __resetLocalAdapterCache(): void { + LocalAdapter.__resetProbeCache(); +} diff --git a/src/types/index.ts b/src/types/index.ts index a9eecac..fe14b8f 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -16,6 +16,12 @@ export interface LLMConfig { maxTokens?: number; temperature?: number; timeout?: number; + /** + * When true, force native Ollama / OpenAI-compatible function calling. When + * false, always use text-based tool contract injection. When unset, probe + * on the first call and cache per-model. + */ + nativeTools?: boolean; /** * Ordered model/provider ladder to fall back to when the PRIMARY model fails for * ANY reason it can't self-recover from — hard errors after same-model retries @@ -79,6 +85,13 @@ export interface LLMToolDefinition { enum?: string[]; items?: { type: string }; default?: unknown; + properties?: Record; + required?: string[]; + additionalProperties?: boolean; }>; required?: string[]; }; @@ -391,6 +404,27 @@ export interface Message { * additionally fire a loud warning. Absent/safe/active tools are ungated. */ export type RiskTier = 'local_read' | 'passive' | 'active' | 'intrusive' | 'credential' | 'dangerous'; +/** Structured error categories for tool failures — enables LLM feedback without leaking internals. */ +export enum ToolErrorCategory { + ScopeDenied = 'scope_denied', + ToolNotFound = 'tool_not_found', + Timeout = 'timeout', + ExecutionError = 'execution_error', + ValidationError = 'validation_error', +} + +/** Structured error thrown by tool execution — carries category for LLM feedback and optional tool name. */ +export class ToolError extends Error { + constructor( + public readonly category: ToolErrorCategory, + message: string, + public readonly toolName?: string, + ) { + super(message); + this.name = 'ToolError'; + } +} + export interface CustomTool { name: string; description: string; @@ -408,6 +442,26 @@ export interface ToolParameter { description: string; required: boolean; default?: unknown; + /** Allowed values — constrains the parameter to a fixed set. */ + enum?: (string | number)[]; + /** Schema for array items (when type === 'array'). */ + items?: { + type: string; + description?: string; + properties?: Record; + }; + /** Nested object properties (when type === 'object'). */ + properties?: Record; + /** Whether additional properties are allowed on object parameters. */ + additionalProperties?: boolean; +} + +/** A single validation error produced by validateToolArgs. */ +export interface ToolValidationError { + field: string; + message: string; + received: unknown; + expected: string; } export interface ToolContext { diff --git a/src/validation/index.ts b/src/validation/index.ts new file mode 100644 index 0000000..79067cb --- /dev/null +++ b/src/validation/index.ts @@ -0,0 +1,350 @@ +/** + * Schema Validation Utility + * + * AJV-based argument validation for tool calls with schema depth enforcement. + * AJV is lazy-loaded on first call to keep startup fast. + */ + +import type { ToolParameter, ToolValidationError } from '../types/index.js'; +import { createRequire } from 'node:module'; + +// Lazy-loaded AJV instance +let ajv: any; // eslint-disable-line @typescript-eslint/no-explicit-any +const require = createRequire(import.meta.url); +function getAjv(): any { // eslint-disable-line @typescript-eslint/no-explicit-any + if (!ajv) { + const AjvModule = require('ajv'); + ajv = new AjvModule.default({ + allErrors: true, + strict: false, + useDefaults: true, + }); + } + return ajv; +} + +const MAX_SCHEMA_DEPTH = 3; + +/** + * Build a JSON Schema object from ToolParameter[] definition. + * Converts the T3MP3ST parameter format to standard JSON Schema. + */ +export function buildJsonSchema(params: ToolParameter[]): Record { + const properties: Record = {}; + const required: string[] = []; + + for (const param of params) { + const prop: Record = { + type: param.type, + description: param.description, + }; + + if (param.enum !== undefined) { + prop.enum = param.enum; + } + + if (param.default !== undefined) { + prop.default = param.default; + } + + if (param.type === 'array' && param.items) { + prop.items = buildItemsSchema(param.items); + } + + if (param.type === 'object' && param.properties) { + const nested = buildObjectProperties(param.properties); + prop.properties = nested.properties; + if (nested.required.length > 0) { + prop.required = nested.required; + } + + if (param.additionalProperties === false) { + prop.additionalProperties = false; + } + } + + if (param.required) { + required.push(param.name); + } + + properties[param.name] = prop; + } + + const schema: Record = { + type: 'object', + properties, + }; + + if (required.length > 0) { + schema.required = required; + } + + return schema; +} + +/** + * Build properties schema for object parameters (recursive). + */ +function buildObjectProperties( + properties: Record, +): { properties: Record; required: string[] } { + const props: Record = {}; + const required: string[] = []; + + for (const [key, param] of Object.entries(properties)) { + const prop: Record = { + type: param.type, + description: param.description, + }; + + if (param.enum !== undefined) { + prop.enum = param.enum; + } + + if (param.default !== undefined) { + prop.default = param.default; + } + + if (param.type === 'array' && param.items) { + prop.items = buildItemsSchema(param.items); + } + + if (param.type === 'object' && param.properties) { + const nested = buildObjectProperties(param.properties); + prop.properties = nested.properties; + if (nested.required.length > 0) { + prop.required = nested.required; + } + if (param.additionalProperties === false) { + prop.additionalProperties = false; + } + } + + if (param.required) { + required.push(key); + } + + props[key] = prop; + } + + return { properties: props, required }; +} + +/** + * Build items schema for array parameters. + */ +function buildItemsSchema(items: ToolParameter['items']): Record { + if (!items) return { type: 'string' }; + + const schema: Record = { + type: items.type, + }; + + if (items.description) { + schema.description = items.description; + } + + if (items.properties) { + const properties: Record = {}; + const required: string[] = []; + + for (const [key, param] of Object.entries(items.properties)) { + properties[key] = { + type: param.type, + description: param.description, + }; + + if (param.enum !== undefined) { + (properties[key] as Record).enum = param.enum; + } + + if (param.required) { + required.push(key); + } + } + + schema.properties = properties; + if (required.length > 0) { + schema.required = required; + } + } + + return schema; +} + +/** + * Enforce maximum schema depth. Throws if depth exceeds limit. + * Called at tool registration time to prevent overly complex schemas. + * Depth counts object/array nesting levels — leaf properties don't add depth. + */ +export function assertSchemaDepth(params: ToolParameter[], maxDepth: number = MAX_SCHEMA_DEPTH): void { + function getMaxDepth(param: ToolParameter, current: number): number { + if (param.type === 'object' && param.properties) { + // Each nested object property adds 1 to depth + let maxNested = current; + for (const nested of Object.values(param.properties)) { + // For object/array children, they are at current+1 depth + // For leaf types, they stay at current depth + const childDepth = (nested.type === 'object' || nested.type === 'array') + ? getMaxDepth(nested, current + 1) + : current; + maxNested = Math.max(maxNested, childDepth); + } + return maxNested; + } + + if (param.type === 'array' && param.items?.properties) { + // Array items type is at current+1; items' properties are at current+2 + let maxNested = current; + for (const nested of Object.values(param.items.properties)) { + const childDepth = (nested.type === 'object' || nested.type === 'array') + ? getMaxDepth(nested, current + 2) + : current + 1; + maxNested = Math.max(maxNested, childDepth); + } + return maxNested; + } + + // Leaf types (string, number, boolean) don't add depth + return current; + } + + for (const param of params) { + const depth = getMaxDepth(param, 1); + if (depth > maxDepth) { + throw new Error( + `Schema depth ${depth} exceeds maximum ${maxDepth} for parameter "${param.name}"`, + ); + } + } +} + +// Compiled schema cache to avoid re-compiling the same schemas +const compiledSchemas = new Map['compile']>(); + +function coerceModelScalars( + values: Record, + schema: ToolParameter[], +): void { + for (const param of schema) { + const value = values[param.name]; + if (typeof value === 'string' && param.type === 'number') { + const trimmed = value.trim(); + if (trimmed !== '' && Number.isFinite(Number(trimmed))) { + values[param.name] = Number(trimmed); + } + } else if (typeof value === 'string' && param.type === 'boolean') { + if (value === 'true') values[param.name] = true; + else if (value === 'false') values[param.name] = false; + } else if (value && typeof value === 'object' && !Array.isArray(value) && param.properties) { + coerceModelScalars( + value as Record, + Object.entries(param.properties).map(([name, nested]) => ({ ...nested, name })), + ); + } else if (Array.isArray(value) && param.items?.properties) { + const itemSchema = Object.entries(param.items.properties) + .map(([name, nested]) => ({ ...nested, name })); + for (const item of value) { + if (item && typeof item === 'object' && !Array.isArray(item)) { + coerceModelScalars(item as Record, itemSchema); + } + } + } + } +} + +/** + * Validate tool arguments against a parameter schema. + * Returns an array of validation errors (empty if valid). + * + * @param toolName - Name of the tool being validated (for error messages) + * @param args - The arguments to validate + * @param schema - The tool's parameter schema + * @returns Array of validation errors, empty if valid + */ +export function validateToolArgs( + _toolName: string, + args: Record, + schema: ToolParameter[], +): ToolValidationError[] { + // No schema → no validation needed + if (!schema || schema.length === 0) { + return []; + } + + // Local/text models commonly serialize numeric and boolean schema values as + // strings. Coerce only those safe directions; never stringify malformed + // numbers or objects to make an invalid call appear valid. + coerceModelScalars(args, schema); + + const jsonSchema = buildJsonSchema(schema); + const schemaKey = JSON.stringify(jsonSchema); + + let validate: any; // eslint-disable-line @typescript-eslint/no-explicit-any + if (compiledSchemas.has(schemaKey)) { + validate = compiledSchemas.get(schemaKey); + } else { + const ajvInstance = getAjv(); + validate = ajvInstance.compile(jsonSchema); + compiledSchemas.set(schemaKey, validate); + } + + const valid = validate(args); + + if (valid) { + return []; + } + + const errors: ToolValidationError[] = []; + + for (const error of validate.errors || []) { + const field = error.instancePath + ? error.instancePath.replace(/^\//, '').replace(/\//g, '.') + : error.params?.missingProperty || error.params?.additionalProperty || 'unknown'; + + errors.push({ + field, + message: error.message || 'Validation failed', + received: getFieldValue(args, field), + expected: formatExpected(error), + }); + } + + return errors; +} + +/** + * Get the value at a dotted path in an object. + */ +function getFieldValue(obj: Record, path: string): unknown { + const parts = path.split('.'); + let current: unknown = obj; + + for (const part of parts) { + if (current === null || current === undefined || typeof current !== 'object') { + return undefined; + } + current = (current as Record)[part]; + } + + return current; +} + +/** + * Format expected value from AJV error. + */ +function formatExpected(error: any): string { // eslint-disable-line @typescript-eslint/no-explicit-any + if (error.keyword === 'type') { + return error.params?.type || 'unknown type'; + } + if (error.keyword === 'enum') { + return `one of: ${(error.params?.allowedValues || []).join(', ')}`; + } + if (error.keyword === 'required') { + return `required field "${error.params?.missingProperty || ''}"`; + } + if (error.keyword === 'additionalProperties') { + return `no additional properties allowed (found "${error.params?.additionalProperty || ''}")`; + } + return error.message || 'valid value'; +}