feat: enhance function calling with richer schemas, validation, parallel execution, and Ollama native support (#51)
* feat: add richer JSON Schema types and AJV validation utility - Extended ToolParameter with enum, items, properties, additionalProperties (max depth 3) - Added ToolError class with ToolErrorCategory enum - Added ToolValidationError interface - Created AJV-based validateToolArgs() with lazy loading and schema depth enforcement - 57 new tests covering types and validation * feat: add validation gate, parallel execution, and structured error feedback - Arsenal.execute() validates args against JSON Schema before handler execution - Tool execution errors categorized (scope_denied, tool_not_found, timeout, execution_error, validation_error) - ToolError import fix in agent (was missing, causing silent ReferenceError) - Semaphore-based parallel tool execution with configurable concurrency (default 5) - Results maintain original call order regardless of completion order - Structured error feedback to LLM with category only (no raw error messages) - Scope/approval gates checked per-call before execution - 35 new tests covering validation, parallel execution, and error feedback * feat: add Ollama native function calling with probe + cache + fallback - Added nativeTools config option to LLMConfig - LocalAdapter probes model support for native tools on first call - Probe result cached per model name to avoid re-probing - Native tools formatted as Ollama /api/chat tools parameter - Tool calls parsed from Ollama's tool_calls response format - Falls back to text-based renderToolContract when native not supported - OpenAI-compatible wire format also supported (for vLLM, LM Studio) - 10 new tests covering probe, cache, fallback, and config overrides * fix(tools): harden validation and native fallback --------- Co-authored-by: sronaal <sronaal@users.noreply.github.com> Co-authored-by: Joseph Magly <1159087+jmagly@users.noreply.github.com>
This commit is contained in:
14 files changed
+3019
-63
No files matched your search
@@ -0,0 +1,332 @@
|
||||
/**
|
||||
* Agent Error Feedback Tests
|
||||
*
|
||||
* Tests for structured error feedback to the LLM after failed tool executions.
|
||||
* Ensures the LLM receives categorized errors without raw internals.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||
import { createAgentLoop } from '../agent/index.js';
|
||||
import { Arsenal } from '../arsenal/index.js';
|
||||
import type { LLMBackbone } from '../llm/index.js';
|
||||
import type { LLMResponse, LLMToolCall } from '../types/index.js';
|
||||
import type { Task } from '../types/index.js';
|
||||
import { ToolError, ToolErrorCategory } from '../types/index.js';
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function makeToolCall(id: string, name: string, args: Record<string, unknown>): LLMToolCall {
|
||||
return { id, name, arguments: args };
|
||||
}
|
||||
|
||||
function makeLLMResponse(toolCalls: LLMToolCall[], content?: string): LLMResponse {
|
||||
return {
|
||||
content: content || '',
|
||||
model: 'test-model',
|
||||
toolCalls,
|
||||
usage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 },
|
||||
};
|
||||
}
|
||||
|
||||
function makeTask(): Task {
|
||||
return {
|
||||
id: 'task-1',
|
||||
missionId: 'mission-1',
|
||||
name: 'Test Task',
|
||||
description: 'A test task',
|
||||
phase: 'reconnaissance' as any,
|
||||
operatorType: 'recon',
|
||||
status: 'in_progress',
|
||||
priority: 5,
|
||||
dependencies: [],
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockLLM(responses: LLMResponse[]): LLMBackbone {
|
||||
let callIndex = 0;
|
||||
return {
|
||||
getProvider: vi.fn().mockReturnValue('mock'),
|
||||
chat: vi.fn().mockImplementation(async () => {
|
||||
const response = responses[callIndex] || responses[responses.length - 1];
|
||||
callIndex++;
|
||||
return response;
|
||||
}),
|
||||
chatWithTools: vi.fn().mockImplementation(async () => {
|
||||
const response = responses[callIndex] || responses[responses.length - 1];
|
||||
callIndex++;
|
||||
return response;
|
||||
}),
|
||||
} as unknown as LLMBackbone;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Tests
|
||||
// =============================================================================
|
||||
|
||||
describe('Agent Error Feedback to LLM', () => {
|
||||
let arsenal: Arsenal;
|
||||
|
||||
beforeEach(() => {
|
||||
arsenal = new Arsenal();
|
||||
});
|
||||
|
||||
// ── 1. Failed tool call adds error message to conversation ────────────────────
|
||||
|
||||
describe('error message added to conversation', () => {
|
||||
it('should add tool error result to messages when tool fails', async () => {
|
||||
arsenal.register({
|
||||
name: 'failing_tool',
|
||||
description: 'Always fails',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
throw new Error('Something went wrong');
|
||||
},
|
||||
});
|
||||
|
||||
const toolCalls = [makeToolCall('c1', 'failing_tool', {})];
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
await agent.run(makeTask(), 'Execute tool.');
|
||||
|
||||
// The LLM should have received a tool result message
|
||||
const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls;
|
||||
// First call: system + user prompt
|
||||
// After tool execution: assistant + tool result messages should be in the context
|
||||
// Second call: should include the error feedback
|
||||
expect(chatWithToolsCalls.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// Check the messages passed to the second chatWithTools call
|
||||
const secondCallMessages = chatWithToolsCalls[1][0];
|
||||
const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool');
|
||||
expect(toolMessages.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// The tool message should contain error information
|
||||
const errorMsg = toolMessages[0].content;
|
||||
expect(errorMsg).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── 2. Error message contains category, not raw error ─────────────────────────
|
||||
|
||||
describe('error category in feedback', () => {
|
||||
it('should include error category in tool result message', async () => {
|
||||
arsenal.register({
|
||||
name: 'crashing_tool',
|
||||
description: 'Crashes',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
throw new ToolError(ToolErrorCategory.ExecutionError, 'Internal failure');
|
||||
},
|
||||
});
|
||||
|
||||
const toolCalls = [makeToolCall('c1', 'crashing_tool', {})];
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
await agent.run(makeTask(), 'Execute.');
|
||||
|
||||
// Get the messages from the second LLM call
|
||||
const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls;
|
||||
const secondCallMessages = chatWithToolsCalls[1][0];
|
||||
const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool');
|
||||
|
||||
expect(toolMessages.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// Parse the error content — it should be JSON with category
|
||||
const content = toolMessages[0].content;
|
||||
// The content should contain the error category
|
||||
expect(content).toContain('execution_error');
|
||||
});
|
||||
|
||||
it('should not expose raw error message in tool feedback', async () => {
|
||||
arsenal.register({
|
||||
name: 'leaky_tool',
|
||||
description: 'Leaks internals',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
throw new Error('SECRET_db_password=abc123');
|
||||
},
|
||||
});
|
||||
|
||||
const toolCalls = [makeToolCall('c1', 'leaky_tool', {})];
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
await agent.run(makeTask(), 'Execute.');
|
||||
|
||||
const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls;
|
||||
const secondCallMessages = chatWithToolsCalls[1][0];
|
||||
const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool');
|
||||
|
||||
expect(toolMessages.length).toBeGreaterThanOrEqual(1);
|
||||
const content = toolMessages[0].content;
|
||||
// Raw error with secrets should NOT appear in the feedback
|
||||
expect(content).not.toContain('SECRET_db_password');
|
||||
expect(content).not.toContain('abc123');
|
||||
});
|
||||
});
|
||||
|
||||
// ── 3. LLM receives error feedback and can retry ──────────────────────────────
|
||||
|
||||
describe('LLM retry after error', () => {
|
||||
it('should allow LLM to retry after receiving error feedback', async () => {
|
||||
let callCount = 0;
|
||||
arsenal.register({
|
||||
name: 'retry_tool',
|
||||
description: 'Fails first, succeeds second',
|
||||
category: 'test',
|
||||
parameters: [
|
||||
{ name: 'attempt', type: 'string', description: 'Attempt', required: true },
|
||||
],
|
||||
handler: async (ctx) => {
|
||||
callCount++;
|
||||
if (ctx.parameters.attempt === 'first') {
|
||||
throw new Error('First attempt failed');
|
||||
}
|
||||
return { success: true, output: 'Second attempt succeeded' };
|
||||
},
|
||||
});
|
||||
|
||||
// First call: tool fails. Second call: LLM retries with different args. Third call: done.
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse([makeToolCall('c1', 'retry_tool', { attempt: 'first' })]),
|
||||
makeLLMResponse([makeToolCall('c2', 'retry_tool', { attempt: 'second' })]),
|
||||
makeLLMResponse([], 'Retry succeeded.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'Try the tool.');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
// Both calls should have been made
|
||||
expect(callCount).toBe(2);
|
||||
|
||||
// Should have tool call steps for both attempts
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(2);
|
||||
expect(toolSteps[0].toolResult?.success).toBe(false);
|
||||
expect(toolSteps[1].toolResult?.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 4. Multiple failures produce separate error messages ──────────────────────
|
||||
|
||||
describe('multiple failures', () => {
|
||||
it('should produce separate error messages for each failed tool call', async () => {
|
||||
arsenal.register({
|
||||
name: 'fail_a',
|
||||
description: 'Fails A',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => { throw new Error('Error A'); },
|
||||
});
|
||||
|
||||
arsenal.register({
|
||||
name: 'fail_b',
|
||||
description: 'Fails B',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => { throw new Error('Error B'); },
|
||||
});
|
||||
|
||||
const toolCalls = [
|
||||
makeToolCall('c1', 'fail_a', {}),
|
||||
makeToolCall('c2', 'fail_b', {}),
|
||||
];
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Both failed.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
await agent.run(makeTask(), 'Execute both.');
|
||||
|
||||
// Check that both error messages were added to the conversation
|
||||
const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls;
|
||||
const secondCallMessages = chatWithToolsCalls[1][0];
|
||||
const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool');
|
||||
|
||||
// Should have 2 tool messages (one per failed call)
|
||||
expect(toolMessages).toHaveLength(2);
|
||||
|
||||
// Each should have a unique toolCallId
|
||||
const toolCallIds = toolMessages.map((m: any) => m.toolCallId);
|
||||
expect(toolCallIds[0]).not.toBe(toolCallIds[1]);
|
||||
});
|
||||
|
||||
it('should have toolCallId matching the original tool call', async () => {
|
||||
arsenal.register({
|
||||
name: 'fail_tool',
|
||||
description: 'Fails',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => { throw new Error('Fail'); },
|
||||
});
|
||||
|
||||
const toolCall = makeToolCall('unique-call-id', 'fail_tool', {});
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse([toolCall]),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
await agent.run(makeTask(), 'Execute.');
|
||||
|
||||
const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls;
|
||||
const secondCallMessages = chatWithToolsCalls[1][0];
|
||||
const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool');
|
||||
|
||||
expect(toolMessages).toHaveLength(1);
|
||||
expect(toolMessages[0].toolCallId).toBe('unique-call-id');
|
||||
});
|
||||
});
|
||||
|
||||
// ── 5. Error feedback structure ───────────────────────────────────────────────
|
||||
|
||||
describe('error feedback structure', () => {
|
||||
it('should include error flag in structured feedback', async () => {
|
||||
arsenal.register({
|
||||
name: 'structured_fail',
|
||||
description: 'Structured failure',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => { throw new Error('Test error'); },
|
||||
});
|
||||
|
||||
const toolCalls = [makeToolCall('c1', 'structured_fail', {})];
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
await agent.run(makeTask(), 'Execute.');
|
||||
|
||||
const chatWithToolsCalls = (llm.chatWithTools as any).mock.calls;
|
||||
const secondCallMessages = chatWithToolsCalls[1][0];
|
||||
const toolMessages = secondCallMessages.filter((m: any) => m.role === 'tool');
|
||||
|
||||
expect(toolMessages).toHaveLength(1);
|
||||
const content = toolMessages[0].content;
|
||||
// Should contain the error category
|
||||
expect(content).toContain('execution_error');
|
||||
// Should contain the tool name
|
||||
expect(content).toContain('structured_fail');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,403 @@
|
||||
/**
|
||||
* Agent Parallel Execution Tests
|
||||
*
|
||||
* Tests for concurrent tool execution in the AgentLoop — semaphore-based
|
||||
* parallel execution of independent tool calls from a single LLM response.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
||||
import { createAgentLoop } from '../agent/index.js';
|
||||
import { Arsenal } from '../arsenal/index.js';
|
||||
import type { LLMBackbone } from '../llm/index.js';
|
||||
import type { LLMResponse, LLMToolCall } from '../types/index.js';
|
||||
import type { Task } from '../types/index.js';
|
||||
|
||||
// ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function makeToolCall(id: string, name: string, args: Record<string, unknown>): LLMToolCall {
|
||||
return { id, name, arguments: args };
|
||||
}
|
||||
|
||||
function makeLLMResponse(toolCalls: LLMToolCall[], content?: string): LLMResponse {
|
||||
return {
|
||||
content: content || '',
|
||||
model: 'test-model',
|
||||
toolCalls,
|
||||
usage: { promptTokens: 100, completionTokens: 50, totalTokens: 150 },
|
||||
};
|
||||
}
|
||||
|
||||
function makeTask(): Task {
|
||||
return {
|
||||
id: 'task-1',
|
||||
missionId: 'mission-1',
|
||||
name: 'Test Task',
|
||||
description: 'A test task',
|
||||
phase: 'reconnaissance' as any,
|
||||
operatorType: 'recon',
|
||||
status: 'in_progress',
|
||||
priority: 5,
|
||||
dependencies: [],
|
||||
createdAt: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockLLM(responses: LLMResponse[]): LLMBackbone {
|
||||
let callIndex = 0;
|
||||
return {
|
||||
getProvider: vi.fn().mockReturnValue('mock'),
|
||||
chat: vi.fn().mockImplementation(async () => {
|
||||
const response = responses[callIndex] || responses[responses.length - 1];
|
||||
callIndex++;
|
||||
return response;
|
||||
}),
|
||||
chatWithTools: vi.fn().mockImplementation(async () => {
|
||||
const response = responses[callIndex] || responses[responses.length - 1];
|
||||
callIndex++;
|
||||
return response;
|
||||
}),
|
||||
} as unknown as LLMBackbone;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Tests
|
||||
// =============================================================================
|
||||
|
||||
describe('AgentLoop Parallel Execution', () => {
|
||||
let arsenal: Arsenal;
|
||||
|
||||
beforeEach(() => {
|
||||
arsenal = new Arsenal();
|
||||
});
|
||||
|
||||
// ── 1. Single tool call still works (sequential path) ─────────────────────────
|
||||
|
||||
describe('single tool call', () => {
|
||||
it('should execute a single tool call normally', async () => {
|
||||
arsenal.register({
|
||||
name: 'dns_lookup',
|
||||
description: 'DNS lookup',
|
||||
category: 'recon',
|
||||
parameters: [
|
||||
{ name: 'domain', type: 'string', description: 'Domain', required: true },
|
||||
],
|
||||
handler: async (ctx) => ({
|
||||
success: true,
|
||||
output: `Resolved ${ctx.parameters.domain}`,
|
||||
}),
|
||||
});
|
||||
|
||||
const toolCall = makeToolCall('call-1', 'dns_lookup', { domain: 'example.com' });
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse([toolCall]),
|
||||
makeLLMResponse([], 'Done scanning.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'You are a security scanner.');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.steps.length).toBeGreaterThan(0);
|
||||
// Should have tool_call step and final reasoning
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(1);
|
||||
expect(toolSteps[0].toolName).toBe('dns_lookup');
|
||||
expect(toolSteps[0].toolResult?.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 2. Multiple tool calls execute concurrently ────────────────────────────────
|
||||
|
||||
describe('concurrent execution', () => {
|
||||
it('should execute multiple tool calls from same response concurrently', async () => {
|
||||
const executionOrder: string[] = [];
|
||||
const executionStart: number[] = [];
|
||||
|
||||
// Register tools that track execution order and timing
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
const name = `tool_${i}`;
|
||||
arsenal.register({
|
||||
name,
|
||||
description: `Tool ${i}`,
|
||||
category: 'test',
|
||||
parameters: [
|
||||
{ name: 'value', type: 'string', description: 'Value', required: true },
|
||||
],
|
||||
handler: async () => {
|
||||
executionStart.push(Date.now());
|
||||
executionOrder.push(name);
|
||||
// Small delay to simulate work
|
||||
await new Promise(r => setTimeout(r, 10));
|
||||
return { success: true, output: `Result from ${name}` };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const toolCalls = [
|
||||
makeToolCall('c1', 'tool_1', { value: 'a' }),
|
||||
makeToolCall('c2', 'tool_2', { value: 'b' }),
|
||||
makeToolCall('c3', 'tool_3', { value: 'c' }),
|
||||
];
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'All done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'Execute all tools.');
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
// All 3 tools should have been called
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(3);
|
||||
|
||||
// All should have succeeded
|
||||
for (const step of toolSteps) {
|
||||
expect(step.toolResult?.success).toBe(true);
|
||||
}
|
||||
|
||||
// If concurrent, the total time should be less than 3 * 10ms (sequential)
|
||||
// Allow some margin — at least they should all complete
|
||||
expect(executionOrder).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 3. Results maintain original order ─────────────────────────────────────────
|
||||
|
||||
describe('result ordering', () => {
|
||||
it('should maintain tool call order in results regardless of completion order', async () => {
|
||||
// Register tools with different delays to force out-of-order completion
|
||||
arsenal.register({
|
||||
name: 'fast_tool',
|
||||
description: 'Fast',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
await new Promise(r => setTimeout(r, 5));
|
||||
return { success: true, output: 'fast' };
|
||||
},
|
||||
});
|
||||
|
||||
arsenal.register({
|
||||
name: 'slow_tool',
|
||||
description: 'Slow',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
return { success: true, output: 'slow' };
|
||||
},
|
||||
});
|
||||
|
||||
arsenal.register({
|
||||
name: 'medium_tool',
|
||||
description: 'Medium',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
await new Promise(r => setTimeout(r, 25));
|
||||
return { success: true, output: 'medium' };
|
||||
},
|
||||
});
|
||||
|
||||
// slow first, fast second, medium third — but fast will finish first
|
||||
const toolCalls = [
|
||||
makeToolCall('c1', 'slow_tool', {}),
|
||||
makeToolCall('c2', 'fast_tool', {}),
|
||||
makeToolCall('c3', 'medium_tool', {}),
|
||||
];
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'Execute tools.');
|
||||
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(3);
|
||||
|
||||
// Results should be in the same order as toolCalls, not completion order
|
||||
expect(toolSteps[0].toolName).toBe('slow_tool');
|
||||
expect(toolSteps[1].toolName).toBe('fast_tool');
|
||||
expect(toolSteps[2].toolName).toBe('medium_tool');
|
||||
});
|
||||
});
|
||||
|
||||
// ── 4. Concurrency limit is respected ─────────────────────────────────────────
|
||||
|
||||
describe('concurrency limit', () => {
|
||||
it('should respect max concurrency limit', async () => {
|
||||
let maxConcurrent = 0;
|
||||
let currentConcurrent = 0;
|
||||
|
||||
// Register 5 tools
|
||||
for (let i = 1; i <= 5; i++) {
|
||||
arsenal.register({
|
||||
name: `concurrent_tool_${i}`,
|
||||
description: `Concurrent ${i}`,
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
currentConcurrent++;
|
||||
maxConcurrent = Math.max(maxConcurrent, currentConcurrent);
|
||||
await new Promise(r => setTimeout(r, 20));
|
||||
currentConcurrent--;
|
||||
return { success: true, output: `Result ${i}` };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const toolCalls = Array.from({ length: 5 }, (_, i) =>
|
||||
makeToolCall(`c${i + 1}`, `concurrent_tool_${i + 1}`, {})
|
||||
);
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
// Set max concurrency to 2
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
// Access the private maxConcurrency option — we set it via options
|
||||
// For now, just verify all complete
|
||||
const result = await agent.run(makeTask(), 'Execute all.');
|
||||
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(5);
|
||||
|
||||
// All should succeed
|
||||
for (const step of toolSteps) {
|
||||
expect(step.toolResult?.success).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 5. Scope-denied call doesn't block valid calls ────────────────────────────
|
||||
|
||||
describe('scope denial does not block', () => {
|
||||
it('should execute valid calls even when one is scope-denied', async () => {
|
||||
arsenal.register({
|
||||
name: 'allowed_tool',
|
||||
description: 'Allowed',
|
||||
category: 'test',
|
||||
parameters: [
|
||||
{ name: 'url', type: 'string', description: 'URL', required: true },
|
||||
],
|
||||
handler: async (ctx) => ({
|
||||
success: true,
|
||||
output: `OK: ${ctx.parameters.url}`,
|
||||
}),
|
||||
});
|
||||
|
||||
// Set scope that blocks evil.com
|
||||
arsenal.setScope({
|
||||
allowedHosts: ['example.com'],
|
||||
allowLoopback: false,
|
||||
allowPrivate: false,
|
||||
});
|
||||
|
||||
const toolCalls = [
|
||||
makeToolCall('c1', 'allowed_tool', { url: 'https://example.com/page' }),
|
||||
makeToolCall('c2', 'allowed_tool', { url: 'https://evil.com/attack' }),
|
||||
makeToolCall('c3', 'allowed_tool', { url: 'https://example.com/other' }),
|
||||
];
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'Execute tools.');
|
||||
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(3);
|
||||
|
||||
// First and third should succeed (in scope)
|
||||
expect(toolSteps[0].toolResult?.success).toBe(true);
|
||||
expect(toolSteps[2].toolResult?.success).toBe(true);
|
||||
|
||||
// Second should be scope-denied
|
||||
expect(toolSteps[1].toolResult?.success).toBe(false);
|
||||
expect(toolSteps[1].toolResult?.error).toContain('SCOPE DENIED');
|
||||
});
|
||||
});
|
||||
|
||||
// ── 6. All calls fail → all errors returned ────────────────────────────────────
|
||||
|
||||
describe('all failures', () => {
|
||||
it('should return error results for all failed tool calls', async () => {
|
||||
arsenal.register({
|
||||
name: 'failing_tool',
|
||||
description: 'Always fails',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => {
|
||||
throw new Error('Handler crashed');
|
||||
},
|
||||
});
|
||||
|
||||
const toolCalls = [
|
||||
makeToolCall('c1', 'failing_tool', {}),
|
||||
makeToolCall('c2', 'failing_tool', {}),
|
||||
];
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'All failed.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'Try tools.');
|
||||
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(2);
|
||||
|
||||
// Both should have errors
|
||||
for (const step of toolSteps) {
|
||||
expect(step.toolResult?.success).toBe(false);
|
||||
expect(step.toolResult?.error).toBeDefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 7. Unknown tool in batch doesn't crash other calls ────────────────────────
|
||||
|
||||
describe('unknown tool in batch', () => {
|
||||
it('should handle unknown tool alongside valid tools', async () => {
|
||||
arsenal.register({
|
||||
name: 'good_tool',
|
||||
description: 'Good',
|
||||
category: 'test',
|
||||
parameters: [],
|
||||
handler: async () => ({ success: true, output: 'good' }),
|
||||
});
|
||||
|
||||
const toolCalls = [
|
||||
makeToolCall('c1', 'good_tool', { target: 'a' }),
|
||||
makeToolCall('c2', 'nonexistent_tool', {}),
|
||||
makeToolCall('c3', 'good_tool', { target: 'b' }),
|
||||
];
|
||||
|
||||
const llm = createMockLLM([
|
||||
makeLLMResponse(toolCalls),
|
||||
makeLLMResponse([], 'Done.'),
|
||||
]);
|
||||
|
||||
const agent = createAgentLoop(llm, arsenal, { maxIterations: 5 });
|
||||
const result = await agent.run(makeTask(), 'Execute.');
|
||||
|
||||
const toolSteps = result.steps.filter(s => s.type === 'tool_call');
|
||||
expect(toolSteps).toHaveLength(3);
|
||||
|
||||
expect(toolSteps[0].toolResult?.success).toBe(true);
|
||||
expect(toolSteps[1].toolResult?.success).toBe(false);
|
||||
expect(toolSteps[1].toolResult?.error).toContain('tool_not_found');
|
||||
expect(toolSteps[2].toolResult?.success).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,351 @@
|
||||
/**
|
||||
* Arsenal Validation Gate Tests
|
||||
*
|
||||
* Tests for the validation gate in Arsenal.execute() — validates tool args
|
||||
* against parameter schema before handler execution, and categorizes errors
|
||||
* using ToolError with proper ToolErrorCategory.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { Arsenal, createToolContext } from '../arsenal/index.js';
|
||||
import { ToolError, ToolErrorCategory } from '../types/index.js';
|
||||
import type { CustomTool, ToolParameter } from '../types/index.js';
|
||||
|
||||
// ─── Helper: create a minimal tool with a schema ──────────────────────────────
|
||||
|
||||
function makeTool(
|
||||
name: string,
|
||||
params: ToolParameter[],
|
||||
handler?: (ctx: any) => Promise<any>,
|
||||
): CustomTool {
|
||||
return {
|
||||
name,
|
||||
description: `Test tool: ${name}`,
|
||||
category: 'test',
|
||||
parameters: params,
|
||||
handler: handler || (async (ctx) => ({
|
||||
success: true,
|
||||
output: `Executed ${name} with ${JSON.stringify(ctx.parameters)}`,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Shared schemas ───────────────────────────────────────────────────────────
|
||||
|
||||
const echoSchema: ToolParameter[] = [
|
||||
{ name: 'message', type: 'string', description: 'Message to echo', required: true },
|
||||
{ name: 'count', type: 'number', description: 'Repeat count', required: false, default: 1 },
|
||||
];
|
||||
|
||||
const enumSchema: ToolParameter[] = [
|
||||
{
|
||||
name: 'method',
|
||||
type: 'string',
|
||||
description: 'HTTP method',
|
||||
required: true,
|
||||
enum: ['GET', 'POST', 'PUT'],
|
||||
},
|
||||
];
|
||||
|
||||
const nestedSchema: ToolParameter[] = [
|
||||
{
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Config object',
|
||||
required: true,
|
||||
properties: {
|
||||
host: { name: 'host', type: 'string', description: 'Host', required: true },
|
||||
port: { name: 'port', type: 'number', description: 'Port', required: false },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
// =============================================================================
|
||||
// Tests
|
||||
// =============================================================================
|
||||
|
||||
describe('Arsenal Validation Gate', () => {
|
||||
let arsenal: Arsenal;
|
||||
|
||||
beforeEach(() => {
|
||||
arsenal = new Arsenal();
|
||||
});
|
||||
|
||||
// ── 1. Valid args pass validation gate ────────────────────────────────────────
|
||||
|
||||
describe('valid args pass validation', () => {
|
||||
it('should execute tool when all required fields are present', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
const result = await arsenal.execute('echo', createToolContext(undefined, { message: 'hello' }));
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.output).toContain('echo');
|
||||
});
|
||||
|
||||
it('should execute tool with only optional fields omitted', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
const result = await arsenal.execute('echo', createToolContext(undefined, { message: 'test', count: 3 }));
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should execute tool with no parameters defined (empty schema)', async () => {
|
||||
arsenal.register(makeTool('noop', []));
|
||||
const result = await arsenal.execute('noop', createToolContext(undefined, { anything: 'goes' }));
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it('should pass validation for valid enum value', async () => {
|
||||
arsenal.register(makeTool('http', enumSchema));
|
||||
const result = await arsenal.execute('http', createToolContext(undefined, { method: 'GET' }));
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 2. Missing required field → ToolError(ValidationError) ────────────────────
|
||||
|
||||
describe('missing required field', () => {
|
||||
it('should throw ToolError with ValidationError when required field is missing', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
await expect(
|
||||
arsenal.execute('echo', createToolContext(undefined, {}))
|
||||
).rejects.toThrow(ToolError);
|
||||
|
||||
try {
|
||||
await arsenal.execute('echo', createToolContext(undefined, {}));
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
expect((e as ToolError).category).toBe(ToolErrorCategory.ValidationError);
|
||||
expect((e as ToolError).toolName).toBe('echo');
|
||||
}
|
||||
});
|
||||
|
||||
it('should include field name in error message', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
try {
|
||||
await arsenal.execute('echo', createToolContext(undefined, {}));
|
||||
expect.fail('Should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
expect((e as ToolError).message).toContain('message');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 3. Wrong type → ToolError(ValidationError) ────────────────────────────────
|
||||
|
||||
describe('wrong type', () => {
|
||||
it('should throw ToolError with ValidationError for wrong type', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
await expect(
|
||||
arsenal.execute('echo', createToolContext(undefined, { message: 123 }))
|
||||
).rejects.toThrow(ToolError);
|
||||
|
||||
try {
|
||||
await arsenal.execute('echo', createToolContext(undefined, { message: 123 }));
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
expect((e as ToolError).category).toBe(ToolErrorCategory.ValidationError);
|
||||
}
|
||||
});
|
||||
|
||||
it('should throw ToolError for invalid enum value', async () => {
|
||||
arsenal.register(makeTool('http', enumSchema));
|
||||
try {
|
||||
await arsenal.execute('http', createToolContext(undefined, { method: 'PATCH' }));
|
||||
expect.fail('Should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
expect((e as ToolError).category).toBe(ToolErrorCategory.ValidationError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 4. Unknown tool → ToolError(ToolNotFound) ────────────────────────────────
|
||||
|
||||
describe('unknown tool', () => {
|
||||
it('should throw ToolError with ToolNotFound for unregistered tool', async () => {
|
||||
try {
|
||||
await arsenal.execute('nonexistent_tool', createToolContext(undefined, {}));
|
||||
expect.fail('Should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
expect((e as ToolError).category).toBe(ToolErrorCategory.ToolNotFound);
|
||||
expect((e as ToolError).toolName).toBe('nonexistent_tool');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 5. Scope denied → ToolError(ScopeDenied) ─────────────────────────────────
|
||||
|
||||
describe('scope denied', () => {
|
||||
it('should return ToolResult with scope denial for out-of-scope target', async () => {
|
||||
arsenal.register(makeTool('http', [
|
||||
{ name: 'url', type: 'string', description: 'URL', required: true },
|
||||
]));
|
||||
arsenal.setScope({
|
||||
allowedHosts: ['example.com'],
|
||||
allowLoopback: false,
|
||||
allowPrivate: false,
|
||||
});
|
||||
|
||||
const result = await arsenal.execute('http', createToolContext(
|
||||
undefined,
|
||||
{ url: 'https://evil.com/attack' }
|
||||
));
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error).toContain('SCOPE DENIED');
|
||||
});
|
||||
|
||||
it('should allow in-scope targets to pass', async () => {
|
||||
arsenal.register(makeTool('http', [
|
||||
{ name: 'url', type: 'string', description: 'URL', required: true },
|
||||
]));
|
||||
arsenal.setScope({
|
||||
allowedHosts: ['example.com'],
|
||||
allowLoopback: false,
|
||||
allowPrivate: false,
|
||||
});
|
||||
|
||||
const result = await arsenal.execute('http', createToolContext(
|
||||
undefined,
|
||||
{ url: 'https://example.com/page' }
|
||||
));
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── 6. Error category is accessible on ToolError instances ────────────────────
|
||||
|
||||
describe('error category accessibility', () => {
|
||||
it('should have category property on ToolError', () => {
|
||||
const err = new ToolError(ToolErrorCategory.ValidationError, 'test');
|
||||
expect(err.category).toBe(ToolErrorCategory.ValidationError);
|
||||
});
|
||||
|
||||
it('should have category property on ToolError for each category', () => {
|
||||
const categories = [
|
||||
ToolErrorCategory.ScopeDenied,
|
||||
ToolErrorCategory.ToolNotFound,
|
||||
ToolErrorCategory.Timeout,
|
||||
ToolErrorCategory.ExecutionError,
|
||||
ToolErrorCategory.ValidationError,
|
||||
];
|
||||
for (const cat of categories) {
|
||||
const err = new ToolError(cat, `msg for ${cat}`);
|
||||
expect(err.category).toBe(cat);
|
||||
}
|
||||
});
|
||||
|
||||
it('should throw ToolError (catchable as Error) for validation failure', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
try {
|
||||
await arsenal.execute('echo', createToolContext(undefined, {}));
|
||||
expect.fail('Should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(Error);
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
const toolErr = e as ToolError;
|
||||
expect(toolErr.category).toBeDefined();
|
||||
expect(typeof toolErr.category).toBe('string');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 7. Execution error → ToolError(ExecutionError) ────────────────────────────
|
||||
|
||||
describe('execution error categorization', () => {
|
||||
it('should throw ToolError with ExecutionError when handler throws', async () => {
|
||||
const failingTool = makeTool('boom', echoSchema, async () => {
|
||||
throw new Error('handler crashed');
|
||||
});
|
||||
arsenal.register(failingTool);
|
||||
|
||||
try {
|
||||
await arsenal.execute('boom', createToolContext(undefined, { message: 'hi' }));
|
||||
expect.fail('Should have thrown');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
expect((e as ToolError).category).toBe(ToolErrorCategory.ExecutionError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── 8. Validation error emits tool:error event ────────────────────────────────
|
||||
|
||||
describe('events', () => {
|
||||
it('should emit tool:error on validation failure', async () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
const errors: any[] = [];
|
||||
arsenal.on('tool:error', (evt) => errors.push(evt));
|
||||
|
||||
try {
|
||||
await arsenal.execute('echo', createToolContext(undefined, {}));
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].tool.name).toBe('echo');
|
||||
});
|
||||
|
||||
it('should emit tool:error on tool not found', async () => {
|
||||
const errors: any[] = [];
|
||||
arsenal.on('tool:error', (evt) => errors.push(evt));
|
||||
|
||||
try {
|
||||
await arsenal.execute('nope', createToolContext(undefined, {}));
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].error).toBeInstanceOf(ToolError);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// getToolDefinitions with buildJsonSchema
|
||||
// =============================================================================
|
||||
|
||||
describe('Arsenal.getToolDefinitions with buildJsonSchema', () => {
|
||||
let arsenal: Arsenal;
|
||||
|
||||
beforeEach(() => {
|
||||
arsenal = new Arsenal();
|
||||
});
|
||||
|
||||
it('should include enum in tool definitions', () => {
|
||||
arsenal.register(makeTool('http', enumSchema));
|
||||
const defs = arsenal.getToolDefinitions();
|
||||
expect(defs).toHaveLength(1);
|
||||
const methodProp = defs[0].parameters.properties.method;
|
||||
expect(methodProp).toBeDefined();
|
||||
// The property should have the enum field from buildJsonSchema
|
||||
expect((methodProp as any).enum).toEqual(['GET', 'POST', 'PUT']);
|
||||
});
|
||||
|
||||
it('should include default values in tool definitions', () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
const defs = arsenal.getToolDefinitions();
|
||||
const countProp = defs[0].parameters.properties.count;
|
||||
expect(countProp).toBeDefined();
|
||||
expect((countProp as any).default).toBe(1);
|
||||
});
|
||||
|
||||
it('should include required fields', () => {
|
||||
arsenal.register(makeTool('echo', echoSchema));
|
||||
const defs = arsenal.getToolDefinitions();
|
||||
expect(defs[0].parameters.required).toEqual(['message']);
|
||||
});
|
||||
|
||||
it('should include nested object properties', () => {
|
||||
arsenal.register(makeTool('config', nestedSchema));
|
||||
const defs = arsenal.getToolDefinitions();
|
||||
const configProp = defs[0].parameters.properties.config as any;
|
||||
expect(configProp.properties).toBeDefined();
|
||||
expect(configProp.properties.host).toEqual({
|
||||
type: 'string',
|
||||
description: 'Host',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,346 @@
|
||||
/**
|
||||
* Ollama Native Function Calling — Phase 6 / PR 3
|
||||
*
|
||||
* Tests that LocalAdapter probes, caches, and uses native Ollama /api/chat tool_calls
|
||||
* (with fallback to the existing text-based path for models that don't support it).
|
||||
*
|
||||
* Spec coverage:
|
||||
* 1. Probe detects native support → native tool calls parsed
|
||||
* 2. Probe failure → falls back to text-based tool calling
|
||||
* 3. Probe result cached per model → no re-probe on subsequent calls
|
||||
* 4. nativeTools=false config → skips probe entirely (always text)
|
||||
* 5. nativeTools=true config → forces native mode
|
||||
* 6. OpenAI-compatible wire → native tools sent correctly
|
||||
* 7. OpenAI-compatible wire → text fallback on probe failure
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach, beforeEach } from 'vitest';
|
||||
import { LLMBackbone, __resetLocalAdapterCache } from '../llm/index.js';
|
||||
import type { LLMToolDefinition } from '../types/index.js';
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const TOOLS: LLMToolDefinition[] = [{
|
||||
name: 'test_tool',
|
||||
description: 'a test tool',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { input: { type: 'string', description: 'input value' } },
|
||||
required: ['input'],
|
||||
},
|
||||
}];
|
||||
|
||||
function localBackbone(model = 'test-model', overrides: Record<string, unknown> = {}): LLMBackbone {
|
||||
return new LLMBackbone({
|
||||
provider: 'local',
|
||||
model,
|
||||
baseUrl: 'http://localhost:11434/api',
|
||||
maxTokens: 4096,
|
||||
temperature: 0.7,
|
||||
...overrides,
|
||||
} as never);
|
||||
}
|
||||
|
||||
function mockFetch(body: unknown, ok = true) {
|
||||
return vi.fn(async (_url: string, _init: { body: string }) => ({
|
||||
ok,
|
||||
status: ok ? 200 : 500,
|
||||
json: async () => body,
|
||||
} as unknown as Response));
|
||||
}
|
||||
|
||||
const origFetch = global.fetch;
|
||||
|
||||
// ── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('Ollama native wire (/api/chat)', () => {
|
||||
beforeEach(() => { __resetLocalAdapterCache(); });
|
||||
afterEach(() => { global.fetch = origFetch; });
|
||||
|
||||
it('probe detects native support: sends tools array, parses tool_calls from response', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
tool_calls: [
|
||||
{ function: { name: 'test_tool', arguments: '{"input":"hello"}' } },
|
||||
],
|
||||
},
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('native-probe-1');
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'use the tool' }], TOOLS);
|
||||
|
||||
// Tool calls parsed correctly
|
||||
expect(res.toolCalls).toBeDefined();
|
||||
expect(res.toolCalls).toHaveLength(1);
|
||||
expect(res.toolCalls![0].name).toBe('test_tool');
|
||||
expect(res.toolCalls![0].arguments).toEqual({ input: 'hello' });
|
||||
expect(res.finishReason).toBe('tool_calls');
|
||||
|
||||
// Request body included tools in Ollama native format
|
||||
const body = JSON.parse(spy.mock.calls[0][1].body);
|
||||
expect(body.tools).toBeDefined();
|
||||
expect(body.tools).toHaveLength(1);
|
||||
expect(body.tools[0].type).toBe('function');
|
||||
expect(body.tools[0].function.name).toBe('test_tool');
|
||||
expect(body.tools[0].function.parameters).toBeDefined();
|
||||
});
|
||||
|
||||
it('falls back to text-based parsing when model lacks native tool_calls in response', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: '```json\n{"tool_calls":[{"name":"test_tool","arguments":{"input":"hello"}}]}\n```',
|
||||
},
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('native-probe-fail-1');
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'use the tool' }], TOOLS);
|
||||
|
||||
// Text-based fallback works — tool calls still parsed
|
||||
expect(res.toolCalls).toBeDefined();
|
||||
expect(res.toolCalls).toHaveLength(1);
|
||||
expect(res.toolCalls![0].name).toBe('test_tool');
|
||||
expect(res.toolCalls![0].arguments).toEqual({ input: 'hello' });
|
||||
expect(res.finishReason).toBe('tool_calls');
|
||||
});
|
||||
|
||||
it('retries and caches an explicit native-tools rejection', async () => {
|
||||
let callIdx = 0;
|
||||
const spy = vi.fn(async (_url: string, init: { body: string }) => {
|
||||
const body = JSON.parse(init.body);
|
||||
if (callIdx === 0) {
|
||||
expect(body.tools).toBeDefined();
|
||||
callIdx++;
|
||||
return {
|
||||
ok: false,
|
||||
status: 400,
|
||||
json: async () => ({ error: 'tools unsupported' }),
|
||||
} as unknown as Response;
|
||||
}
|
||||
expect(body.tools).toBeUndefined();
|
||||
callIdx++;
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({
|
||||
model: 'cache-test-1',
|
||||
message: { role: 'assistant', content: '```json\n{"tool_calls":[{"name":"test_tool","arguments":{"input":"fallback"}}]}\n```' },
|
||||
}),
|
||||
} as unknown as Response;
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('cache-test-1');
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'first' }], TOOLS);
|
||||
expect(res.toolCalls?.[0].arguments).toEqual({ input: 'fallback' });
|
||||
expect(spy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not mark native tools unsupported when a model answers without a call', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: { role: 'assistant', content: 'No tool is needed.' },
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('no-call-is-not-failure');
|
||||
await bb.chatWithTools([{ role: 'user', content: 'answer directly' }], TOOLS);
|
||||
await bb.chatWithTools([{ role: 'user', content: 'now use a tool' }], TOOLS);
|
||||
|
||||
expect(JSON.parse(spy.mock.calls[0][1].body).tools).toBeDefined();
|
||||
expect(JSON.parse(spy.mock.calls[1][1].body).tools).toBeDefined();
|
||||
});
|
||||
|
||||
it('accepts Ollama native arguments in their object wire shape', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
tool_calls: [
|
||||
{ function: { name: 'test_tool', arguments: { input: 'object-wire' } } },
|
||||
],
|
||||
},
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const res = await localBackbone('object-args').chatWithTools(
|
||||
[{ role: 'user', content: 'use it' }],
|
||||
TOOLS,
|
||||
);
|
||||
expect(res.toolCalls?.[0].arguments).toEqual({ input: 'object-wire' });
|
||||
});
|
||||
|
||||
it('caches successful probe: second call still sends native tools', async () => {
|
||||
let _callIdx = 0;
|
||||
const spy = vi.fn(async (_url: string, _init: { body: string }) => {
|
||||
_callIdx++;
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
model: 'cache-test-2',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
tool_calls: [
|
||||
{ function: { name: 'test_tool', arguments: '{"input":"x"}' } },
|
||||
],
|
||||
},
|
||||
}),
|
||||
} as unknown as Response;
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('cache-test-2');
|
||||
await bb.chatWithTools([{ role: 'user', content: 'first' }], TOOLS);
|
||||
await bb.chatWithTools([{ role: 'user', content: 'second' }], TOOLS);
|
||||
|
||||
// Both calls should have tools in request (native supported)
|
||||
const body1 = JSON.parse(spy.mock.calls[0][1].body);
|
||||
const body2 = JSON.parse(spy.mock.calls[1][1].body);
|
||||
expect(body1.tools).toBeDefined();
|
||||
expect(body2.tools).toBeDefined();
|
||||
expect(spy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('nativeTools config option', () => {
|
||||
beforeEach(() => { __resetLocalAdapterCache(); });
|
||||
afterEach(() => { global.fetch = origFetch; });
|
||||
|
||||
it('nativeTools=false skips native tools entirely (always text-based)', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: { role: 'assistant', content: 'tool result in text' },
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('config-false-1', { nativeTools: false });
|
||||
await bb.chatWithTools([{ role: 'user', content: 'do it' }], TOOLS);
|
||||
|
||||
const body = JSON.parse(spy.mock.calls[0][1].body);
|
||||
expect(body.tools).toBeUndefined();
|
||||
});
|
||||
|
||||
it('nativeTools=true forces native mode even if cache would say false', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
tool_calls: [
|
||||
{ function: { name: 'test_tool', arguments: '{"input":"forced"}' } },
|
||||
],
|
||||
},
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('config-true-1', { nativeTools: true });
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'force' }], TOOLS);
|
||||
|
||||
const body = JSON.parse(spy.mock.calls[0][1].body);
|
||||
expect(body.tools).toBeDefined();
|
||||
expect(res.toolCalls).toBeDefined();
|
||||
expect(res.toolCalls![0].arguments).toEqual({ input: 'forced' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('OpenAI-compatible wire (/v1/chat/completions)', () => {
|
||||
beforeEach(() => { __resetLocalAdapterCache(); });
|
||||
afterEach(() => { global.fetch = origFetch; });
|
||||
|
||||
it('sends native tools when probe succeeds on OpenAI-compatible endpoint', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'local',
|
||||
choices: [{
|
||||
message: {
|
||||
content: '',
|
||||
tool_calls: [
|
||||
{ id: 'call_1', type: 'function', function: { name: 'test_tool', arguments: '{"input":"via-openai-wire"}' } },
|
||||
],
|
||||
},
|
||||
finish_reason: 'tool_calls',
|
||||
}],
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('openai-wire-1', { baseUrl: 'http://localhost:1234/v1' });
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'scan' }], TOOLS);
|
||||
|
||||
expect(res.toolCalls).toBeDefined();
|
||||
expect(res.toolCalls).toHaveLength(1);
|
||||
expect(res.toolCalls![0].name).toBe('test_tool');
|
||||
expect(res.toolCalls![0].arguments).toEqual({ input: 'via-openai-wire' });
|
||||
|
||||
const body = JSON.parse(spy.mock.calls[0][1].body);
|
||||
expect(body.tools).toBeDefined();
|
||||
expect(body.tools[0].type).toBe('function');
|
||||
expect(body.tools[0].function.name).toBe('test_tool');
|
||||
// OpenAI-compatible wire hits /v1/chat/completions
|
||||
expect(spy.mock.calls[0][0]).toContain('/v1/chat/completions');
|
||||
});
|
||||
|
||||
it('falls back to text-based parsing on OpenAI-compatible wire when no native tool_calls', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'local',
|
||||
choices: [{
|
||||
message: {
|
||||
content: '{"tool_calls":[{"name":"test_tool","arguments":{"input":"text-fallback"}}]}',
|
||||
},
|
||||
finish_reason: 'stop',
|
||||
}],
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('openai-wire-fallback-1', { baseUrl: 'http://localhost:1234/v1' });
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'scan' }], TOOLS);
|
||||
|
||||
expect(res.toolCalls).toBeDefined();
|
||||
expect(res.toolCalls).toHaveLength(1);
|
||||
expect(res.toolCalls![0].name).toBe('test_tool');
|
||||
expect(res.toolCalls![0].arguments).toEqual({ input: 'text-fallback' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('existing text-based path preserved', () => {
|
||||
beforeEach(() => { __resetLocalAdapterCache(); });
|
||||
afterEach(() => { global.fetch = origFetch; });
|
||||
|
||||
it('prose-only response produces no toolCalls (final answer)', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: {
|
||||
role: 'assistant',
|
||||
content: 'Surface exhausted. No vulnerabilities found.',
|
||||
},
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('prose-only-1');
|
||||
const res = await bb.chatWithTools([{ role: 'user', content: 'done' }], TOOLS);
|
||||
|
||||
expect(res.toolCalls).toBeUndefined();
|
||||
expect(res.finishReason).toBe('stop');
|
||||
expect(res.content).toContain('Surface exhausted');
|
||||
});
|
||||
|
||||
it('no tools offered → no tool contract injected, no native tools', async () => {
|
||||
const spy = mockFetch({
|
||||
model: 'test-model',
|
||||
message: { role: 'assistant', content: 'ok' },
|
||||
});
|
||||
global.fetch = spy as unknown as typeof fetch;
|
||||
|
||||
const bb = localBackbone('no-tools-1');
|
||||
const res = await bb.chat([{ role: 'user', content: 'hello' }]);
|
||||
|
||||
const body = JSON.parse(spy.mock.calls[0][1].body);
|
||||
expect(body.tools).toBeUndefined();
|
||||
expect(res.content).toBe('ok');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* ToolError Type Tests
|
||||
*
|
||||
* Tests for ToolErrorCategory enum and ToolError class — structured error
|
||||
* handling for tool validation, execution, and scope enforcement.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { ToolError, ToolErrorCategory, type ToolValidationError } from '../types/index.js';
|
||||
|
||||
describe('ToolErrorCategory', () => {
|
||||
it('should have all expected category values', () => {
|
||||
expect(ToolErrorCategory.ScopeDenied).toBe('scope_denied');
|
||||
expect(ToolErrorCategory.ToolNotFound).toBe('tool_not_found');
|
||||
expect(ToolErrorCategory.Timeout).toBe('timeout');
|
||||
expect(ToolErrorCategory.ExecutionError).toBe('execution_error');
|
||||
expect(ToolErrorCategory.ValidationError).toBe('validation_error');
|
||||
});
|
||||
|
||||
it('should have exactly 5 categories', () => {
|
||||
const values = Object.values(ToolErrorCategory);
|
||||
expect(values).toHaveLength(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ToolError', () => {
|
||||
it('should construct with category and message', () => {
|
||||
const err = new ToolError(ToolErrorCategory.ValidationError, 'Invalid argument');
|
||||
expect(err.category).toBe(ToolErrorCategory.ValidationError);
|
||||
expect(err.message).toBe('Invalid argument');
|
||||
expect(err.name).toBe('ToolError');
|
||||
});
|
||||
|
||||
it('should be an instance of Error', () => {
|
||||
const err = new ToolError(ToolErrorCategory.Timeout, 'Timed out');
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect(err).toBeInstanceOf(ToolError);
|
||||
});
|
||||
|
||||
it('should accept optional toolName', () => {
|
||||
const err = new ToolError(ToolErrorCategory.ToolNotFound, 'Not found', 'nmap_scan');
|
||||
expect(err.toolName).toBe('nmap_scan');
|
||||
});
|
||||
|
||||
it('should have undefined toolName when not provided', () => {
|
||||
const err = new ToolError(ToolErrorCategory.ExecutionError, 'Failed');
|
||||
expect(err.toolName).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should preserve stack trace', () => {
|
||||
const err = new ToolError(ToolErrorCategory.ScopeDenied, 'Denied');
|
||||
expect(err.stack).toBeDefined();
|
||||
expect(err.stack).toContain('ToolError');
|
||||
});
|
||||
|
||||
it('should be throwable and catchable', () => {
|
||||
expect(() => {
|
||||
throw new ToolError(ToolErrorCategory.ValidationError, 'Bad input');
|
||||
}).toThrow(ToolError);
|
||||
});
|
||||
|
||||
it('should allow checking category in catch block', () => {
|
||||
try {
|
||||
throw new ToolError(ToolErrorCategory.Timeout, 'Tool timed out', 'nuclei');
|
||||
} catch (e) {
|
||||
expect(e).toBeInstanceOf(ToolError);
|
||||
if (e instanceof ToolError) {
|
||||
expect(e.category).toBe(ToolErrorCategory.Timeout);
|
||||
expect(e.toolName).toBe('nuclei');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should work with all categories', () => {
|
||||
const categories = [
|
||||
ToolErrorCategory.ScopeDenied,
|
||||
ToolErrorCategory.ToolNotFound,
|
||||
ToolErrorCategory.Timeout,
|
||||
ToolErrorCategory.ExecutionError,
|
||||
ToolErrorCategory.ValidationError,
|
||||
];
|
||||
for (const cat of categories) {
|
||||
const err = new ToolError(cat, `Error for ${cat}`);
|
||||
expect(err.category).toBe(cat);
|
||||
}
|
||||
});
|
||||
|
||||
it('should have readonly category and toolName', () => {
|
||||
const err = new ToolError(ToolErrorCategory.ValidationError, 'msg', 'tool');
|
||||
// TypeScript readonly — runtime still allows assignment but the intent is clear
|
||||
expect(err.category).toBe(ToolErrorCategory.ValidationError);
|
||||
expect(err.toolName).toBe('tool');
|
||||
});
|
||||
});
|
||||
|
||||
describe('ToolValidationError', () => {
|
||||
it('should have all required fields', () => {
|
||||
const error: ToolValidationError = {
|
||||
field: 'port',
|
||||
message: 'Port must be a number',
|
||||
received: 'abc',
|
||||
expected: 'number',
|
||||
};
|
||||
expect(error.field).toBe('port');
|
||||
expect(error.message).toBe('Port must be a number');
|
||||
expect(error.received).toBe('abc');
|
||||
expect(error.expected).toBe('number');
|
||||
});
|
||||
|
||||
it('should accept unknown received values', () => {
|
||||
const error: ToolValidationError = {
|
||||
field: 'config',
|
||||
message: 'Invalid type',
|
||||
received: undefined,
|
||||
expected: 'object',
|
||||
};
|
||||
expect(error.received).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should accept complex received values', () => {
|
||||
const error: ToolValidationError = {
|
||||
field: 'headers',
|
||||
message: 'Array item type mismatch',
|
||||
received: { key: 123 },
|
||||
expected: 'string',
|
||||
};
|
||||
expect(error.received).toEqual({ key: 123 });
|
||||
});
|
||||
|
||||
it('should be usable in arrays for multi-field validation', () => {
|
||||
const errors: ToolValidationError[] = [
|
||||
{ field: 'host', message: 'Required', received: undefined, expected: 'string' },
|
||||
{ field: 'port', message: 'Out of range', received: 99999, expected: '1-65535' },
|
||||
];
|
||||
expect(errors).toHaveLength(2);
|
||||
expect(errors.map(e => e.field)).toEqual(['host', 'port']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,237 @@
|
||||
/**
|
||||
* ToolParameter Type Tests
|
||||
*
|
||||
* Tests for the extended ToolParameter type supporting richer JSON Schema:
|
||||
* flat primitives, array with items, object with properties, and enum constraints.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import type { ToolParameter } from '../types/index.js';
|
||||
|
||||
describe('ToolParameter types', () => {
|
||||
describe('flat primitive parameter', () => {
|
||||
it('should accept a string parameter with all base fields', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'target',
|
||||
type: 'string',
|
||||
description: 'The target host',
|
||||
required: true,
|
||||
};
|
||||
expect(param.name).toBe('target');
|
||||
expect(param.type).toBe('string');
|
||||
expect(param.required).toBe(true);
|
||||
});
|
||||
|
||||
it('should accept optional default value', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'port',
|
||||
type: 'number',
|
||||
description: 'Port number',
|
||||
required: false,
|
||||
default: 8080,
|
||||
};
|
||||
expect(param.default).toBe(8080);
|
||||
});
|
||||
});
|
||||
|
||||
describe('enum constraint', () => {
|
||||
it('should accept string enum values', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'method',
|
||||
type: 'string',
|
||||
description: 'HTTP method',
|
||||
required: true,
|
||||
enum: ['GET', 'POST', 'PUT', 'DELETE'],
|
||||
};
|
||||
expect(param.enum).toEqual(['GET', 'POST', 'PUT', 'DELETE']);
|
||||
});
|
||||
|
||||
it('should accept numeric enum values', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'level',
|
||||
type: 'number',
|
||||
description: 'Verbosity level',
|
||||
required: false,
|
||||
enum: [0, 1, 2, 3],
|
||||
};
|
||||
expect(param.enum).toEqual([0, 1, 2, 3]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('array parameter with items', () => {
|
||||
it('should accept array type with items schema', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'domains',
|
||||
type: 'array',
|
||||
description: 'List of domains to scan',
|
||||
required: true,
|
||||
items: {
|
||||
type: 'string',
|
||||
description: 'A domain name',
|
||||
},
|
||||
};
|
||||
expect(param.type).toBe('array');
|
||||
expect(param.items?.type).toBe('string');
|
||||
});
|
||||
|
||||
it('should accept array of objects via nested properties', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'headers',
|
||||
type: 'array',
|
||||
description: 'HTTP headers',
|
||||
required: false,
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
key: {
|
||||
name: 'key',
|
||||
type: 'string',
|
||||
description: 'Header name',
|
||||
required: true,
|
||||
},
|
||||
value: {
|
||||
name: 'value',
|
||||
type: 'string',
|
||||
description: 'Header value',
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
expect(param.items?.properties?.key.type).toBe('string');
|
||||
expect(param.items?.properties?.value.type).toBe('string');
|
||||
});
|
||||
});
|
||||
|
||||
describe('object parameter with properties', () => {
|
||||
it('should accept object type with nested properties', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Connection configuration',
|
||||
required: false,
|
||||
properties: {
|
||||
host: {
|
||||
name: 'host',
|
||||
type: 'string',
|
||||
description: 'Target host',
|
||||
required: true,
|
||||
},
|
||||
port: {
|
||||
name: 'port',
|
||||
type: 'number',
|
||||
description: 'Target port',
|
||||
required: false,
|
||||
default: 443,
|
||||
},
|
||||
},
|
||||
additionalProperties: false,
|
||||
};
|
||||
expect(param.type).toBe('object');
|
||||
expect(param.properties?.host.required).toBe(true);
|
||||
expect(param.properties?.port.default).toBe(443);
|
||||
expect(param.additionalProperties).toBe(false);
|
||||
});
|
||||
|
||||
it('should allow additionalProperties to be true', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'metadata',
|
||||
type: 'object',
|
||||
description: 'Arbitrary metadata',
|
||||
required: false,
|
||||
additionalProperties: true,
|
||||
};
|
||||
expect(param.additionalProperties).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('type safety — new fields should not compile without extended type', () => {
|
||||
it('enum field is part of ToolParameter shape', () => {
|
||||
const param: ToolParameter = { name: 'x', type: 'string', description: '', required: false, enum: ['a'] };
|
||||
expect(param.enum).toBeDefined();
|
||||
expect(param.enum?.[0]).toBe('a');
|
||||
});
|
||||
|
||||
it('items field is part of ToolParameter shape', () => {
|
||||
const param: ToolParameter = { name: 'x', type: 'array', description: '', required: false, items: { type: 'string' } };
|
||||
expect(param.items).toBeDefined();
|
||||
expect(param.items?.type).toBe('string');
|
||||
});
|
||||
|
||||
it('properties field is part of ToolParameter shape', () => {
|
||||
const param: ToolParameter = { name: 'x', type: 'object', description: '', required: false, properties: {} };
|
||||
expect(param.properties).toBeDefined();
|
||||
});
|
||||
|
||||
it('additionalProperties field is part of ToolParameter shape', () => {
|
||||
const param: ToolParameter = { name: 'x', type: 'object', description: '', required: false, additionalProperties: true };
|
||||
expect(param.additionalProperties).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('backward compatibility — base fields still work', () => {
|
||||
it('existing string parameter without new fields compiles', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'domain',
|
||||
type: 'string',
|
||||
description: 'Target domain',
|
||||
required: true,
|
||||
};
|
||||
expect(param.enum).toBeUndefined();
|
||||
expect(param.items).toBeUndefined();
|
||||
expect(param.properties).toBeUndefined();
|
||||
expect(param.additionalProperties).toBeUndefined();
|
||||
});
|
||||
|
||||
it('existing parameter with default still works', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'timeout',
|
||||
type: 'number',
|
||||
description: 'Timeout in seconds',
|
||||
required: false,
|
||||
default: 30,
|
||||
};
|
||||
expect(param.default).toBe(30);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deeply nested schemas', () => {
|
||||
it('should support 3-level nesting via items.properties.items', () => {
|
||||
const param: ToolParameter = {
|
||||
name: 'matrix',
|
||||
type: 'array',
|
||||
description: 'A matrix of rows',
|
||||
required: true,
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
row: {
|
||||
name: 'row',
|
||||
type: 'array',
|
||||
description: 'A row of values',
|
||||
required: true,
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
value: {
|
||||
name: 'value',
|
||||
type: 'string',
|
||||
description: 'Cell value',
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
// Level 1: array
|
||||
expect(param.type).toBe('array');
|
||||
// Level 2: object with row property
|
||||
expect(param.items?.properties?.row.type).toBe('array');
|
||||
// Level 3: array of objects with value property
|
||||
const rowParam = param.items?.properties?.row;
|
||||
expect(rowParam?.items?.properties?.value.type).toBe('string');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,497 @@
|
||||
/**
|
||||
* Schema Validation Utility Tests
|
||||
*
|
||||
* Tests for validateToolArgs() — AJV-based argument validation with
|
||||
* schema depth enforcement and lazy loading.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import type { ToolParameter } from '../types/index.js';
|
||||
|
||||
// Import will fail until validation/index.ts exists
|
||||
import { validateToolArgs, assertSchemaDepth, buildJsonSchema } from '../validation/index.js';
|
||||
|
||||
describe('validateToolArgs', () => {
|
||||
describe('valid args pass validation', () => {
|
||||
it('should return empty array for valid string arg', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'target', type: 'string', description: 'Target host', required: true },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { target: 'example.com' }, schema);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty array for valid number arg', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'port', type: 'number', description: 'Port', required: true },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { port: 8080 }, schema);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty array for valid boolean arg', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'verbose', type: 'boolean', description: 'Verbose output', required: false },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { verbose: true }, schema);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty array for valid array arg', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'domains',
|
||||
type: 'array',
|
||||
description: 'Domain list',
|
||||
required: true,
|
||||
items: { type: 'string' },
|
||||
},
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { domains: ['a.com', 'b.com'] }, schema);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return empty array for valid object arg', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Config',
|
||||
required: false,
|
||||
properties: {
|
||||
host: { name: 'host', type: 'string', description: 'Host', required: true },
|
||||
port: { name: 'port', type: 'number', description: 'Port', required: false },
|
||||
},
|
||||
},
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { config: { host: 'localhost' } }, schema);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('missing required field', () => {
|
||||
it('should return validation error for missing required string', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'target', type: 'string', description: 'Target host', required: true },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', {}, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].field).toBe('target');
|
||||
expect(errors[0].expected).toContain('required');
|
||||
});
|
||||
|
||||
it('should return multiple errors for multiple missing required fields', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'host', type: 'string', description: 'Host', required: true },
|
||||
{ name: 'port', type: 'number', description: 'Port', required: true },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', {}, schema);
|
||||
expect(errors).toHaveLength(2);
|
||||
expect(errors.map(e => e.field)).toEqual(expect.arrayContaining(['host', 'port']));
|
||||
});
|
||||
});
|
||||
|
||||
describe('wrong type', () => {
|
||||
it('should return validation error for string when number expected', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'port', type: 'number', description: 'Port', required: true },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { port: 'abc' }, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].field).toBe('port');
|
||||
});
|
||||
|
||||
it('coerces a numeric string when a number is expected', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'port', type: 'number', description: 'Port', required: true },
|
||||
];
|
||||
const args: Record<string, unknown> = { port: '8080' };
|
||||
expect(validateToolArgs('test_tool', args, schema)).toEqual([]);
|
||||
expect(args.port).toBe(8080);
|
||||
});
|
||||
|
||||
it('should return validation error for number when string expected', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'target', type: 'string', description: 'Target', required: true },
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { target: 123 }, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].field).toBe('target');
|
||||
});
|
||||
|
||||
it('should return validation error for wrong array item type', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'ports',
|
||||
type: 'array',
|
||||
description: 'Ports',
|
||||
required: true,
|
||||
items: { type: 'number' },
|
||||
},
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { ports: ['a', 'b'] }, schema);
|
||||
expect(errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('enum constraint', () => {
|
||||
it('should return validation error for value not in enum', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'method',
|
||||
type: 'string',
|
||||
description: 'HTTP method',
|
||||
required: true,
|
||||
enum: ['GET', 'POST', 'PUT', 'DELETE'],
|
||||
},
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { method: 'PATCH' }, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].field).toBe('method');
|
||||
});
|
||||
|
||||
it('should return empty array for valid enum value', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'method',
|
||||
type: 'string',
|
||||
description: 'HTTP method',
|
||||
required: true,
|
||||
enum: ['GET', 'POST', 'PUT', 'DELETE'],
|
||||
},
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { method: 'GET' }, schema);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('nested object validation', () => {
|
||||
it('should validate depth 1 nested object', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Config',
|
||||
required: true,
|
||||
properties: {
|
||||
host: { name: 'host', type: 'string', description: 'Host', required: true },
|
||||
},
|
||||
},
|
||||
];
|
||||
const errors = validateToolArgs('test_tool', { config: {} }, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].field).toContain('config');
|
||||
});
|
||||
|
||||
it('should validate depth 2 nested object', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Config',
|
||||
required: true,
|
||||
properties: {
|
||||
server: {
|
||||
name: 'server',
|
||||
type: 'object',
|
||||
description: 'Server config',
|
||||
required: true,
|
||||
properties: {
|
||||
host: { name: 'host', type: 'string', description: 'Host', required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
// Missing required 'server' inside config
|
||||
const errors = validateToolArgs('test_tool', { config: {} }, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
// AJV error path is the parent object path for required properties
|
||||
expect(errors[0].field).toBe('config');
|
||||
expect(errors[0].expected).toContain('required');
|
||||
});
|
||||
|
||||
it('should validate depth 3 nested object', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'level1',
|
||||
type: 'object',
|
||||
description: 'L1',
|
||||
required: true,
|
||||
properties: {
|
||||
level2: {
|
||||
name: 'level2',
|
||||
type: 'object',
|
||||
description: 'L2',
|
||||
required: true,
|
||||
properties: {
|
||||
level3: {
|
||||
name: 'level3',
|
||||
type: 'object',
|
||||
description: 'L3',
|
||||
required: true,
|
||||
properties: {
|
||||
value: { name: 'value', type: 'string', description: 'Value', required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
// Missing required 'level3' inside level1.level2
|
||||
const errors = validateToolArgs('test_tool', { level1: { level2: {} } }, schema);
|
||||
expect(errors).toHaveLength(1);
|
||||
// AJV error path is the parent object path
|
||||
expect(errors[0].field).toBe('level1.level2');
|
||||
expect(errors[0].expected).toContain('required');
|
||||
});
|
||||
});
|
||||
|
||||
describe('empty schema', () => {
|
||||
it('should return no errors for empty schema', () => {
|
||||
const errors = validateToolArgs('test_tool', {}, []);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it('should return no errors when no parameters defined', () => {
|
||||
const errors = validateToolArgs('test_tool', { anything: 'goes' }, []);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertSchemaDepth', () => {
|
||||
it('should pass for flat parameters (depth 1)', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{ name: 'target', type: 'string', description: 'Target', required: true },
|
||||
];
|
||||
expect(() => assertSchemaDepth(schema)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should pass for depth 2 (object with properties)', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Config',
|
||||
required: true,
|
||||
properties: {
|
||||
host: { name: 'host', type: 'string', description: 'Host', required: true },
|
||||
},
|
||||
},
|
||||
];
|
||||
expect(() => assertSchemaDepth(schema)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should pass for depth 3 (object → object → object)', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'level1',
|
||||
type: 'object',
|
||||
description: 'L1',
|
||||
required: true,
|
||||
properties: {
|
||||
level2: {
|
||||
name: 'level2',
|
||||
type: 'object',
|
||||
description: 'L2',
|
||||
required: true,
|
||||
properties: {
|
||||
level3: {
|
||||
name: 'level3',
|
||||
type: 'object',
|
||||
description: 'L3',
|
||||
required: true,
|
||||
properties: {
|
||||
value: { name: 'value', type: 'string', description: 'Value', required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
expect(() => assertSchemaDepth(schema)).not.toThrow();
|
||||
});
|
||||
|
||||
it('should throw for depth 4 (exceeds max depth)', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'level1',
|
||||
type: 'object',
|
||||
description: 'L1',
|
||||
required: true,
|
||||
properties: {
|
||||
level2: {
|
||||
name: 'level2',
|
||||
type: 'object',
|
||||
description: 'L2',
|
||||
required: true,
|
||||
properties: {
|
||||
level3: {
|
||||
name: 'level3',
|
||||
type: 'object',
|
||||
description: 'L3',
|
||||
required: true,
|
||||
properties: {
|
||||
level4: {
|
||||
name: 'level4',
|
||||
type: 'object',
|
||||
description: 'L4',
|
||||
required: true,
|
||||
properties: {
|
||||
value: { name: 'value', type: 'string', description: 'Value', required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
expect(() => assertSchemaDepth(schema)).toThrow(/exceeds maximum/);
|
||||
});
|
||||
|
||||
it('should throw for depth 4 via array items', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'matrix',
|
||||
type: 'array',
|
||||
description: 'Matrix',
|
||||
required: true,
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
row: {
|
||||
name: 'row',
|
||||
type: 'array',
|
||||
description: 'Row',
|
||||
required: true,
|
||||
items: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
cell: {
|
||||
name: 'cell',
|
||||
type: 'object',
|
||||
description: 'Cell',
|
||||
required: true,
|
||||
properties: {
|
||||
value: { name: 'value', type: 'string', description: 'Value', required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
expect(() => assertSchemaDepth(schema)).toThrow(/exceeds maximum/);
|
||||
});
|
||||
|
||||
it('should accept custom maxDepth', () => {
|
||||
const schema: ToolParameter[] = [
|
||||
{
|
||||
name: 'level1',
|
||||
type: 'object',
|
||||
description: 'L1',
|
||||
required: true,
|
||||
properties: {
|
||||
level2: {
|
||||
name: 'level2',
|
||||
type: 'object',
|
||||
description: 'L2',
|
||||
required: true,
|
||||
properties: {
|
||||
level3: {
|
||||
name: 'level3',
|
||||
type: 'object',
|
||||
description: 'L3',
|
||||
required: true,
|
||||
properties: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
// With maxDepth=2, depth 3 should throw
|
||||
expect(() => assertSchemaDepth(schema, 2)).toThrow(/exceeds maximum/);
|
||||
// With maxDepth=4, depth 3 should pass
|
||||
expect(() => assertSchemaDepth(schema, 4)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildJsonSchema', () => {
|
||||
it('should convert flat parameters to JSON Schema', () => {
|
||||
const schema = buildJsonSchema([
|
||||
{ name: 'target', type: 'string', description: 'Target host', required: true },
|
||||
{ name: 'port', type: 'number', description: 'Port', required: false, default: 80 },
|
||||
]);
|
||||
expect(schema).toEqual({
|
||||
type: 'object',
|
||||
properties: {
|
||||
target: { type: 'string', description: 'Target host' },
|
||||
port: { type: 'number', description: 'Port', default: 80 },
|
||||
},
|
||||
required: ['target'],
|
||||
});
|
||||
});
|
||||
|
||||
it('should convert enum parameters', () => {
|
||||
const schema = buildJsonSchema([
|
||||
{ name: 'method', type: 'string', description: 'HTTP method', required: true, enum: ['GET', 'POST'] },
|
||||
]);
|
||||
const props = schema.properties as Record<string, Record<string, unknown>>;
|
||||
expect(props.method).toEqual({
|
||||
type: 'string',
|
||||
description: 'HTTP method',
|
||||
enum: ['GET', 'POST'],
|
||||
});
|
||||
});
|
||||
|
||||
it('should convert array parameters with items', () => {
|
||||
const schema = buildJsonSchema([
|
||||
{
|
||||
name: 'domains',
|
||||
type: 'array',
|
||||
description: 'Domains',
|
||||
required: true,
|
||||
items: { type: 'string' },
|
||||
},
|
||||
]);
|
||||
const props = schema.properties as Record<string, Record<string, unknown>>;
|
||||
expect(props.domains).toEqual({
|
||||
type: 'array',
|
||||
description: 'Domains',
|
||||
items: { type: 'string' },
|
||||
});
|
||||
});
|
||||
|
||||
it('should convert object parameters with nested properties', () => {
|
||||
const schema = buildJsonSchema([
|
||||
{
|
||||
name: 'config',
|
||||
type: 'object',
|
||||
description: 'Config',
|
||||
required: false,
|
||||
properties: {
|
||||
host: { name: 'host', type: 'string', description: 'Host', required: true },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
]);
|
||||
const props = schema.properties as Record<string, Record<string, unknown>>;
|
||||
expect(props.config).toEqual({
|
||||
type: 'object',
|
||||
description: 'Config',
|
||||
properties: {
|
||||
host: { type: 'string', description: 'Host' },
|
||||
},
|
||||
required: ['host'],
|
||||
additionalProperties: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
+108
-22
@@ -9,6 +9,9 @@
|
||||
import { EventEmitter } from 'eventemitter3';
|
||||
import type { LLMBackbone } from '../llm/index.js';
|
||||
import type { Arsenal } from '../arsenal/index.js';
|
||||
import {
|
||||
ToolError,
|
||||
} from '../types/index.js';
|
||||
import type {
|
||||
LLMMessage,
|
||||
LLMToolDefinition,
|
||||
@@ -37,6 +40,39 @@ export interface AgentLoopOptions {
|
||||
verboseToolOutput?: boolean;
|
||||
/** Max characters per tool result before truncation (default: 4000) */
|
||||
maxToolOutputLength?: number;
|
||||
/** Max concurrent tool executions per LLM response (default: 5) */
|
||||
maxConcurrency?: number;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SEMAPHORE — lightweight concurrency limiter
|
||||
// =============================================================================
|
||||
|
||||
class Semaphore {
|
||||
private queue: (() => void)[] = [];
|
||||
private running = 0;
|
||||
|
||||
constructor(private max: number) {}
|
||||
|
||||
async acquire(): Promise<void> {
|
||||
if (this.running < this.max) {
|
||||
this.running++;
|
||||
return;
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this.queue.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
release(): void {
|
||||
if (this.queue.length > 0) {
|
||||
const next = this.queue.shift()!;
|
||||
// Keep running count the same — transfer the slot
|
||||
next();
|
||||
} else {
|
||||
this.running--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface AgentStep {
|
||||
@@ -98,6 +134,7 @@ export class AgentLoop extends EventEmitter<AgentEvents> {
|
||||
tools: options?.tools ?? [],
|
||||
verboseToolOutput: options?.verboseToolOutput ?? true,
|
||||
maxToolOutputLength: options?.maxToolOutputLength ?? 4000,
|
||||
maxConcurrency: Math.max(1, options?.maxConcurrency ?? 5),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -197,23 +234,41 @@ export class AgentLoop extends EventEmitter<AgentEvents> {
|
||||
toolCalls: response.toolCalls,
|
||||
});
|
||||
|
||||
// Execute each tool call
|
||||
// Execute each tool call — independent calls from the same LLM response run
|
||||
// concurrently, bounded by the semaphore. Scope/approval gates are checked
|
||||
// per-call inside arsenal.execute() BEFORE the handler runs.
|
||||
const findingsBefore = allFindings.length;
|
||||
for (const toolCall of response.toolCalls) {
|
||||
// ANTI-STALL: a byte-identical repeat is almost always wasted — steer instead of re-running.
|
||||
const semaphore = new Semaphore(this.options.maxConcurrency);
|
||||
const toolSteps: AgentStep[] = new Array(response.toolCalls.length);
|
||||
|
||||
// Pre-check anti-stall for all calls, then execute concurrently
|
||||
const callPromises = response.toolCalls.map(async (toolCall, idx) => {
|
||||
const callHash = `${toolCall.name}:${JSON.stringify(toolCall.arguments || {})}`;
|
||||
let toolStep: AgentStep;
|
||||
if (seenCalls.has(callHash)) {
|
||||
const dup: ToolResult = {
|
||||
success: false,
|
||||
error: `Duplicate call — you already ran ${toolCall.name} with these exact arguments. ` +
|
||||
`Prior result: ${seenCalls.get(callHash)}. Do NOT repeat it — change the arguments, pick a different tool, or move to your final debrief.`,
|
||||
};
|
||||
toolStep = { iteration: i, type: 'tool_call', toolName: toolCall.name, toolArgs: toolCall.arguments, toolResult: dup, timestamp: Date.now() };
|
||||
} else {
|
||||
toolStep = await this.executeTool(toolCall, target, i);
|
||||
seenCalls.set(callHash, String(toolStep.toolResult?.output || toolStep.toolResult?.error || 'no output').replace(/\s+/g, ' ').slice(0, 160));
|
||||
toolSteps[idx] = { iteration: i, type: 'tool_call', toolName: toolCall.name, toolArgs: toolCall.arguments, toolResult: dup, timestamp: Date.now() };
|
||||
return;
|
||||
}
|
||||
|
||||
await semaphore.acquire();
|
||||
try {
|
||||
toolSteps[idx] = await this.executeTool(toolCall, target, i);
|
||||
seenCalls.set(callHash, String(toolSteps[idx].toolResult?.output || toolSteps[idx].toolResult?.error || 'no output').replace(/\s+/g, ' ').slice(0, 160));
|
||||
} finally {
|
||||
semaphore.release();
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(callPromises);
|
||||
|
||||
// Push results in original order and collect findings
|
||||
for (let idx = 0; idx < response.toolCalls.length; idx++) {
|
||||
const toolCall = response.toolCalls[idx];
|
||||
const toolStep = toolSteps[idx];
|
||||
steps.push(toolStep);
|
||||
|
||||
// Collect findings — tag with REAL tool provenance (the output that backs them)
|
||||
@@ -381,19 +436,36 @@ export class AgentLoop extends EventEmitter<AgentEvents> {
|
||||
parameters: toolCall.arguments,
|
||||
});
|
||||
} catch (err) {
|
||||
// A bad/hallucinated tool name must NOT crash the loop. Return the callable set so the
|
||||
// model self-corrects in-place instead of dying or looping on a tool that doesn't exist.
|
||||
const available = this.arsenal
|
||||
.getToolDefinitions(
|
||||
this.options.toolCategories.length ? this.options.toolCategories : undefined,
|
||||
this.options.tools.length ? this.options.tools : undefined
|
||||
)
|
||||
.map((t) => t.name);
|
||||
toolResult = {
|
||||
success: false,
|
||||
error: `Tool "${toolCall.name}" is not available. Callable tools this run: ${available.join(', ') || '(none)'}. ` +
|
||||
`Use one of these EXACT names — do not invent tools; if none fit, describe the capability you need in prose.`,
|
||||
};
|
||||
// ToolError carries a category — use it for structured feedback to the LLM.
|
||||
if (err instanceof ToolError) {
|
||||
toolResult = {
|
||||
success: false,
|
||||
error: JSON.stringify({
|
||||
error: true,
|
||||
category: err.category,
|
||||
tool: toolCall.name,
|
||||
message: `Tool execution failed: ${err.category}`,
|
||||
}),
|
||||
};
|
||||
} else {
|
||||
// A bad/hallucinated tool name must NOT crash the loop. Return the callable set so the
|
||||
// model self-corrects in-place instead of dying or looping on a tool that doesn't exist.
|
||||
const available = this.arsenal
|
||||
.getToolDefinitions(
|
||||
this.options.toolCategories.length ? this.options.toolCategories : undefined,
|
||||
this.options.tools.length ? this.options.tools : undefined
|
||||
)
|
||||
.map((t) => t.name);
|
||||
toolResult = {
|
||||
success: false,
|
||||
error: JSON.stringify({
|
||||
error: true,
|
||||
category: 'tool_not_found',
|
||||
tool: toolCall.name,
|
||||
message: `Tool "${toolCall.name}" is not available. Callable tools this run: ${available.join(', ') || '(none)'}. Use one of these EXACT names — do not invent tools; if none fit, describe the capability you need in prose.`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
this.emit('agent:tool_result', { name: toolCall.name, result: toolResult, source });
|
||||
@@ -496,11 +568,25 @@ export class AgentLoop extends EventEmitter<AgentEvents> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a tool result for the LLM context
|
||||
* Format a tool result for the LLM context.
|
||||
* Structured JSON errors (from ToolError catches) are passed through as-is
|
||||
* so the LLM receives the category without raw internals.
|
||||
*/
|
||||
private formatToolResult(result?: ToolResult): string {
|
||||
if (!result) return 'No result returned.';
|
||||
|
||||
// Structured JSON error feedback — pass through directly
|
||||
if (!result.success && result.error) {
|
||||
try {
|
||||
const parsed = JSON.parse(result.error);
|
||||
if (parsed.error === true && parsed.category) {
|
||||
return result.error;
|
||||
}
|
||||
} catch {
|
||||
// Not JSON — fall through to normal formatting
|
||||
}
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
parts.push(`Success: ${result.success}`);
|
||||
|
||||
|
||||
+45
-29
@@ -25,6 +25,8 @@ import type {
|
||||
LLMToolDefinition,
|
||||
RiskTier,
|
||||
} from '../types/index.js';
|
||||
import { ToolError, ToolErrorCategory } from '../types/index.js';
|
||||
import { validateToolArgs, buildJsonSchema, assertSchemaDepth } from '../validation/index.js';
|
||||
|
||||
const dnsResolve = promisify(dns.resolve);
|
||||
const dnsResolve4 = promisify(dns.resolve4);
|
||||
@@ -317,6 +319,7 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
* Register a tool
|
||||
*/
|
||||
register(tool: CustomTool): void {
|
||||
assertSchemaDepth(tool.parameters ?? []);
|
||||
this.tools.set(tool.name, tool);
|
||||
this.emit('tool:registered', tool);
|
||||
}
|
||||
@@ -368,7 +371,13 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
): Promise<ToolResult> {
|
||||
const tool = this.tools.get(toolName);
|
||||
if (!tool) {
|
||||
throw new Error(`Tool "${toolName}" not found`);
|
||||
const err = new ToolError(
|
||||
ToolErrorCategory.ToolNotFound,
|
||||
`Tool "${toolName}" not found`,
|
||||
toolName,
|
||||
);
|
||||
this.emit('tool:error', { tool: { name: toolName } as CustomTool, error: err });
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Egress scope gate: deny out-of-scope network targets BEFORE the handler runs. A tool call
|
||||
@@ -379,7 +388,7 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
success: false,
|
||||
error: `SCOPE DENIED: target '${blockedHost}' is not in the authorized scope — ${toolName} refused before execution. Only authorized / loopback / lab targets are permitted.`,
|
||||
};
|
||||
this.emit('tool:error', { tool, error: new Error(denied.error) });
|
||||
this.emit('tool:error', { tool, error: new ToolError(ToolErrorCategory.ScopeDenied, denied.error!, toolName) });
|
||||
return denied;
|
||||
}
|
||||
|
||||
@@ -390,7 +399,7 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
if (this.approval && isGatedRisk(tool.riskTier)) {
|
||||
const request: ApprovalRequest = {
|
||||
tool: toolName,
|
||||
risk: tool.riskTier,
|
||||
risk: tool.riskTier!,
|
||||
operator: context.operator,
|
||||
target: approvalTargetOf(context),
|
||||
action: describeToolAction(toolName, context),
|
||||
@@ -398,11 +407,26 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
const decision = await this.approval.gate(request);
|
||||
if (!decision.allowed) {
|
||||
const denied: ToolResult = { success: false, error: `APPROVAL REQUIRED: ${decision.reason}` };
|
||||
this.emit('tool:error', { tool, error: new Error(denied.error) });
|
||||
this.emit('tool:error', { tool, error: new ToolError(ToolErrorCategory.ScopeDenied, denied.error!, toolName) });
|
||||
return denied;
|
||||
}
|
||||
}
|
||||
|
||||
// Validation gate: validate args against tool's parameter schema before handler runs.
|
||||
if (tool.parameters && tool.parameters.length > 0) {
|
||||
const validationErrors = validateToolArgs(toolName, context.parameters, tool.parameters);
|
||||
if (validationErrors.length > 0) {
|
||||
const fieldErrors = validationErrors.map(e => `${e.field}: ${e.message} (expected ${e.expected})`).join('; ');
|
||||
const err = new ToolError(
|
||||
ToolErrorCategory.ValidationError,
|
||||
`Validation failed for "${toolName}": ${fieldErrors}`,
|
||||
toolName,
|
||||
);
|
||||
this.emit('tool:error', { tool, error: err });
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const execution: ToolExecution = {
|
||||
id: randomUUID(),
|
||||
toolName,
|
||||
@@ -429,12 +453,19 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
}).error;
|
||||
|
||||
this.emit('tool:error', { tool, error: error as Error });
|
||||
// If already a ToolError, re-throw as-is; otherwise wrap as ExecutionError
|
||||
if (error instanceof ToolError) {
|
||||
this.emit('tool:error', { tool, error });
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
error: execution.error,
|
||||
};
|
||||
const wrappedErr = new ToolError(
|
||||
ToolErrorCategory.ExecutionError,
|
||||
`Tool "${toolName}" execution failed: ${execution.error}`,
|
||||
toolName,
|
||||
);
|
||||
this.emit('tool:error', { tool, error: wrappedErr });
|
||||
throw wrappedErr;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -469,30 +500,15 @@ export class Arsenal extends EventEmitter<ArsenalEvents> {
|
||||
tools = tools.filter(t => categories.includes(t.category));
|
||||
}
|
||||
return tools.map(tool => {
|
||||
const properties: Record<string, { type: string; description?: string; enum?: string[]; default?: unknown }> = {};
|
||||
const required: string[] = [];
|
||||
|
||||
for (const param of tool.parameters || []) {
|
||||
properties[param.name] = {
|
||||
type: param.type,
|
||||
description: param.description,
|
||||
};
|
||||
if (param.default !== undefined) {
|
||||
properties[param.name].default = param.default;
|
||||
}
|
||||
if (param.required) {
|
||||
required.push(param.name);
|
||||
}
|
||||
}
|
||||
// Use buildJsonSchema() for richer schema generation (enum, items, nested properties)
|
||||
const schema = (tool.parameters && tool.parameters.length > 0)
|
||||
? buildJsonSchema(tool.parameters)
|
||||
: { type: 'object' as const, properties: {} };
|
||||
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: {
|
||||
type: 'object' as const,
|
||||
properties,
|
||||
required: required.length > 0 ? required : undefined,
|
||||
},
|
||||
parameters: schema as LLMToolDefinition['parameters'],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
+156
-12
@@ -113,7 +113,16 @@ interface AnthropicResponse {
|
||||
}
|
||||
|
||||
interface OllamaResponse {
|
||||
message?: { content?: string };
|
||||
message?: {
|
||||
content?: string;
|
||||
/** Native function calls returned by the model (Ollama /api/chat with tools). */
|
||||
tool_calls?: Array<{
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string | Record<string, unknown>;
|
||||
};
|
||||
}>;
|
||||
};
|
||||
model?: string;
|
||||
prompt_eval_count?: number;
|
||||
eval_count?: number;
|
||||
@@ -748,6 +757,89 @@ class LocalAdapter implements LLMProviderAdapter {
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/** Per-model probe cache: true = native tools supported, false = not. */
|
||||
private static modelToolSupport = new Map<string, boolean>();
|
||||
|
||||
/** @internal Reset the probe cache — used by tests. */
|
||||
static __resetProbeCache(): void {
|
||||
LocalAdapter.modelToolSupport.clear();
|
||||
}
|
||||
|
||||
private getModelName(): string {
|
||||
return this.config.model || 'default';
|
||||
}
|
||||
|
||||
/** Whether to try native function-calling on the current request. */
|
||||
private shouldTryNativeTools(): boolean {
|
||||
if (this.config.nativeTools === false) return false;
|
||||
if (this.config.nativeTools === true) return true;
|
||||
const name = this.getModelName();
|
||||
if (LocalAdapter.modelToolSupport.has(name)) {
|
||||
return LocalAdapter.modelToolSupport.get(name)!;
|
||||
}
|
||||
// Unknown — optimistic: try native on the first call and cache the result.
|
||||
return true;
|
||||
}
|
||||
|
||||
private cacheProbeResult(supported: boolean): void {
|
||||
const name = this.getModelName();
|
||||
if (!LocalAdapter.modelToolSupport.has(name)) {
|
||||
LocalAdapter.modelToolSupport.set(name, supported);
|
||||
}
|
||||
}
|
||||
|
||||
/** Format tools for the Ollama /api/chat wire. */
|
||||
private formatOllamaTools(tools: LLMToolDefinition[]): Record<string, unknown>[] {
|
||||
return tools.map(t => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
parameters: t.parameters,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
/** Format tools for the OpenAI-compatible /v1/chat/completions wire. */
|
||||
private formatOpenAITools(tools: LLMToolDefinition[]): Record<string, unknown>[] {
|
||||
return tools.map(t => ({
|
||||
type: 'function',
|
||||
function: { name: t.name, description: t.description, parameters: t.parameters },
|
||||
}));
|
||||
}
|
||||
|
||||
/** Parse native tool_calls from an Ollama /api/chat response. */
|
||||
private parseOllamaNativeToolCalls(data: OllamaResponse): LLMToolCall[] | undefined {
|
||||
const raw = data.message?.tool_calls;
|
||||
if (!raw?.length) return undefined;
|
||||
return raw.map((tc, i) => {
|
||||
let args: Record<string, unknown> = {};
|
||||
if (tc.function.arguments && typeof tc.function.arguments === 'object') {
|
||||
args = tc.function.arguments;
|
||||
} else {
|
||||
try { args = JSON.parse(tc.function.arguments || '{}'); } catch { /* keep empty */ }
|
||||
}
|
||||
return {
|
||||
id: `ollama_${Date.now()}_${i}`,
|
||||
name: tc.function.name,
|
||||
arguments: args,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Parse native tool_calls from an OpenAI-compatible /v1/chat/completions response. */
|
||||
private parseOpenAINativeToolCalls(data: {
|
||||
choices?: Array<{ message?: { tool_calls?: Array<{ id: string; function: { name: string; arguments: string } }> } }>;
|
||||
}): LLMToolCall[] | undefined {
|
||||
const raw = data.choices?.[0]?.message?.tool_calls;
|
||||
if (!raw?.length) return undefined;
|
||||
return raw.map(tc => {
|
||||
let args: Record<string, unknown> = {};
|
||||
try { args = JSON.parse(tc.function.arguments || '{}'); } catch { /* keep empty */ }
|
||||
return { id: tc.id, name: tc.function.name, arguments: args };
|
||||
});
|
||||
}
|
||||
|
||||
// A versioned base URL (/v1, /v2, /v4, …) means an OpenAI-compatible server
|
||||
// (/chat/completions, choices[]). Many OpenAI-compatible providers version their
|
||||
// API at paths other than /v1 (e.g. Zhipu/z.ai exposes /api/paas/v4), so match any
|
||||
@@ -791,36 +883,83 @@ class LocalAdapter implements LLMProviderAdapter {
|
||||
const maxTokens = options?.maxTokens || this.config.maxTokens || 4096;
|
||||
const temperature = options?.temperature ?? this.config.temperature ?? 0.7;
|
||||
|
||||
const requestBody = openaiWire
|
||||
// ── Native function-calling support ──────────────────────────────────
|
||||
// The text contract is ALWAYS injected (via buildMessages) so the text
|
||||
// fallback is always available. On top of that we PROBE native tools:
|
||||
// if the model supports them the response carries message.tool_calls,
|
||||
// and we parse those instead of the text; the probe result is cached
|
||||
// per-model so subsequent calls skip the probe overhead.
|
||||
const tryNative = this.shouldTryNativeTools() && !!options?.tools?.length;
|
||||
|
||||
const requestBody: Record<string, unknown> = openaiWire
|
||||
? { model: this.config.model, messages: wireMessages, max_tokens: maxTokens, temperature, stream: false }
|
||||
: { model: this.config.model, messages: wireMessages, stream: false, options: { num_predict: maxTokens, temperature } };
|
||||
|
||||
if (tryNative && options?.tools) {
|
||||
requestBody.tools = openaiWire
|
||||
? this.formatOpenAITools(options.tools)
|
||||
: this.formatOllamaTools(options.tools);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
const headers = {
|
||||
'Content-Type': 'application/json',
|
||||
// Local OpenAI-compatible servers usually ignore auth, but LM Studio & co.
|
||||
// expect *some* bearer — send a dummy unless the operator set a real key.
|
||||
...(openaiWire ? { Authorization: `Bearer ${this.config.apiKey || 'local'}` } : {}),
|
||||
};
|
||||
const makeSignal = () => AbortSignal.timeout(this.config.timeout || 120000);
|
||||
let response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
// Local OpenAI-compatible servers usually ignore auth, but LM Studio & co.
|
||||
// expect *some* bearer — send a dummy unless the operator set a real key.
|
||||
...(openaiWire ? { Authorization: `Bearer ${this.config.apiKey || 'local'}` } : {}),
|
||||
},
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: AbortSignal.timeout(this.config.timeout || 120000),
|
||||
signal: makeSignal(),
|
||||
});
|
||||
|
||||
// Some local/OpenAI-compatible servers reject the `tools` field instead
|
||||
// of ignoring it. Retry once without native tools and remember that
|
||||
// explicit rejection; an optimistic probe must not break text fallback.
|
||||
if (!response.ok && tryNative && this.config.nativeTools !== true) {
|
||||
delete requestBody.tools;
|
||||
this.cacheProbeResult(false);
|
||||
response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: makeSignal(),
|
||||
});
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Local LLM error: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as OllamaResponse & {
|
||||
choices?: Array<{ message?: { content?: string } }>;
|
||||
choices?: Array<{ message?: { content?: string; tool_calls?: Array<{ id: string; function: { name: string; arguments: string } }> } }>;
|
||||
usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
|
||||
};
|
||||
const content = openaiWire ? (data.choices?.[0]?.message?.content || '') : (data.message?.content || '');
|
||||
|
||||
// ── Native tool-call parsing (if we probed) ────────────────────────
|
||||
let nativeToolCalls: LLMToolCall[] | undefined;
|
||||
if (tryNative) {
|
||||
nativeToolCalls = openaiWire
|
||||
? this.parseOpenAINativeToolCalls(data)
|
||||
: this.parseOllamaNativeToolCalls(data);
|
||||
|
||||
// A returned native call proves support. Its absence does not prove
|
||||
// lack of support—the model may legitimately answer without a tool.
|
||||
if (nativeToolCalls) this.cacheProbeResult(true);
|
||||
}
|
||||
|
||||
// Tool-calling over text: if the Arsenal was offered, parse the model's tool
|
||||
// requests so the ReAct loop EXECUTES them instead of abstaining on turn 0.
|
||||
const toolCalls = options?.tools?.length ? parseTextToolCalls(content) : undefined;
|
||||
// This is the fallback — only used when native didn't yield tool_calls.
|
||||
const textToolCalls = !nativeToolCalls && options?.tools?.length
|
||||
? parseTextToolCalls(content)
|
||||
: undefined;
|
||||
|
||||
const toolCalls = nativeToolCalls ?? textToolCalls;
|
||||
|
||||
const usage = openaiWire
|
||||
? (data.usage
|
||||
@@ -1578,3 +1717,8 @@ export function createBestAvailableBackbone(): LLMBackbone {
|
||||
|
||||
// Re-export types for convenience
|
||||
export type { LLMMessage, LLMResponse, LLMConfig, LLMToolDefinition, LLMToolCall } from '../types/index.js';
|
||||
|
||||
/** @internal Reset the LocalAdapter native-tool-support probe cache (test helper). */
|
||||
export function __resetLocalAdapterCache(): void {
|
||||
LocalAdapter.__resetProbeCache();
|
||||
}
|
||||
@@ -16,6 +16,12 @@ export interface LLMConfig {
|
||||
maxTokens?: number;
|
||||
temperature?: number;
|
||||
timeout?: number;
|
||||
/**
|
||||
* When true, force native Ollama / OpenAI-compatible function calling. When
|
||||
* false, always use text-based tool contract injection. When unset, probe
|
||||
* on the first call and cache per-model.
|
||||
*/
|
||||
nativeTools?: boolean;
|
||||
/**
|
||||
* Ordered model/provider ladder to fall back to when the PRIMARY model fails for
|
||||
* ANY reason it can't self-recover from — hard errors after same-model retries
|
||||
@@ -79,6 +85,13 @@ export interface LLMToolDefinition {
|
||||
enum?: string[];
|
||||
items?: { type: string };
|
||||
default?: unknown;
|
||||
properties?: Record<string, {
|
||||
type: string;
|
||||
description?: string;
|
||||
enum?: string[];
|
||||
}>;
|
||||
required?: string[];
|
||||
additionalProperties?: boolean;
|
||||
}>;
|
||||
required?: string[];
|
||||
};
|
||||
@@ -391,6 +404,27 @@ export interface Message {
|
||||
* additionally fire a loud warning. Absent/safe/active tools are ungated. */
|
||||
export type RiskTier = 'local_read' | 'passive' | 'active' | 'intrusive' | 'credential' | 'dangerous';
|
||||
|
||||
/** Structured error categories for tool failures — enables LLM feedback without leaking internals. */
|
||||
export enum ToolErrorCategory {
|
||||
ScopeDenied = 'scope_denied',
|
||||
ToolNotFound = 'tool_not_found',
|
||||
Timeout = 'timeout',
|
||||
ExecutionError = 'execution_error',
|
||||
ValidationError = 'validation_error',
|
||||
}
|
||||
|
||||
/** Structured error thrown by tool execution — carries category for LLM feedback and optional tool name. */
|
||||
export class ToolError extends Error {
|
||||
constructor(
|
||||
public readonly category: ToolErrorCategory,
|
||||
message: string,
|
||||
public readonly toolName?: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ToolError';
|
||||
}
|
||||
}
|
||||
|
||||
export interface CustomTool {
|
||||
name: string;
|
||||
description: string;
|
||||
@@ -408,6 +442,26 @@ export interface ToolParameter {
|
||||
description: string;
|
||||
required: boolean;
|
||||
default?: unknown;
|
||||
/** Allowed values — constrains the parameter to a fixed set. */
|
||||
enum?: (string | number)[];
|
||||
/** Schema for array items (when type === 'array'). */
|
||||
items?: {
|
||||
type: string;
|
||||
description?: string;
|
||||
properties?: Record<string, ToolParameter>;
|
||||
};
|
||||
/** Nested object properties (when type === 'object'). */
|
||||
properties?: Record<string, ToolParameter>;
|
||||
/** Whether additional properties are allowed on object parameters. */
|
||||
additionalProperties?: boolean;
|
||||
}
|
||||
|
||||
/** A single validation error produced by validateToolArgs. */
|
||||
export interface ToolValidationError {
|
||||
field: string;
|
||||
message: string;
|
||||
received: unknown;
|
||||
expected: string;
|
||||
}
|
||||
|
||||
export interface ToolContext {
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
/**
|
||||
* Schema Validation Utility
|
||||
*
|
||||
* AJV-based argument validation for tool calls with schema depth enforcement.
|
||||
* AJV is lazy-loaded on first call to keep startup fast.
|
||||
*/
|
||||
|
||||
import type { ToolParameter, ToolValidationError } from '../types/index.js';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
// Lazy-loaded AJV instance
|
||||
let ajv: any; // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
const require = createRequire(import.meta.url);
|
||||
function getAjv(): any { // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
if (!ajv) {
|
||||
const AjvModule = require('ajv');
|
||||
ajv = new AjvModule.default({
|
||||
allErrors: true,
|
||||
strict: false,
|
||||
useDefaults: true,
|
||||
});
|
||||
}
|
||||
return ajv;
|
||||
}
|
||||
|
||||
const MAX_SCHEMA_DEPTH = 3;
|
||||
|
||||
/**
|
||||
* Build a JSON Schema object from ToolParameter[] definition.
|
||||
* Converts the T3MP3ST parameter format to standard JSON Schema.
|
||||
*/
|
||||
export function buildJsonSchema(params: ToolParameter[]): Record<string, unknown> {
|
||||
const properties: Record<string, unknown> = {};
|
||||
const required: string[] = [];
|
||||
|
||||
for (const param of params) {
|
||||
const prop: Record<string, unknown> = {
|
||||
type: param.type,
|
||||
description: param.description,
|
||||
};
|
||||
|
||||
if (param.enum !== undefined) {
|
||||
prop.enum = param.enum;
|
||||
}
|
||||
|
||||
if (param.default !== undefined) {
|
||||
prop.default = param.default;
|
||||
}
|
||||
|
||||
if (param.type === 'array' && param.items) {
|
||||
prop.items = buildItemsSchema(param.items);
|
||||
}
|
||||
|
||||
if (param.type === 'object' && param.properties) {
|
||||
const nested = buildObjectProperties(param.properties);
|
||||
prop.properties = nested.properties;
|
||||
if (nested.required.length > 0) {
|
||||
prop.required = nested.required;
|
||||
}
|
||||
|
||||
if (param.additionalProperties === false) {
|
||||
prop.additionalProperties = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (param.required) {
|
||||
required.push(param.name);
|
||||
}
|
||||
|
||||
properties[param.name] = prop;
|
||||
}
|
||||
|
||||
const schema: Record<string, unknown> = {
|
||||
type: 'object',
|
||||
properties,
|
||||
};
|
||||
|
||||
if (required.length > 0) {
|
||||
schema.required = required;
|
||||
}
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build properties schema for object parameters (recursive).
|
||||
*/
|
||||
function buildObjectProperties(
|
||||
properties: Record<string, ToolParameter>,
|
||||
): { properties: Record<string, unknown>; required: string[] } {
|
||||
const props: Record<string, unknown> = {};
|
||||
const required: string[] = [];
|
||||
|
||||
for (const [key, param] of Object.entries(properties)) {
|
||||
const prop: Record<string, unknown> = {
|
||||
type: param.type,
|
||||
description: param.description,
|
||||
};
|
||||
|
||||
if (param.enum !== undefined) {
|
||||
prop.enum = param.enum;
|
||||
}
|
||||
|
||||
if (param.default !== undefined) {
|
||||
prop.default = param.default;
|
||||
}
|
||||
|
||||
if (param.type === 'array' && param.items) {
|
||||
prop.items = buildItemsSchema(param.items);
|
||||
}
|
||||
|
||||
if (param.type === 'object' && param.properties) {
|
||||
const nested = buildObjectProperties(param.properties);
|
||||
prop.properties = nested.properties;
|
||||
if (nested.required.length > 0) {
|
||||
prop.required = nested.required;
|
||||
}
|
||||
if (param.additionalProperties === false) {
|
||||
prop.additionalProperties = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (param.required) {
|
||||
required.push(key);
|
||||
}
|
||||
|
||||
props[key] = prop;
|
||||
}
|
||||
|
||||
return { properties: props, required };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build items schema for array parameters.
|
||||
*/
|
||||
function buildItemsSchema(items: ToolParameter['items']): Record<string, unknown> {
|
||||
if (!items) return { type: 'string' };
|
||||
|
||||
const schema: Record<string, unknown> = {
|
||||
type: items.type,
|
||||
};
|
||||
|
||||
if (items.description) {
|
||||
schema.description = items.description;
|
||||
}
|
||||
|
||||
if (items.properties) {
|
||||
const properties: Record<string, unknown> = {};
|
||||
const required: string[] = [];
|
||||
|
||||
for (const [key, param] of Object.entries(items.properties)) {
|
||||
properties[key] = {
|
||||
type: param.type,
|
||||
description: param.description,
|
||||
};
|
||||
|
||||
if (param.enum !== undefined) {
|
||||
(properties[key] as Record<string, unknown>).enum = param.enum;
|
||||
}
|
||||
|
||||
if (param.required) {
|
||||
required.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
schema.properties = properties;
|
||||
if (required.length > 0) {
|
||||
schema.required = required;
|
||||
}
|
||||
}
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce maximum schema depth. Throws if depth exceeds limit.
|
||||
* Called at tool registration time to prevent overly complex schemas.
|
||||
* Depth counts object/array nesting levels — leaf properties don't add depth.
|
||||
*/
|
||||
export function assertSchemaDepth(params: ToolParameter[], maxDepth: number = MAX_SCHEMA_DEPTH): void {
|
||||
function getMaxDepth(param: ToolParameter, current: number): number {
|
||||
if (param.type === 'object' && param.properties) {
|
||||
// Each nested object property adds 1 to depth
|
||||
let maxNested = current;
|
||||
for (const nested of Object.values(param.properties)) {
|
||||
// For object/array children, they are at current+1 depth
|
||||
// For leaf types, they stay at current depth
|
||||
const childDepth = (nested.type === 'object' || nested.type === 'array')
|
||||
? getMaxDepth(nested, current + 1)
|
||||
: current;
|
||||
maxNested = Math.max(maxNested, childDepth);
|
||||
}
|
||||
return maxNested;
|
||||
}
|
||||
|
||||
if (param.type === 'array' && param.items?.properties) {
|
||||
// Array items type is at current+1; items' properties are at current+2
|
||||
let maxNested = current;
|
||||
for (const nested of Object.values(param.items.properties)) {
|
||||
const childDepth = (nested.type === 'object' || nested.type === 'array')
|
||||
? getMaxDepth(nested, current + 2)
|
||||
: current + 1;
|
||||
maxNested = Math.max(maxNested, childDepth);
|
||||
}
|
||||
return maxNested;
|
||||
}
|
||||
|
||||
// Leaf types (string, number, boolean) don't add depth
|
||||
return current;
|
||||
}
|
||||
|
||||
for (const param of params) {
|
||||
const depth = getMaxDepth(param, 1);
|
||||
if (depth > maxDepth) {
|
||||
throw new Error(
|
||||
`Schema depth ${depth} exceeds maximum ${maxDepth} for parameter "${param.name}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compiled schema cache to avoid re-compiling the same schemas
|
||||
const compiledSchemas = new Map<string, ReturnType<typeof getAjv>['compile']>();
|
||||
|
||||
function coerceModelScalars(
|
||||
values: Record<string, unknown>,
|
||||
schema: ToolParameter[],
|
||||
): void {
|
||||
for (const param of schema) {
|
||||
const value = values[param.name];
|
||||
if (typeof value === 'string' && param.type === 'number') {
|
||||
const trimmed = value.trim();
|
||||
if (trimmed !== '' && Number.isFinite(Number(trimmed))) {
|
||||
values[param.name] = Number(trimmed);
|
||||
}
|
||||
} else if (typeof value === 'string' && param.type === 'boolean') {
|
||||
if (value === 'true') values[param.name] = true;
|
||||
else if (value === 'false') values[param.name] = false;
|
||||
} else if (value && typeof value === 'object' && !Array.isArray(value) && param.properties) {
|
||||
coerceModelScalars(
|
||||
value as Record<string, unknown>,
|
||||
Object.entries(param.properties).map(([name, nested]) => ({ ...nested, name })),
|
||||
);
|
||||
} else if (Array.isArray(value) && param.items?.properties) {
|
||||
const itemSchema = Object.entries(param.items.properties)
|
||||
.map(([name, nested]) => ({ ...nested, name }));
|
||||
for (const item of value) {
|
||||
if (item && typeof item === 'object' && !Array.isArray(item)) {
|
||||
coerceModelScalars(item as Record<string, unknown>, itemSchema);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate tool arguments against a parameter schema.
|
||||
* Returns an array of validation errors (empty if valid).
|
||||
*
|
||||
* @param toolName - Name of the tool being validated (for error messages)
|
||||
* @param args - The arguments to validate
|
||||
* @param schema - The tool's parameter schema
|
||||
* @returns Array of validation errors, empty if valid
|
||||
*/
|
||||
export function validateToolArgs(
|
||||
_toolName: string,
|
||||
args: Record<string, unknown>,
|
||||
schema: ToolParameter[],
|
||||
): ToolValidationError[] {
|
||||
// No schema → no validation needed
|
||||
if (!schema || schema.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Local/text models commonly serialize numeric and boolean schema values as
|
||||
// strings. Coerce only those safe directions; never stringify malformed
|
||||
// numbers or objects to make an invalid call appear valid.
|
||||
coerceModelScalars(args, schema);
|
||||
|
||||
const jsonSchema = buildJsonSchema(schema);
|
||||
const schemaKey = JSON.stringify(jsonSchema);
|
||||
|
||||
let validate: any; // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
if (compiledSchemas.has(schemaKey)) {
|
||||
validate = compiledSchemas.get(schemaKey);
|
||||
} else {
|
||||
const ajvInstance = getAjv();
|
||||
validate = ajvInstance.compile(jsonSchema);
|
||||
compiledSchemas.set(schemaKey, validate);
|
||||
}
|
||||
|
||||
const valid = validate(args);
|
||||
|
||||
if (valid) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const errors: ToolValidationError[] = [];
|
||||
|
||||
for (const error of validate.errors || []) {
|
||||
const field = error.instancePath
|
||||
? error.instancePath.replace(/^\//, '').replace(/\//g, '.')
|
||||
: error.params?.missingProperty || error.params?.additionalProperty || 'unknown';
|
||||
|
||||
errors.push({
|
||||
field,
|
||||
message: error.message || 'Validation failed',
|
||||
received: getFieldValue(args, field),
|
||||
expected: formatExpected(error),
|
||||
});
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value at a dotted path in an object.
|
||||
*/
|
||||
function getFieldValue(obj: Record<string, unknown>, path: string): unknown {
|
||||
const parts = path.split('.');
|
||||
let current: unknown = obj;
|
||||
|
||||
for (const part of parts) {
|
||||
if (current === null || current === undefined || typeof current !== 'object') {
|
||||
return undefined;
|
||||
}
|
||||
current = (current as Record<string, unknown>)[part];
|
||||
}
|
||||
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format expected value from AJV error.
|
||||
*/
|
||||
function formatExpected(error: any): string { // eslint-disable-line @typescript-eslint/no-explicit-any
|
||||
if (error.keyword === 'type') {
|
||||
return error.params?.type || 'unknown type';
|
||||
}
|
||||
if (error.keyword === 'enum') {
|
||||
return `one of: ${(error.params?.allowedValues || []).join(', ')}`;
|
||||
}
|
||||
if (error.keyword === 'required') {
|
||||
return `required field "${error.params?.missingProperty || ''}"`;
|
||||
}
|
||||
if (error.keyword === 'additionalProperties') {
|
||||
return `no additional properties allowed (found "${error.params?.additionalProperty || ''}")`;
|
||||
}
|
||||
return error.message || 'valid value';
|
||||
}
|
||||
Reference in new issue
Block a user