fix(llm): populate usage for the Claude Code local-agent path (#140)
* fix(llm): populate usage for the Claude Code local-agent path LocalAgentAdapter.chat() never set `usage` on its LLMResponse, so AgentLoop's token budget check (tokensUsed >= maxTokens) silently never fired for Claude Code missions. maxIterations was the only brake, and it bounds turn count, not actual spend. Claude Code's --output-format json (instead of text) returns exact per-call token counts, including prompt-cache creation/read tokens that a text-length estimate has no way to see. localAgentChat now requests that format for the claude agent id only. LocalAgentAdapter parses the envelope into usage.promptTokens/completionTokens, with a character-based estimate as a fallback if parsing fails, so usage never goes back to undefined. Verified directly against the CLI: a live run now reports promptTokens 24458, completionTokens 1662 instead of undefined/0. Full suite: 675/675, no regressions. Fixes #139 * test(llm): harden Claude usage fallback --------- Co-authored-by: Joseph Magly <1159087+jmagly@users.noreply.github.com>
This commit is contained in:
1 parent
9020e5fc08
commit
d471da6603
3 files changed
+118
-2
No files matched your search
+55
-1
@@ -1308,6 +1308,51 @@ class CodexAdapter implements LLMProviderAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
// Claude Code's `--output-format json` envelope (requested only for agentId 'claude', see
|
||||
// localAgentChat in local-agents.ts). Carries REAL per-call token accounting straight from the
|
||||
// CLI — input/output tokens plus the prompt-cache creation/read tokens from Claude Code's own
|
||||
// bootstrap (CLAUDE.md, skills, MCP tool manifests), which a text-length estimate has no way to
|
||||
// see and which measured 4-25k tokens on a single trivial call in testing.
|
||||
interface ClaudeJsonEnvelope {
|
||||
result?: string;
|
||||
usage?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
cache_creation_input_tokens?: number;
|
||||
cache_read_input_tokens?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse the envelope; invalid usage is omitted so the caller can retain content and estimate. */
|
||||
function parseClaudeJsonEnvelope(raw: string): { content: string; usage?: LLMResponse['usage'] } | null {
|
||||
let json: ClaudeJsonEnvelope;
|
||||
try {
|
||||
json = JSON.parse(raw);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof json.result !== 'string') return null;
|
||||
const u = json.usage || {};
|
||||
const tokenValues = [u.input_tokens, u.output_tokens, u.cache_creation_input_tokens, u.cache_read_input_tokens];
|
||||
if (!tokenValues.some((v) => v !== undefined)
|
||||
|| tokenValues.some((v) => v !== undefined && (!Number.isFinite(v) || v < 0))) {
|
||||
return { content: json.result };
|
||||
}
|
||||
const promptTokens = (u.input_tokens ?? 0) + (u.cache_creation_input_tokens ?? 0) + (u.cache_read_input_tokens ?? 0);
|
||||
const completionTokens = u.output_tokens ?? 0;
|
||||
if (promptTokens + completionTokens === 0) return { content: json.result };
|
||||
return { content: json.result, usage: { promptTokens, completionTokens, totalTokens: promptTokens + completionTokens } };
|
||||
}
|
||||
|
||||
// Character-based fallback ONLY — used when the real envelope can't be parsed (older CLI,
|
||||
// unexpected output shape). Rough on purpose: the goal is to give AgentLoop's budget check SOME
|
||||
// number instead of a permanently-zero one, not to be precise.
|
||||
function estimateUsage(promptText: string, completionText: string): LLMResponse['usage'] {
|
||||
const promptTokens = Math.ceil(promptText.length / 4);
|
||||
const completionTokens = Math.ceil(completionText.length / 4);
|
||||
return { promptTokens, completionTokens, totalTokens: promptTokens + completionTokens };
|
||||
}
|
||||
|
||||
// Generic adapter that drives a CONNECTED local agent CLI (Claude Code / Codex / Hermes) as the LLM
|
||||
// backend — NO API key needed; each CLI uses its own login. The agent id travels in `config.model`,
|
||||
// optionally with an underlying model after a `::` separator ("claude::opus", "claude::claude-opus-4-8")
|
||||
@@ -1347,7 +1392,15 @@ class LocalAgentAdapter implements LLMProviderAdapter {
|
||||
const { agentId, agentModel } = this.parseAgentSpec();
|
||||
const prompt = this.formatPrompt(messages, options);
|
||||
const timeoutMs = typeof this.config.timeout === 'number' && this.config.timeout > 0 ? this.config.timeout : undefined;
|
||||
const content = (await localAgentChat(agentId, prompt, { model: agentModel, timeoutMs })).trim();
|
||||
const raw = (await localAgentChat(agentId, prompt, { model: agentModel, timeoutMs })).trim();
|
||||
// Claude requests --output-format json (see local-agents.ts) so this parses to REAL usage.
|
||||
// Anything else (parse failure, or a non-claude agent) falls back to the raw text — claude
|
||||
// additionally gets a character-based usage ESTIMATE so its budget check is never blind again.
|
||||
const parsed = agentId === 'claude' ? parseClaudeJsonEnvelope(raw) : null;
|
||||
const content = parsed ? parsed.content : raw;
|
||||
const usage = agentId === 'claude'
|
||||
? (parsed?.usage ?? estimateUsage(prompt, parsed?.content ?? raw))
|
||||
: undefined;
|
||||
// Tool-calling over text: if the Arsenal was offered, parse the agent's tool requests so the
|
||||
// ReAct loop EXECUTES them instead of treating this planning turn as the (abstaining) final answer.
|
||||
const toolCalls = options?.tools?.length ? parseTextToolCalls(content) : undefined;
|
||||
@@ -1356,6 +1409,7 @@ class LocalAgentAdapter implements LLMProviderAdapter {
|
||||
model: `local-agent:${agentId}${agentModel ? '/' + agentModel : ''}`,
|
||||
finishReason: toolCalls?.length ? 'tool_calls' : 'stop',
|
||||
toolCalls,
|
||||
usage,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user