diff --git a/src/__tests__/litellm-provider.test.ts b/src/__tests__/litellm-provider.test.ts new file mode 100644 index 0000000..d22f09d --- /dev/null +++ b/src/__tests__/litellm-provider.test.ts @@ -0,0 +1,44 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { config, AVAILABLE_MODELS } from '../config/index.js'; +import { createLiteLLMBackbone } from '../llm/index.js'; + +const PREV_BASE = process.env.LITELLM_BASE_URL; +const PREV_KEY = process.env.LITELLM_API_KEY; +const PREV_MODEL = process.env.LITELLM_MODEL; + +describe('LiteLLM provider wiring (#45)', () => { + afterEach(() => { + if (PREV_BASE === undefined) delete process.env.LITELLM_BASE_URL; + else process.env.LITELLM_BASE_URL = PREV_BASE; + if (PREV_KEY === undefined) delete process.env.LITELLM_API_KEY; + else process.env.LITELLM_API_KEY = PREV_KEY; + if (PREV_MODEL === undefined) delete process.env.LITELLM_MODEL; + else process.env.LITELLM_MODEL = PREV_MODEL; + }); + + it('resolves LiteLLM as an OpenAI-compatible proxy from env without requiring a key', () => { + process.env.LITELLM_BASE_URL = 'http://localhost:4000/v1'; + delete process.env.LITELLM_API_KEY; + process.env.LITELLM_MODEL = 'anthropic/claude-sonnet-4.5'; + + const cfg = config.getLLMConfig('litellm'); + expect(cfg.provider).toBe('litellm'); + expect(cfg.baseUrl).toBe('http://localhost:4000/v1'); + expect(cfg.model).toBe('anthropic/claude-sonnet-4.5'); + expect(cfg.apiKey).toBeUndefined(); + expect(config.getConfiguredProviders()).toContain('litellm'); + }); + + it('the LiteLLM backbone validates when only a proxy base URL is configured', () => { + process.env.LITELLM_BASE_URL = 'http://localhost:4000/v1'; + const backbone = createLiteLLMBackbone(); + + expect(backbone.getProvider()).toBe('litellm'); + expect(backbone.validateConfig().valid).toBe(true); + }); + + it('surfaces a static LiteLLM model catalog entry', () => { + expect(AVAILABLE_MODELS.litellm?.length ?? 0).toBeGreaterThan(0); + expect(AVAILABLE_MODELS.litellm[0].provider).toBe('LiteLLM'); + }); +}); diff --git a/src/config/index.ts b/src/config/index.ts index b97fded..417e647 100755 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -11,6 +11,8 @@ import { join } from 'path'; import { existsSync, readFileSync, writeFileSync } from 'fs'; import type { LLMProvider, LLMConfig, FallbackEntry, OpsecLevel } from '../types/index.js'; +type ApiKeyProvider = 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'litellm' | 'local'; + // ============================================================================= // CONFIGURATION SCHEMA // ============================================================================= @@ -24,6 +26,7 @@ export interface TempestSettings { openai?: string; xai?: string; gemini?: string; + litellm?: string; local?: string; }; @@ -63,6 +66,12 @@ export interface TempestSettings { defaultModel: string; }; + // LiteLLM AI gateway proxy — routes to 100+ LLM providers via unified API + litellm: { + baseUrl: string; + defaultModel: string; + }; + // Google Gemini — native Gemini API via its OpenAI-compatible endpoint gemini: { baseUrl: string; @@ -132,6 +141,11 @@ const DEFAULT_SETTINGS: TempestSettings = { defaultModel: 'grok-build-0.1', }, + litellm: { + baseUrl: 'http://localhost:4000/v1', + defaultModel: 'gpt-4o', + }, + // Gemini's OpenAI-compatible surface lives under /v1beta/openai. The OpenAIAdapter posts to // `${baseUrl}/chat/completions`, so the /openai path segment is REQUIRED here — without it, // requests hit /v1beta/chat/completions, which is not a valid Gemini endpoint. @@ -397,6 +411,16 @@ export const AVAILABLE_MODELS: Record = { capabilities: ['reasoning', 'code', 'analysis', 'vision', 'fast'], }, ], + litellm: [ + { + id: 'gpt-4o', + name: 'Any model via LiteLLM proxy (default: gpt-4o)', + provider: 'LiteLLM', + contextWindow: 128000, + maxOutput: 4096, + capabilities: ['reasoning', 'code', 'analysis', 'vision', 'tools'], + }, + ], codex: [ { id: 'codex-default', @@ -544,7 +568,7 @@ class ConfigManager { const envContent = readFileSync(envPath, 'utf-8'); const lines = envContent.split('\n'); // Validation variables added - const VALID_PROVIDERS = ['openrouter', 'venice', 'anthropic', 'openai', 'xai', 'gemini', 'local']; + const VALID_PROVIDERS = ['openrouter', 'venice', 'anthropic', 'openai', 'xai', 'gemini', 'litellm', 'local']; for (const line of lines) { const trimmed = line.trim(); @@ -604,7 +628,7 @@ class ConfigManager { /** * Set an API key for a provider */ - setApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'local', key: string): void { + setApiKey(provider: ApiKeyProvider, key: string): void { const apiKeys = this.config.get('apiKeys'); apiKeys[provider] = key; this.config.set('apiKeys', apiKeys); @@ -613,7 +637,7 @@ class ConfigManager { /** * Get an API key for a provider */ - getApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'local'): string | undefined { + getApiKey(provider: ApiKeyProvider): string | undefined { // First check environment variables (highest priority) // local provider: a self-hosted/OpenAI-compatible server MAY require a bearer // (Zhipu/z.ai, Together, etc.) — accept TEMPEST_LOCAL_API_KEY or provider-specific vars. @@ -629,6 +653,7 @@ class ConfigManager { openai: 'OPENAI_API_KEY', xai: 'XAI_API_KEY', gemini: 'GEMINI_API_KEY', + litellm: 'LITELLM_API_KEY', }; // Force a fully UNCONFIGURED server (no key from env OR the saved store) — used by @@ -647,7 +672,7 @@ class ConfigManager { /** * Check if a provider has a valid API key configured */ - hasApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'local'): boolean { + hasApiKey(provider: ApiKeyProvider): boolean { const key = this.getApiKey(provider); return !!key && key.length > 10; } @@ -655,12 +680,22 @@ class ConfigManager { /** * Remove an API key */ - removeApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'local'): void { + removeApiKey(provider: ApiKeyProvider): void { const apiKeys = this.config.get('apiKeys'); delete apiKeys[provider]; this.config.set('apiKeys', apiKeys); } + /** + * Check if a LiteLLM proxy is configured (base URL set via env or config) + */ + hasLiteLLMProxy(): boolean { + const envUrl = process.env.LITELLM_BASE_URL?.trim(); + if (envUrl) return true; + const storedUrl = this.config.get('litellm')?.baseUrl; + return !!storedUrl && storedUrl !== 'http://localhost:4000/v1'; + } + /** * Get configured providers (those with API keys) */ @@ -672,6 +707,7 @@ class ConfigManager { if (this.hasApiKey('anthropic')) providers.push('anthropic'); if (this.hasApiKey('openai')) providers.push('openai'); if (this.hasApiKey('xai')) providers.push('xai'); + if (this.hasLiteLLMProxy()) providers.push('litellm'); if (this.hasApiKey('gemini')) providers.push('gemini'); // Codex uses the local Codex CLI/account auth instead of API-key storage. @@ -729,6 +765,11 @@ class ConfigManager { baseUrl = this.config.get('xai').baseUrl; actualModel = model || this.config.get('xai').defaultModel; break; + case 'litellm': + apiKey = this.getApiKey('litellm'); + baseUrl = process.env.LITELLM_BASE_URL?.trim() || this.config.get('litellm').baseUrl; + actualModel = model || process.env.LITELLM_MODEL?.trim() || this.config.get('litellm').defaultModel; + break; case 'gemini': // Google Gemini via its OpenAI-compatible endpoint (native tool-calling). apiKey = this.getApiKey('gemini'); @@ -784,8 +825,9 @@ class ConfigManager { const flag = (process.env.TEMPEST_MODEL_FALLBACK || '').trim().toLowerCase(); if (!flag || ['0', 'false', 'off', 'no'].includes(flag)) return []; const chain: FallbackEntry[] = []; - const add = (p: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini') => { - if (p === primary || !this.hasApiKey(p)) return; + const add = (p: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'litellm') => { + if (p === primary) return; + if (p === 'litellm' ? !this.hasLiteLLMProxy() : !this.hasApiKey(p)) return; chain.push({ provider: p, model: this.config.get(p).defaultModel, @@ -795,6 +837,7 @@ class ConfigManager { }; add('openrouter'); add('venice'); + add('litellm'); add('anthropic'); add('openai'); add('xai'); @@ -822,6 +865,15 @@ class ConfigManager { case 'openai': this.config.set('defaultModel', this.config.get('openai').defaultModel); break; + case 'xai': + this.config.set('defaultModel', this.config.get('xai').defaultModel); + break; + case 'gemini': + this.config.set('defaultModel', this.config.get('gemini').defaultModel); + break; + case 'litellm': + this.config.set('defaultModel', this.config.get('litellm').defaultModel); + break; case 'codex': this.config.set('defaultModel', this.config.get('codex').defaultModel); break; @@ -848,6 +900,15 @@ class ConfigManager { case 'openai': this.config.set('openai', { ...this.config.get('openai'), defaultModel: model }); break; + case 'xai': + this.config.set('xai', { ...this.config.get('xai'), defaultModel: model }); + break; + case 'gemini': + this.config.set('gemini', { ...this.config.get('gemini'), defaultModel: model }); + break; + case 'litellm': + this.config.set('litellm', { ...this.config.get('litellm'), defaultModel: model }); + break; case 'codex': this.config.set('codex', { ...this.config.get('codex'), defaultModel: model }); break; @@ -887,6 +948,7 @@ class ConfigManager { openai: settings.apiKeys.openai ? '***REDACTED***' : undefined, xai: settings.apiKeys.xai ? '***REDACTED***' : undefined, gemini: settings.apiKeys.gemini ? '***REDACTED***' : undefined, + litellm: settings.apiKeys.litellm ? '***REDACTED***' : undefined, }, }; writeFileSync(filePath, JSON.stringify(safeSettings, null, 2)); @@ -920,6 +982,12 @@ OPENAI_API_KEY= # Get your key at: https://console.x.ai/ XAI_API_KEY= +# LiteLLM Proxy (AI gateway — 100+ providers via single API) +# Docs: https://docs.litellm.ai/docs/proxy/quick_start +LITELLM_BASE_URL=http://localhost:4000/v1 +LITELLM_API_KEY= +LITELLM_MODEL=gpt-4o + # Google Gemini API Key (direct Gemini API via its OpenAI-compatible endpoint) # Get your key at: https://aistudio.google.com/apikey GEMINI_API_KEY= @@ -946,8 +1014,8 @@ TEMPEST_LOCAL_API_KEY= export const config = new ConfigManager(); // Helper functions for quick access -export const getApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini') => config.getApiKey(provider); -export const setApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini', key: string) => config.setApiKey(provider, key); -export const hasApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini') => config.hasApiKey(provider); +export const getApiKey = (provider: ApiKeyProvider) => config.getApiKey(provider); +export const setApiKey = (provider: ApiKeyProvider, key: string) => config.setApiKey(provider, key); +export const hasApiKey = (provider: ApiKeyProvider) => config.hasApiKey(provider); export const getLLMConfig = (provider?: LLMProvider, model?: string) => config.getLLMConfig(provider, model); export const getConfiguredProviders = () => config.getConfiguredProviders(); diff --git a/src/index.ts b/src/index.ts index 04289ca..c6e3f3e 100755 --- a/src/index.ts +++ b/src/index.ts @@ -44,6 +44,7 @@ export { createAnthropicBackbone, createOpenRouterBackbone, createOpenAIBackbone, + createLiteLLMBackbone, createMockBackbone, createLocalBackbone, createBestAvailableBackbone, diff --git a/src/llm/index.ts b/src/llm/index.ts index 48265dc..d07da60 100755 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -4,6 +4,7 @@ * Multi-provider LLM integration supporting: * - OpenRouter (recommended - access to Claude, GPT-4, Llama, etc.) * - Venice (OpenAI-compatible, privacy-focused / uncensored models) + * - LiteLLM (OpenAI-compatible AI gateway — 100+ providers via unified proxy) * - Anthropic (direct Claude access) * - OpenAI (GPT models) * - Mock (for testing) @@ -360,6 +361,28 @@ class VeniceAdapter extends OpenRouterAdapter { } } +// ============================================================================= +// LITELLM ADAPTER +// ============================================================================= +// LiteLLM is an AI gateway proxy that routes to 100+ LLM providers through a +// single OpenAI-compatible API. Same wire shape as OpenRouter (Bearer auth, +// POST /chat/completions, identical request/response + tool-calling + SSE +// stream), so it reuses the entire OpenRouter adapter and only differs in its +// default base URL and the key-required error message. +class LiteLLMAdapter extends OpenRouterAdapter { + name = 'litellm'; + + validateConfig(): { valid: boolean; error?: string } { + if (!this.config.baseUrl) { + return { + valid: false, + error: 'LiteLLM proxy base URL is required. Set LITELLM_BASE_URL or configure via tempest setup. Docs: https://docs.litellm.ai/docs/proxy/quick_start', + }; + } + return { valid: true }; + } +} + // ============================================================================= // ANTHROPIC ADAPTER // ============================================================================= @@ -1204,6 +1227,8 @@ export class LLMBackbone extends EventEmitter { return new OpenRouterAdapter(config); case 'venice': return new VeniceAdapter(config); + case 'litellm': + return new LiteLLMAdapter(config); case 'anthropic': return new AnthropicAdapter(config); case 'openai': @@ -1486,6 +1511,13 @@ export function createOpenAIBackbone(apiKey?: string, model?: string): LLMBackbo return new LLMBackbone(llmConfig); } +export function createLiteLLMBackbone(apiKey?: string, model?: string, baseUrl?: string): LLMBackbone { + const llmConfig = config.getLLMConfig('litellm', model); + if (apiKey) llmConfig.apiKey = apiKey; + if (baseUrl) llmConfig.baseUrl = baseUrl; + return new LLMBackbone(llmConfig); +} + export function createMockBackbone(): LLMBackbone { return new LLMBackbone({ provider: 'mock', @@ -1509,7 +1541,7 @@ export function createLocalBackbone(model?: string, baseUrl?: string): LLMBackbo * Create the best available backbone based on configured API keys */ export function createBestAvailableBackbone(): LLMBackbone { - // Priority: OpenRouter > Venice > Anthropic > OpenAI > Local > Mock + // Priority: OpenRouter > Venice > LiteLLM > Anthropic > OpenAI > Local > Mock const providers = config.getConfiguredProviders(); if (providers.includes('openrouter')) { @@ -1518,6 +1550,9 @@ export function createBestAvailableBackbone(): LLMBackbone { if (providers.includes('venice')) { return createVeniceBackbone(); } + if (providers.includes('litellm')) { + return createLiteLLMBackbone(); + } if (providers.includes('anthropic')) { return createAnthropicBackbone(); } diff --git a/src/setup.ts b/src/setup.ts index b2802b7..ceda7a6 100755 --- a/src/setup.ts +++ b/src/setup.ts @@ -240,6 +240,88 @@ async function setupOpenAIKey(): Promise { } } +async function setupLiteLLMProxy(): Promise { + console.log(''); + showInfo('LiteLLM is an AI gateway proxy that routes to 100+ LLM providers.'); + showInfo('Docs: ' + chalk.underline('https://docs.litellm.ai/docs/proxy/quick_start')); + console.log(''); + + const { baseUrl } = await inquirer.prompt([ + { + type: 'input', + name: 'baseUrl', + message: 'Enter your LiteLLM proxy base URL:', + default: 'http://localhost:4000/v1', + validate: (input: string) => { + if (!input || !input.startsWith('http')) { + return 'Please enter a valid URL (e.g. http://localhost:4000/v1)'; + } + return true; + }, + }, + ]); + + const { apiKey } = await inquirer.prompt([ + { + type: 'password', + name: 'apiKey', + message: 'Enter your LiteLLM API key (leave blank if not required):', + mask: '*', + }, + ]); + + const { model } = await inquirer.prompt([ + { + type: 'input', + name: 'model', + message: 'Enter the default model (uses LiteLLM model format, e.g. gpt-4o, anthropic/claude-sonnet-4-6):', + default: 'gpt-4o', + }, + ]); + + const spinner = ora('Testing LiteLLM proxy connection...').start(); + + try { + const llm = new LLMBackbone({ + provider: 'litellm', + model, + apiKey: apiKey || undefined, + baseUrl, + maxTokens: 10, + temperature: 0, + }); + + await llm.prompt('Hello', undefined, { maxTokens: 10 }); + spinner.succeed('LiteLLM proxy connection successful!'); + + config.set('litellm', { baseUrl, defaultModel: model }); + if (apiKey) setApiKey('litellm', apiKey); + showSuccess('LiteLLM proxy configured successfully!'); + + return true; + } catch (error) { + spinner.fail('LiteLLM proxy connection failed'); + showError(`Error: ${error instanceof Error ? error.message : String(error)}`); + + const { saveAnyway } = await inquirer.prompt([ + { + type: 'confirm', + name: 'saveAnyway', + message: 'Save the configuration anyway? (proxy may not be running yet)', + default: true, + }, + ]); + + if (saveAnyway) { + config.set('litellm', { baseUrl, defaultModel: model }); + if (apiKey) setApiKey('litellm', apiKey); + showSuccess('LiteLLM proxy configuration saved.'); + return true; + } + return false; + } +} + // ============================================================================= // MODEL SELECTION // ============================================================================= @@ -286,6 +368,7 @@ You can use a local model with no API key, or add a key from one of these provid • ${chalk.cyan('OpenRouter')} (recommended) - Access multiple models • ${chalk.cyan('Anthropic')} - Direct Claude access • ${chalk.cyan('OpenAI')} - GPT models +• ${chalk.cyan('LiteLLM')} - AI gateway proxy (100+ providers) The setup will guide you through: 1. Adding API key(s), if you use a hosted provider @@ -384,6 +467,10 @@ async function setupApiKeys(): Promise { name: `OpenAI ${hasApiKey('openai') ? chalk.green('(configured)') : ''}`, value: 'openai', }, + { + name: `LiteLLM Proxy ${config.hasLiteLLMProxy() ? chalk.green('(configured)') : chalk.gray('(100+ providers via gateway)')}`, + value: 'litellm', + }, ], }, ]); @@ -402,6 +489,9 @@ async function setupApiKeys(): Promise { case 'openai': await setupOpenAIKey(); break; + case 'litellm': + await setupLiteLLMProxy(); + break; } } } @@ -414,6 +504,7 @@ async function setupProvider(): Promise { if (hasApiKey('venice')) configuredProviders.push({ name: 'Venice', value: 'venice' }); if (hasApiKey('anthropic')) configuredProviders.push({ name: 'Anthropic', value: 'anthropic' }); if (hasApiKey('openai')) configuredProviders.push({ name: 'OpenAI', value: 'openai' }); + if (config.hasLiteLLMProxy()) configuredProviders.push({ name: 'LiteLLM Proxy', value: 'litellm' }); const { provider } = await inquirer.prompt([ { diff --git a/src/types/index.ts b/src/types/index.ts index 9417e4a..bd5835e 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -6,7 +6,7 @@ // LLM CONFIGURATION // ============================================================================= -export type LLMProvider = 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'codex' | 'mock' | 'local' | 'local-agent'; +export type LLMProvider = 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'gemini' | 'litellm' | 'codex' | 'mock' | 'local' | 'local-agent'; export interface LLMConfig { provider: LLMProvider;