mm/t3mp3st
0
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add LiteLLM as AI gateway provider

Browse Source
This commit is contained in:
RheagalFire committed 2026-07-06 21:46:48 +05:30
1 parent 2f20ee5820
commit 26217cbe33
5 files changed
+192 -11
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/config/index.ts
+63 -9
View File
@@ -23,6 +23,7 @@ export interface TempestSettings {
anthropic?: string;
openai?: string;
xai?: string;
litellm?: string;
};
// Default LLM settings
@@ -61,6 +62,12 @@ export interface TempestSettings {
defaultModel: string;
};
// LiteLLM AI gateway proxy — routes to 100+ LLM providers via unified API
litellm: {
baseUrl: string;
defaultModel: string;
};
// Codex CLI/account subscription backend
codex: {
command: string;
@@ -124,6 +131,11 @@ const DEFAULT_SETTINGS: TempestSettings = {
defaultModel: 'grok-build-0.1',
},
litellm: {
baseUrl: 'http://localhost:4000/v1',
defaultModel: 'gpt-4o',
},
codex: {
command: 'codex',
defaultModel: 'codex-default',
@@ -381,6 +393,16 @@ export const AVAILABLE_MODELS: Record<LLMProvider, ModelInfo[]> = {
capabilities: ['reasoning', 'code', 'analysis', 'vision', 'fast'],
},
],
litellm: [
{
id: 'gpt-4o',
name: 'Any model via LiteLLM proxy (default: gpt-4o)',
provider: 'LiteLLM',
contextWindow: 128000,
maxOutput: 4096,
capabilities: ['reasoning', 'code', 'analysis', 'vision', 'tools'],
},
],
codex: [
{
id: 'codex-default',
@@ -547,7 +569,7 @@ class ConfigManager {
/**
* Set an API key for a provider
*/
setApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai', key: string): void {
setApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm', key: string): void {
const apiKeys = this.config.get('apiKeys');
apiKeys[provider] = key;
this.config.set('apiKeys', apiKeys);
@@ -556,7 +578,7 @@ class ConfigManager {
/**
* Get an API key for a provider
*/
getApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai'): string | undefined {
getApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm'): string | undefined {
// First check environment variables (highest priority)
const envVarMap = {
openrouter: 'OPENROUTER_API_KEY',
@@ -564,6 +586,7 @@ class ConfigManager {
anthropic: 'ANTHROPIC_API_KEY',
openai: 'OPENAI_API_KEY',
xai: 'XAI_API_KEY',
litellm: 'LITELLM_API_KEY',
};
// Force a fully UNCONFIGURED server (no key from env OR the saved store) — used by
@@ -582,7 +605,7 @@ class ConfigManager {
/**
* Check if a provider has a valid API key configured
*/
hasApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai'): boolean {
hasApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm'): boolean {
const key = this.getApiKey(provider);
return !!key && key.length > 10;
}
@@ -590,12 +613,22 @@ class ConfigManager {
/**
* Remove an API key
*/
removeApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai'): void {
removeApiKey(provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm'): void {
const apiKeys = this.config.get('apiKeys');
delete apiKeys[provider];
this.config.set('apiKeys', apiKeys);
}
/**
* Check if a LiteLLM proxy is configured (base URL set via env or config)
*/
hasLiteLLMProxy(): boolean {
const envUrl = process.env.LITELLM_BASE_URL?.trim();
if (envUrl) return true;
const storedUrl = this.config.get('litellm')?.baseUrl;
return !!storedUrl && storedUrl !== 'http://localhost:4000/v1';
}
/**
* Get configured providers (those with API keys)
*/
@@ -607,6 +640,7 @@ class ConfigManager {
if (this.hasApiKey('anthropic')) providers.push('anthropic');
if (this.hasApiKey('openai')) providers.push('openai');
if (this.hasApiKey('xai')) providers.push('xai');
if (this.hasLiteLLMProxy()) providers.push('litellm');
// Codex uses the local Codex CLI/account auth instead of API-key storage.
providers.push('codex');
@@ -654,6 +688,11 @@ class ConfigManager {
baseUrl = this.config.get('xai').baseUrl;
actualModel = model || this.config.get('xai').defaultModel;
break;
case 'litellm':
apiKey = this.getApiKey('litellm');
baseUrl = process.env.LITELLM_BASE_URL?.trim() || this.config.get('litellm').baseUrl;
actualModel = model || process.env.LITELLM_MODEL?.trim() || this.config.get('litellm').defaultModel;
break;
case 'codex':
actualModel = model || this.config.get('codex').defaultModel;
break;
@@ -699,8 +738,9 @@ class ConfigManager {
const flag = (process.env.TEMPEST_MODEL_FALLBACK || '').trim().toLowerCase();
if (!flag || ['0', 'false', 'off', 'no'].includes(flag)) return [];
const chain: FallbackEntry[] = [];
const add = (p: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai') => {
if (p === primary || !this.hasApiKey(p)) return;
const add = (p: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm') => {
if (p === primary) return;
if (p === 'litellm' ? !this.hasLiteLLMProxy() : !this.hasApiKey(p)) return;
chain.push({
provider: p,
model: this.config.get(p).defaultModel,
@@ -710,6 +750,7 @@ class ConfigManager {
};
add('openrouter');
add('venice');
add('litellm');
add('anthropic');
add('openai');
add('xai');
@@ -736,6 +777,9 @@ class ConfigManager {
case 'openai':
this.config.set('defaultModel', this.config.get('openai').defaultModel);
break;
case 'litellm':
this.config.set('defaultModel', this.config.get('litellm').defaultModel);
break;
case 'codex':
this.config.set('defaultModel', this.config.get('codex').defaultModel);
break;
@@ -759,6 +803,9 @@ class ConfigManager {
case 'openai':
this.config.set('openai', { ...this.config.get('openai'), defaultModel: model });
break;
case 'litellm':
this.config.set('litellm', { ...this.config.get('litellm'), defaultModel: model });
break;
case 'codex':
this.config.set('codex', { ...this.config.get('codex'), defaultModel: model });
break;
@@ -797,6 +844,7 @@ class ConfigManager {
anthropic: settings.apiKeys.anthropic ? '***REDACTED***' : undefined,
openai: settings.apiKeys.openai ? '***REDACTED***' : undefined,
xai: settings.apiKeys.xai ? '***REDACTED***' : undefined,
litellm: settings.apiKeys.litellm ? '***REDACTED***' : undefined,
},
};
writeFileSync(filePath, JSON.stringify(safeSettings, null, 2));
@@ -829,6 +877,12 @@ OPENAI_API_KEY=
# xAI API Key
# Get your key at: https://console.x.ai/
XAI_API_KEY=
# LiteLLM Proxy (AI gateway — 100+ providers via single API)
# Docs: https://docs.litellm.ai/docs/proxy/quick_start
LITELLM_BASE_URL=http://localhost:4000/v1
LITELLM_API_KEY=
LITELLM_MODEL=gpt-4o
`;
writeFileSync(filePath, template);
}
@@ -841,8 +895,8 @@ XAI_API_KEY=
export const config = new ConfigManager();
// Helper functions for quick access
export const getApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai') => config.getApiKey(provider);
export const setApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai', key: string) => config.setApiKey(provider, key);
export const hasApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai') => config.hasApiKey(provider);
export const getApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm') => config.getApiKey(provider);
export const setApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm', key: string) => config.setApiKey(provider, key);
export const hasApiKey = (provider: 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm') => config.hasApiKey(provider);
export const getLLMConfig = (provider?: LLMProvider, model?: string) => config.getLLMConfig(provider, model);
export const getConfiguredProviders = () => config.getConfiguredProviders();
src/index.ts
+1
View File
@@ -44,6 +44,7 @@ export {
createAnthropicBackbone,
createOpenRouterBackbone,
createOpenAIBackbone,
createLiteLLMBackbone,
createMockBackbone,
createLocalBackbone,
createBestAvailableBackbone,
src/llm/index.ts
+36 -1
View File
@@ -4,6 +4,7 @@
* Multi-provider LLM integration supporting:
* - OpenRouter (recommended - access to Claude, GPT-4, Llama, etc.)
* - Venice (OpenAI-compatible, privacy-focused / uncensored models)
* - LiteLLM (OpenAI-compatible AI gateway — 100+ providers via unified proxy)
* - Anthropic (direct Claude access)
* - OpenAI (GPT models)
* - Mock (for testing)
@@ -360,6 +361,28 @@ class VeniceAdapter extends OpenRouterAdapter {
}
}
// =============================================================================
// LITELLM ADAPTER
// =============================================================================
// LiteLLM is an AI gateway proxy that routes to 100+ LLM providers through a
// single OpenAI-compatible API. Same wire shape as OpenRouter (Bearer auth,
// POST /chat/completions, identical request/response + tool-calling + SSE
// stream), so it reuses the entire OpenRouter adapter and only differs in its
// default base URL and the key-required error message.
class LiteLLMAdapter extends OpenRouterAdapter {
name = 'litellm';
validateConfig(): { valid: boolean; error?: string } {
if (!this.config.baseUrl) {
return {
valid: false,
error: 'LiteLLM proxy base URL is required. Set LITELLM_BASE_URL or configure via tempest setup. Docs: https://docs.litellm.ai/docs/proxy/quick_start',
};
}
return { valid: true };
}
}
// =============================================================================
// ANTHROPIC ADAPTER
// =============================================================================
@@ -1197,6 +1220,8 @@ export class LLMBackbone extends EventEmitter<LLMEvents> {
return new OpenRouterAdapter(config);
case 'venice':
return new VeniceAdapter(config);
case 'litellm':
return new LiteLLMAdapter(config);
case 'anthropic':
return new AnthropicAdapter(config);
case 'openai':
@@ -1477,6 +1502,13 @@ export function createOpenAIBackbone(apiKey?: string, model?: string): LLMBackbo
return new LLMBackbone(llmConfig);
}
export function createLiteLLMBackbone(apiKey?: string, model?: string, baseUrl?: string): LLMBackbone {
const llmConfig = config.getLLMConfig('litellm', model);
if (apiKey) llmConfig.apiKey = apiKey;
if (baseUrl) llmConfig.baseUrl = baseUrl;
return new LLMBackbone(llmConfig);
}
export function createMockBackbone(): LLMBackbone {
return new LLMBackbone({
provider: 'mock',
@@ -1500,7 +1532,7 @@ export function createLocalBackbone(model?: string, baseUrl?: string): LLMBackbo
* Create the best available backbone based on configured API keys
*/
export function createBestAvailableBackbone(): LLMBackbone {
// Priority: OpenRouter > Venice > Anthropic > OpenAI > Local > Mock
// Priority: OpenRouter > Venice > LiteLLM > Anthropic > OpenAI > Local > Mock
const providers = config.getConfiguredProviders();
if (providers.includes('openrouter')) {
@@ -1509,6 +1541,9 @@ export function createBestAvailableBackbone(): LLMBackbone {
if (providers.includes('venice')) {
return createVeniceBackbone();
}
if (providers.includes('litellm')) {
return createLiteLLMBackbone();
}
if (providers.includes('anthropic')) {
return createAnthropicBackbone();
}
src/setup.ts
+91
View File
@@ -240,6 +240,88 @@ async function setupOpenAIKey(): Promise<boolean> {
}
}
async function setupLiteLLMProxy(): Promise<boolean> {
console.log('');
showInfo('LiteLLM is an AI gateway proxy that routes to 100+ LLM providers.');
showInfo('Docs: ' + chalk.underline('https://docs.litellm.ai/docs/proxy/quick_start'));
console.log('');
const { baseUrl } = await inquirer.prompt([
{
type: 'input',
name: 'baseUrl',
message: 'Enter your LiteLLM proxy base URL:',
default: 'http://localhost:4000/v1',
validate: (input: string) => {
if (!input || !input.startsWith('http')) {
return 'Please enter a valid URL (e.g. http://localhost:4000/v1)';
}
return true;
},
},
]);
const { apiKey } = await inquirer.prompt([
{
type: 'password',
name: 'apiKey',
message: 'Enter your LiteLLM API key (leave blank if not required):',
mask: '*',
},
]);
const { model } = await inquirer.prompt([
{
type: 'input',
name: 'model',
message: 'Enter the default model (uses LiteLLM model format, e.g. gpt-4o, anthropic/claude-sonnet-4-6):',
default: 'gpt-4o',
},
]);
const spinner = ora('Testing LiteLLM proxy connection...').start();
try {
const llm = new LLMBackbone({
provider: 'litellm',
model,
apiKey: apiKey || undefined,
baseUrl,
maxTokens: 10,
temperature: 0,
});
await llm.prompt('Hello', undefined, { maxTokens: 10 });
spinner.succeed('LiteLLM proxy connection successful!');
config.set('litellm', { baseUrl, defaultModel: model });
if (apiKey) setApiKey('litellm', apiKey);
showSuccess('LiteLLM proxy configured successfully!');
return true;
} catch (error) {
spinner.fail('LiteLLM proxy connection failed');
showError(`Error: ${error instanceof Error ? error.message : String(error)}`);
const { saveAnyway } = await inquirer.prompt([
{
type: 'confirm',
name: 'saveAnyway',
message: 'Save the configuration anyway? (proxy may not be running yet)',
default: true,
},
]);
if (saveAnyway) {
config.set('litellm', { baseUrl, defaultModel: model });
if (apiKey) setApiKey('litellm', apiKey);
showSuccess('LiteLLM proxy configuration saved.');
return true;
}
return false;
}
}
// =============================================================================
// MODEL SELECTION
// =============================================================================
@@ -285,6 +367,7 @@ You'll need an API key from one of these providers:
• ${chalk.cyan('OpenRouter')} (recommended) - Access multiple models
• ${chalk.cyan('Anthropic')} - Direct Claude access
• ${chalk.cyan('OpenAI')} - GPT models
• ${chalk.cyan('LiteLLM')} - AI gateway proxy (100+ providers)
The setup will guide you through:
1. Adding your API key(s)
@@ -384,6 +467,10 @@ async function setupApiKeys(): Promise<void> {
name: `OpenAI ${hasApiKey('openai') ? chalk.green('(configured)') : ''}`,
value: 'openai',
},
{
name: `LiteLLM Proxy ${config.hasLiteLLMProxy() ? chalk.green('(configured)') : chalk.gray('(100+ providers via gateway)')}`,
value: 'litellm',
},
],
},
]);
@@ -402,6 +489,9 @@ async function setupApiKeys(): Promise<void> {
case 'openai':
await setupOpenAIKey();
break;
case 'litellm':
await setupLiteLLMProxy();
break;
}
}
}
@@ -413,6 +503,7 @@ async function setupProvider(): Promise<void> {
if (hasApiKey('venice')) configuredProviders.push({ name: 'Venice', value: 'venice' });
if (hasApiKey('anthropic')) configuredProviders.push({ name: 'Anthropic', value: 'anthropic' });
if (hasApiKey('openai')) configuredProviders.push({ name: 'OpenAI', value: 'openai' });
if (config.hasLiteLLMProxy()) configuredProviders.push({ name: 'LiteLLM Proxy', value: 'litellm' });
if (configuredProviders.length === 0) {
showWarning('No API keys configured. Please add at least one API key first.');
src/types/index.ts
+1 -1
View File
@@ -6,7 +6,7 @@
// LLM CONFIGURATION
// =============================================================================
export type LLMProvider = 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'codex' | 'mock' | 'local' | 'local-agent';
export type LLMProvider = 'openrouter' | 'venice' | 'anthropic' | 'openai' | 'xai' | 'litellm' | 'codex' | 'mock' | 'local' | 'local-agent';
export interface LLMConfig {
provider: LLMProvider;