feat(settings): Universal API Config panel with server-side model fetch (#90) (#101)

* feat(settings): Universal API Config panel with server-side model fetch (#90)

Consolidates provider configuration into one "Universal API Config" panel in
Settings — provider dropdown (OpenAI-compatible / Anthropic), base URL, API key,
model with a live "Fetch models" button, and max context tokens — above the
existing per-provider key sections.

Backward-compatible by design: the panel reads and writes the SAME state.settings
keys the individual sections already use (${provider}Key, selectedModel), so
nothing migrates and nothing is lost; the individual sections keep working and
stay in sync.

- src/config/provider-models.ts (new): listProviderModels queries a provider's
  own model endpoint (OpenAI-compatible Bearer /models, Anthropic x-api-key
  /v1/models, OpenRouter public list); resolveModels wraps it fail-open.
- POST /api/models (new route): server-side so the browser dodges provider CORS.
  Binds key+baseUrl by source — a caller-supplied base URL must bring its own
  key, so the server never lends its configured key to a body-chosen URL. Always
  responds, failing open to the static AVAILABLE_MODELS list on a missing key,
  network error, or unknown provider (incl. local-agent) — never hangs.
- Model options are built via the DOM API (createElement/textContent), never
  innerHTML, so untrusted provider-returned model ids cannot inject markup.

Scope notes: augment-not-replace (removing the individual sections is deferred
to maintainer preference); base URL + context cap are stored and used for model
discovery and server/agent flows — in-browser mission routing still uses
OpenRouter/Venice, matching the existing "not wired" hints; the "Model Reasoning
(Eino)" sub-proposal is a separate follow-up pending a mapping.

Verified on macOS: 12 unit tests (RED->GREEN) for the model module, full suite
462 green, and Playwright QA (17 checks) driving the real panel — render,
provider switch, persistence across reload, backward-compat, and both the static
and live Fetch paths.

Refs #90

* fix(settings): harden model list fetch

---------

Co-authored-by: Joseph Magly <1159087+jmagly@users.noreply.github.com>

src/__tests__/provider-models.test.ts

@@ -0,0 +1,149 @@
+/**
+ * Model-list fetching for the Universal API Config panel (issue #90).
+ *
+ * The Settings "Fetch models" button can't call api.openai.com/v1/models etc. directly from the
+ * browser (CORS), and today there are TWO hardcoded, already-drifting model lists (config
+ * AVAILABLE_MODELS vs the UI's MODELS array). `listProviderModels` is the server-side, testable core
+ * that queries a provider's own model endpoint with the OpenAI-compatible / Anthropic wire shape.
+ * It THROWS on any failure so the caller (the /api/models route) can fail open to the static list —
+ * a broken/keyless provider must never leave the panel empty.
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { listProviderModels, resolveModels } from '../config/provider-models.js';
+
+/** A fake fetch returning `body` as JSON with the given ok/status. Records the call. */
+function fakeFetch(body: unknown, ok = true, status = 200) {
+  return vi.fn(async (_url: string, _init?: unknown) => ({
+    ok, status, json: async () => body,
+  }));
+}
+
+describe('listProviderModels — OpenAI-compatible providers', () => {
+  it('GETs {baseUrl}/models with a Bearer key and returns the data[].id list', async () => {
+    const f = fakeFetch({ data: [{ id: 'gpt-4o' }, { id: 'gpt-4o-mini' }] });
+    const models = await listProviderModels('openai', { baseUrl: 'https://api.openai.com/v1', apiKey: 'sk-test', fetchImpl: f });
+
+    expect(models.map((m) => m.id)).toEqual(['gpt-4o', 'gpt-4o-mini']);
+    const [url, init] = f.mock.calls[0];
+    expect(url).toBe('https://api.openai.com/v1/models');
+    expect((init as { headers: Record<string, string> }).headers.Authorization).toBe('Bearer sk-test');
+  });
+
+  it('passes an abort signal so a non-responsive provider cannot hold the route open indefinitely', async () => {
+    const f = fakeFetch({ data: [{ id: 'gpt-4o' }] });
+    await listProviderModels('openai', { baseUrl: 'https://api.openai.com/v1', apiKey: 'k', fetchImpl: f, timeoutMs: 123 });
+
+    const init = f.mock.calls[0][1] as { signal?: AbortSignal };
+    expect(init.signal).toBeTruthy();
+  });
+
+  it('handles venice/xai/local the same OpenAI-compatible way (base already ends in /v1)', async () => {
+    const f = fakeFetch({ data: [{ id: 'llama-3.3-70b' }] });
+    const models = await listProviderModels('venice', { baseUrl: 'https://api.venice.ai/api/v1', apiKey: 'k', fetchImpl: f });
+    expect(models.map((m) => m.id)).toEqual(['llama-3.3-70b']);
+    expect(f.mock.calls[0][0]).toBe('https://api.venice.ai/api/v1/models');
+  });
+});
+
+describe('listProviderModels — Anthropic wire', () => {
+  it('GETs {baseUrl}/v1/models with x-api-key + anthropic-version headers', async () => {
+    const f = fakeFetch({ data: [{ id: 'claude-opus-4-8' }, { id: 'claude-sonnet-4-5' }] });
+    const models = await listProviderModels('anthropic', { baseUrl: 'https://api.anthropic.com', apiKey: 'sk-ant', fetchImpl: f });
+
+    expect(models.map((m) => m.id)).toEqual(['claude-opus-4-8', 'claude-sonnet-4-5']);
+    const [url, init] = f.mock.calls[0];
+    expect(url).toBe('https://api.anthropic.com/v1/models');
+    const headers = (init as { headers: Record<string, string> }).headers;
+    expect(headers['x-api-key']).toBe('sk-ant');
+    expect(headers['anthropic-version']).toBeTruthy();
+    expect(headers.Authorization).toBeUndefined();  // Anthropic does NOT use Bearer
+  });
+});
+
+describe('listProviderModels — OpenRouter', () => {
+  it('lists from the OpenRouter models endpoint (works without a key)', async () => {
+    const f = fakeFetch({ data: [{ id: 'anthropic/claude-opus-4.8' }, { id: 'google/gemini-2.5-pro' }] });
+    const models = await listProviderModels('openrouter', { fetchImpl: f });
+    expect(models.map((m) => m.id)).toEqual(['anthropic/claude-opus-4.8', 'google/gemini-2.5-pro']);
+    expect(String(f.mock.calls[0][0])).toContain('openrouter.ai');
+  });
+
+  it('honors a custom OpenRouter-compatible base URL when provided', async () => {
+    const f = fakeFetch({ data: [{ id: 'x' }] });
+    await listProviderModels('openrouter', { baseUrl: 'https://my-gateway.local/v1', fetchImpl: f });
+    expect(f.mock.calls[0][0]).toBe('https://my-gateway.local/v1/models');
+  });
+});
+
+describe('listProviderModels — base URL normalization', () => {
+  it('does not double /v1 for Anthropic when the base already ends in /v1', async () => {
+    const f = fakeFetch({ data: [{ id: 'claude-x' }] });
+    await listProviderModels('anthropic', { baseUrl: 'https://api.anthropic.com/v1', apiKey: 'k', fetchImpl: f });
+    expect(f.mock.calls[0][0]).toBe('https://api.anthropic.com/v1/models');   // not /v1/v1/models
+  });
+});
+
+describe('listProviderModels — failure modes (caller fails open to the static list)', () => {
+  it('throws on a non-ok HTTP response', async () => {
+    const f = fakeFetch({ error: 'unauthorized' }, false, 401);
+    await expect(listProviderModels('openai', { baseUrl: 'https://api.openai.com/v1', apiKey: 'bad', fetchImpl: f }))
+      .rejects.toThrow();
+  });
+
+  it('throws for a keyless/local-only provider that has no remote model list', async () => {
+    const f = fakeFetch({});
+    await expect(listProviderModels('codex', { fetchImpl: f })).rejects.toThrow();
+    expect(f).not.toHaveBeenCalled();  // no pointless network call
+  });
+
+  it('throws for an unknown provider instead of treating it as OpenAI-compatible', async () => {
+    const f = fakeFetch({});
+    await expect(listProviderModels('not-a-provider', { baseUrl: 'https://example.test/v1', fetchImpl: f }))
+      .rejects.toThrow(/not supported/);
+    expect(f).not.toHaveBeenCalled();
+  });
+
+  it('throws when the response has no data[] array (malformed)', async () => {
+    const f = fakeFetch({ nonsense: true });
+    await expect(listProviderModels('openai', { baseUrl: 'https://api.openai.com/v1', apiKey: 'k', fetchImpl: f }))
+      .rejects.toThrow();
+  });
+});
+
+describe('resolveModels — fail-open orchestrator (never throws; the panel is never empty)', () => {
+  const STATIC = [{ id: 'static-fallback-model' }];
+
+  it('returns live models when the fetch succeeds', async () => {
+    const f = fakeFetch({ data: [{ id: 'gpt-4o' }] });
+    const r = await resolveModels('openai', { baseUrl: 'https://api.openai.com/v1', apiKey: 'k', fetchImpl: f, staticFallback: STATIC });
+    expect(r.source).toBe('live');
+    expect(r.models.map((m) => m.id)).toEqual(['gpt-4o']);
+  });
+
+  it('fails open to the static list (with a note) when the fetch errors', async () => {
+    const f = fakeFetch({ error: 'nope' }, false, 500);
+    const r = await resolveModels('openai', { baseUrl: 'https://api.openai.com/v1', apiKey: 'k', fetchImpl: f, staticFallback: STATIC });
+    expect(r.source).toBe('static');
+    expect(r.models).toEqual(STATIC);
+    expect(r.note).toContain('500');
+  });
+
+  it('fails open for a keyless provider with no remote list', async () => {
+    const r = await resolveModels('codex', { staticFallback: STATIC, fetchImpl: fakeFetch({}) });
+    expect(r.source).toBe('static');
+    expect(r.models).toEqual(STATIC);
+  });
+
+  it('fails open for an unknown provider without calling a caller-supplied base URL', async () => {
+    const f = fakeFetch({});
+    const r = await resolveModels('not-a-provider', {
+      baseUrl: 'https://example.test/v1',
+      staticFallback: STATIC,
+      fetchImpl: f,
+    });
+
+    expect(r.source).toBe('static');
+    expect(r.models).toEqual(STATIC);
+    expect(f).not.toHaveBeenCalled();
+  });
+});

src/config/provider-models.ts

@@ -0,0 +1,125 @@
+/**
+ * Server-side model-list fetching for the Universal API Config panel (issue #90).
+ *
+ * The Settings "Fetch models" button can't hit api.openai.com/v1/models (etc.) directly from the
+ * browser (CORS), so the /api/models route calls this. It queries a provider's own model endpoint
+ * using the correct wire shape (OpenAI-compatible Bearer `/models`, or the Anthropic `x-api-key`
+ * `/v1/models`), and THROWS on any failure so the route can fail open to the static AVAILABLE_MODELS
+ * list — a broken/keyless provider must never leave the panel empty.
+ */
+
+export interface ProviderModel {
+  id: string;
+}
+
+type FetchLike = (
+  url: string,
+  init?: { headers?: Record<string, string>; signal?: AbortSignal },
+) => Promise<{ ok: boolean; status: number; json: () => Promise<unknown> }>;
+
+const ANTHROPIC_VERSION = '2023-06-01';
+const OPENROUTER_MODELS_URL = 'https://openrouter.ai/api/v1/models';
+const DEFAULT_MODEL_LIST_TIMEOUT_MS = 15_000;
+
+// Pseudo-providers with no remote model list (CLI-driven or in-process).
+const NO_REMOTE_LIST = new Set(['codex', 'mock', 'local-agent']);
+const OPENAI_COMPATIBLE_REMOTE_LIST = new Set(['openai', 'venice', 'xai', 'gemini', 'local']);
+const DIRECT_REMOTE_LIST = new Set(['anthropic', 'openrouter']);
+
+const stripTrailingSlash = (u: string): string => u.replace(/\/+$/, '');
+
+export function providerCanListModels(provider: string): boolean {
+  return DIRECT_REMOTE_LIST.has(provider) || OPENAI_COMPATIBLE_REMOTE_LIST.has(provider);
+}
+
+function timeoutSignal(timeoutMs: number): AbortSignal | undefined {
+  if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return undefined;
+  if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') {
+    return AbortSignal.timeout(timeoutMs);
+  }
+  if (typeof AbortController === 'undefined') return undefined;
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), timeoutMs);
+  if (typeof timeout.unref === 'function') timeout.unref();
+  return controller.signal;
+}
+
+/** Extract `data[].id` from an OpenAI-/Anthropic-shaped model-list response, or throw if malformed/empty. */
+function parseModelList(body: unknown): ProviderModel[] {
+  const data = (body as { data?: unknown } | null)?.data;
+  if (!Array.isArray(data)) throw new Error('provider model-list response has no data[] array');
+  const models = data
+    .map((m) => (m && typeof m === 'object' ? (m as { id?: unknown }).id : undefined))
+    .filter((id): id is string => typeof id === 'string' && id.length > 0)
+    .map((id) => ({ id }));
+  if (!models.length) throw new Error('provider returned an empty model list');
+  return models;
+}
+
+/**
+ * List a provider's available models via its own endpoint. Throws on any failure (missing baseUrl,
+ * non-2xx, malformed/empty body, or a provider with no remote list) — the caller fails open.
+ */
+export async function listProviderModels(
+  provider: string,
+  opts: { baseUrl?: string; apiKey?: string; fetchImpl?: FetchLike; timeoutMs?: number } = {},
+): Promise<ProviderModel[]> {
+  if (NO_REMOTE_LIST.has(provider)) {
+    throw new Error(`provider '${provider}' has no remote model list`);
+  }
+  if (!providerCanListModels(provider)) {
+    throw new Error(`provider '${provider}' is not supported for remote model listing`);
+  }
+  const doFetch = opts.fetchImpl ?? (globalThis.fetch as unknown as FetchLike | undefined);
+  if (!doFetch) throw new Error('no fetch implementation available');
+
+  let url: string;
+  const headers: Record<string, string> = {};
+
+  if (provider === 'anthropic') {
+    // Strip a trailing /v1 first so a user who types ".../v1" (matching every other provider's base)
+    // doesn't get a double /v1/v1/models → 404.
+    const abase = stripTrailingSlash(opts.baseUrl || 'https://api.anthropic.com').replace(/\/v1$/, '');
+    url = `${abase}/v1/models`;
+    headers['anthropic-version'] = ANTHROPIC_VERSION;                       // Anthropic uses x-api-key, NOT Bearer
+    if (opts.apiKey) headers['x-api-key'] = opts.apiKey;
+  } else if (provider === 'openrouter') {
+    // Honor a custom OpenRouter-compatible base URL if given; else the public list.
+    url = opts.baseUrl && opts.baseUrl.trim() ? `${stripTrailingSlash(opts.baseUrl)}/models` : OPENROUTER_MODELS_URL;
+    if (opts.apiKey) headers.Authorization = `Bearer ${opts.apiKey}`;
+  } else {
+    // OpenAI-compatible: openai / venice / xai / gemini / local / litellm / openai-compat.
+    const base = opts.baseUrl && opts.baseUrl.trim() ? stripTrailingSlash(opts.baseUrl) : undefined;
+    if (!base) throw new Error(`provider '${provider}' requires a baseUrl to list models`);
+    url = `${base}/models`;
+    if (opts.apiKey) headers.Authorization = `Bearer ${opts.apiKey}`;
+  }
+
+  const signal = timeoutSignal(opts.timeoutMs ?? DEFAULT_MODEL_LIST_TIMEOUT_MS);
+  const res = await doFetch(url, { headers, signal });
+  if (!res.ok) throw new Error(`model-list request failed: HTTP ${res.status}`);
+  return parseModelList(await res.json());
+}
+
+export interface ResolvedModels {
+  source: 'live' | 'static';
+  models: ProviderModel[];
+  note?: string;
+}
+
+/**
+ * Live models with a guaranteed fail-open to `staticFallback` — NEVER throws. The /api/models route
+ * uses this so a missing key, CORS/network error, or unlisted provider still returns the static
+ * AVAILABLE_MODELS list (the panel is never empty) with `source:'static'` so the UI can flag it.
+ */
+export async function resolveModels(
+  provider: string,
+  opts: { baseUrl?: string; apiKey?: string; fetchImpl?: FetchLike; timeoutMs?: number; staticFallback: ProviderModel[] },
+): Promise<ResolvedModels> {
+  try {
+    return { source: 'live', models: await listProviderModels(provider, opts) };
+  } catch (e) {
+    return { source: 'static', models: opts.staticFallback, note: (e as Error).message };
+  }
+}

src/server.ts

@@ -16,7 +16,8 @@ import { appendFile, mkdir, readFile, writeFile } from 'fs/promises';
 import { join } from 'path';
 import { promisify } from 'util';
 import { createHash, randomUUID } from 'crypto';
-import { config } from './config/index.js';
+import { config, AVAILABLE_MODELS } from './config/index.js';
+import { resolveModels } from './config/provider-models.js';
 import { redactString, redactLedgerText, redactSecrets } from './redact.js';
 import { LLMBackbone } from './llm/index.js';
 import { TempestCommand } from './index.js';
@@ -28,13 +29,17 @@ import { AGENT_PROMPT_PACKS, FOREFRONT_PRESSURE_LANES, OPERATOR_RUNBOOKS, RESOUR
 import { AI_REDTEAM_PLAYBOOK, AI_REDTEAM_TECHNIQUE_IDS, aiRedTeamBriefing } from './resources/ai-redteam-playbook.js';
 import { OPERATOR_SYSTEM_PROMPTS, PLINIAN_OPERATOR_DOCTRINE, THE_FIXER_SYSTEM_PROMPT } from './prompts/index.js';
 import { createTargetFromUrl, createTargetFromIP } from './target/index.js';
-import type { OperatorArchetype } from './types/index.js';
+import type { OperatorArchetype, LLMProvider } from './types/index.js';
 import { listOperatorPrompts, setOperatorOverride, resetOperatorOverride, type OperatorOverride } from './operators/index.js';
 import { ingestRepoToSourceContext, runWhiteboxAnalysis, resolveRepoSourceForAnalysis, RepoCloneError, RepoPathError } from './recon/whitebox.js';
 import { redactCredential } from './evidence/index.js';
 
 const execFileAsync = promisify(execFile);
 
+function isKnownLLMProvider(provider: string): provider is LLMProvider {
+  return Object.prototype.hasOwnProperty.call(AVAILABLE_MODELS, provider);
+}
+
 // =============================================================================
 // PAYLOAD DATABASES (Shared with MCP server)
 // =============================================================================
@@ -6088,6 +6093,34 @@ app.get('/api/llm/status', (_req: Request, res: Response) => {
   });
 });
 
+// POST /api/models — live model list for the Universal API Config panel (#90).
+// POST (not GET) so the browser can pass its own localStorage-held key/baseUrl in the body — the
+// standalone UI holds the key, not the server (consistent with the existing per-request mission key,
+// and it stays on localhost). Body values override server config; missing → server config. Runs
+// server-side so the browser dodges provider CORS, and fails open to the static AVAILABLE_MODELS list
+// (source:'static') so a missing key / network error never leaves the panel empty.
+app.post('/api/models', async (req: Request, res: Response): Promise<void> => {
+  const body = (req.body ?? {}) as { provider?: string; apiKey?: string; baseUrl?: string };
+  const defaultProvider = config.getLLMConfig().provider;
+  const requestedProvider = (typeof body.provider === 'string' && body.provider.trim()) || defaultProvider;
+  const provider = isKnownLLMProvider(requestedProvider) ? requestedProvider : defaultProvider;
+  // getLLMConfig throws for any provider it doesn't switch on (e.g. the legitimate 'local-agent', or a
+  // garbage string). A synchronous throw inside this async handler would hang the request (Express 4
+  // never responds), so swallow it and fall through to the static list — the route must always answer.
+  let cfg: { baseUrl?: string; apiKey?: string } = {};
+  try { cfg = config.getLLMConfig(provider); } catch { /* provider has no remote/server config → static fallback */ }
+  const bodyBaseUrl = typeof body.baseUrl === 'string' && body.baseUrl.trim() ? body.baseUrl : undefined;
+  const bodyKey = typeof body.apiKey === 'string' && body.apiKey ? body.apiKey : undefined;
+  // Bind key+baseUrl by source: a caller-supplied custom baseUrl must bring its OWN key — never lend
+  // the server's configured apiKey to a body-chosen URL (that would disclose the stored key). The
+  // server key is only ever sent to the server's own configured provider endpoint.
+  const baseUrl = bodyBaseUrl ?? cfg.baseUrl;
+  const apiKey = bodyBaseUrl ? bodyKey : (bodyKey ?? cfg.apiKey);
+  const staticFallback = AVAILABLE_MODELS[provider].map((m: { id: string }) => ({ id: m.id }));
+  const resolved = await resolveModels(provider, { baseUrl, apiKey, staticFallback });
+  res.json({ provider, ...resolved });
+});
+
 // =============================================================================
 // TOOL EXECUTION ENDPOINTS
 // =============================================================================