Report stale local agent backends

docs/index.html

@@ -17921,7 +17921,7 @@ IMPORTANT: This is an authorized security testing/red team platform. All operati
             // ── Local agents: fetch ONCE before the backbone label so both reflect the server's
             // actual agent state from the first poll (and t3mpPreferredAgent can route to a live one) ──
             let __agents = [];
-            try { const s = await (await fetch(base + '/api/agents/local/status')).json(); __agents = s.connected || []; } catch (_) {}
+            try { const s = await (await fetch(base + '/api/agents/local/status?check=1')).json(); __agents = s.connected || []; } catch (_) {}
             window.__t3mpServerAgents = __agents;
             window.__t3mpConnectedAgents = __agents.length;
             window.__t3mpLiveAgents = __agents.filter(a => a.lastPing && a.lastPing.ok).length;
@@ -17931,10 +17931,12 @@ IMPORTANT: This is an authorized security testing/red team platform. All operati
                 const key = (typeof getApiKey === 'function') ? (getApiKey() || '') : '';
                 let txt, color;
                 const serverLLM = !!(health && health.llm && health.llm.connected);
+                const staleAgents = __agents.filter(a => !a.lastPing || !a.lastPing.ok).length;
                 const hasBackend = !!(agent || key || serverLLM);
                 if (!key && agent) { txt = '⚡ ' + agent.toUpperCase() + ' · local, no key'; color = '#00ff88'; }
                 else if (key) { txt = (key.startsWith('sk-ant-') ? 'Anthropic' : key.startsWith('sk-or-') ? 'OpenRouter' : key.startsWith('sk-') ? 'OpenAI' : 'OpenRouter') + ' · your key'; color = '#0088ff'; }
                 else if (serverLLM) { txt = (health.llm.provider || 'server') + ' · server key'; color = '#0088ff'; }
+                else if (staleAgents) { txt = 'local agent stale — model unreachable'; color = '#ffaa00'; }
                 else { txt = 'none — connect agent / add key'; color = '#ff6b6b'; }
                 $('sysLLM').innerHTML = '<span style="color:'+color+'">'+txt+'</span>';
                 // Keyless hero callout: when NO backend is present (no agent, no key, no server LLM),
@@ -17959,7 +17961,11 @@ IMPORTANT: This is an authorized security testing/red team platform. All operati
             // ── Local agents cell (reuse the state fetched above; keeps the stale-count latch fresh) ──
             if ($('sysAgents')) {
                 $('sysAgents').innerHTML = __agents.length
-                    ? __agents.map(a => { const live = a.lastPing && a.lastPing.ok; return '<span style="color:'+(live?'#00ff88':'#ffaa00')+'">●</span><span style="color:#ccc">'+a.id+'</span>'; }).join('&nbsp;&nbsp;')
+                    ? __agents.map(a => {
+                        const live = a.lastPing && a.lastPing.ok;
+                        const err = a.lastPing && a.lastPing.error ? ' title="'+String(a.lastPing.error).replace(/"/g,'&quot;')+'"' : '';
+                        return '<span'+err+'><span style="color:'+(live?'#00ff88':'#ffaa00')+'">●</span><span style="color:#ccc">'+a.id+'</span><span style="color:#777;font-weight:normal;font-size:10px">'+(live?' live':' stale')+'</span></span>';
+                      }).join('&nbsp;&nbsp;')
                     : '<span style="color:#888;font-weight:normal">none — click to connect</span>';
             }
             // ── Drive the SitRep pipeline from /api/mission/status (sysPhases/sysFindings status cells were consolidated out) ──
@@ -24747,10 +24753,10 @@ Be specific and actionable. Format as structured JSON.`;
     else if (!a.authed) { status='Not authed'; color='amber'; }
     else if (stale) { status='⚠ Auth stale'; color='red'; }
     else if (live) { status='● Live · '+ps.latencyMs+'ms'; color='green'; }
-    else if (connected) { status='● Active'; color='green'; }
+    else if (connected) { status='● Connected · unchecked'; color='amber'; }
     else { status='Detected'; color='cyan'; }
     const disabled = !a.ready;
-    const borderColor = stale ? 'rgba(255,80,80,0.4)' : (connected||live) ? 'rgba(0,255,136,0.35)' : 'rgba(0,200,255,0.14)';
+    const borderColor = stale ? 'rgba(255,80,80,0.4)' : live ? 'rgba(0,255,136,0.35)' : connected ? 'rgba(255,170,0,0.28)' : 'rgba(0,200,255,0.14)';
     const sub = !a.installed ? 'binary not found on PATH'
               : stale ? ('⚠ '+(ps.error ? ps.error.slice(0,70) : 'authentication failed — re-login this CLI'))
               : (a.invokeHint || a.bin);
@@ -24786,10 +24792,10 @@ Be specific and actionable. Format as structured JSON.`;
     const live = Object.keys(pingStatus).filter(function(k){return pingStatus[k].ok;}).length;
     const sum = document.getElementById('localAgentsSummary');
     if (sum) sum.innerHTML = connectedIds.length+' active'+(live?' · '+live+' live':'')+' · '+ready+' ready · '+installed+'/'+agents.length+' installed';
-    // expose for the War Room backend-gate, and dismiss the nudge once ANY agent is connected
-    // (a plain connect counts — not only a successful live-ping)
+    // expose live/connected counts separately. A plain connect is just a registration; only a
+    // fresh successful ping proves the local model is currently usable.
     window.__t3mpLiveAgents = live; window.__t3mpConnectedAgents = connectedIds.length;
-    if (connectedIds.length > 0 || live > 0) { const nb = document.getElementById('noBackendNudge'); if (nb) nb.style.display = 'none'; }
+    if (live > 0) { const nb = document.getElementById('noBackendNudge'); if (nb) nb.style.display = 'none'; }
   }
 
   // ── remember connected agents + silently re-establish them after a refresh ──
@@ -24866,8 +24872,9 @@ Be specific and actionable. Format as structured JSON.`;
     // _isStandalone (no API server) is NOT a backend — client-side mode still needs an API key.
     // Counting it here let keyless standalone deploys bypass the "connect an agent / add a key"
     // gate and silently fall through to a failing OpenRouter pipeline. A backend = a real LLM:
-    // the server's own LLM (llmAvailable) or a connected local agent.
-    return !!(api.llmAvailable || window.__t3mpLiveAgents > 0 || window.__t3mpConnectedAgents > 0);
+    // the server's own LLM (llmAvailable) or a live local agent. A connected-but-stale
+    // agent is not enough, because the underlying local model may have gone down.
+    return !!(api.llmAvailable || window.__t3mpLiveAgents > 0);
   }
   window.t3mpHasBackend = hasBackend;
   // Pick the best connected agent to serve as the mission LLM backend: prefer a live-pinged one, and
@@ -24876,15 +24883,14 @@ Be specific and actionable. Format as structured JSON.`;
     var order = ['codex','hermes','claude'];
     var connected = (connectedIds || []).slice();
     var live = connected.filter(function(id){ return pingStatus[id] && pingStatus[id].ok; });
-    var pool = live.length ? live : connected;
+    var pool = live;
     // Fall back to the SERVER's reported agents (/api/agents/local/status, stashed by
     // refreshSystemStatus). Without this, a War Room loaded without first visiting Settings has an
     // empty client connectedIds and would route missions to a key-provider even though the server
     // already has a live agent — the exact AI-backbone/mission-routing mismatch we want to avoid.
     if (!pool.length && Array.isArray(window.__t3mpServerAgents)) {
       var srv = window.__t3mpServerAgents.slice();
-      var srvLive = srv.filter(function(a){ return a.lastPing && a.lastPing.ok; });
-      pool = (srvLive.length ? srvLive : srv).map(function(a){ return a.id; });
+      pool = srv.filter(function(a){ return a.lastPing && a.lastPing.ok; }).map(function(a){ return a.id; });
     }
     pool.sort(function(a,b){ return order.indexOf(a) - order.indexOf(b); });
     return pool[0] || null;

src/server.ts

@@ -7350,8 +7350,35 @@ app.get('/api/bounty/credentials', (_req: Request, res: Response) => {
 // Detect agent CLIs that are already installed + logged-in on this machine and enlist them as
 // operators — no keys are entered or read (auth is detected by artifact PRESENCE only; see
 // src/agent/local-agents.ts). In-memory registry of agents connected this session:
-const connectedLocalAgents = new Map<string, { id: string; label: string; version?: string; connectedAt: number; lastPing?: AgentRunSummary | null }>();
 type AgentRunSummary = { ok: boolean; latencyMs: number; output: string; error?: string };
+type ConnectedLocalAgent = {
+  id: string;
+  label: string;
+  version?: string;
+  connectedAt: number;
+  lastPing?: AgentRunSummary | null;
+  lastHealthCheckAt?: number;
+};
+const connectedLocalAgents = new Map<string, ConnectedLocalAgent>();
+const LOCAL_AGENT_HEALTH_TTL_MS = 30_000;
+const LOCAL_AGENT_HEALTH_TIMEOUT_MS = 15_000;
+
+async function refreshConnectedLocalAgentHealth(force = false): Promise<void> {
+  const now = Date.now();
+  const due = Array.from(connectedLocalAgents.values()).filter((agent) => (
+    force ||
+    !agent.lastHealthCheckAt ||
+    now - agent.lastHealthCheckAt > LOCAL_AGENT_HEALTH_TTL_MS
+  ));
+
+  await Promise.all(due.map(async (agent) => {
+    const result = await pingLocalAgent(agent.id, undefined, LOCAL_AGENT_HEALTH_TIMEOUT_MS);
+    const current = connectedLocalAgents.get(agent.id);
+    if (!current) return;
+    current.lastPing = result;
+    current.lastHealthCheckAt = Date.now();
+  }));
+}
 
 // GET /api/agents/local/detect — which agents are installed / authed / ready (no tokens spent)
 app.get('/api/agents/local/detect', async (_req: Request, res: Response): Promise<void> => {
@@ -7378,7 +7405,14 @@ app.post('/api/agents/local/connect', async (req: Request, res: Response): Promi
     if (!d.authed) { results.push({ id, status: 'not-authed', version: d.version }); continue; }
     let ping: AgentRunSummary | null = null;
     if (doPing) ping = await pingLocalAgent(id);
-    connectedLocalAgents.set(id, { id, label: d.label, version: d.version, connectedAt: Date.now(), lastPing: ping });
+    connectedLocalAgents.set(id, {
+      id,
+      label: d.label,
+      version: d.version,
+      connectedAt: Date.now(),
+      lastPing: ping,
+      lastHealthCheckAt: ping ? Date.now() : undefined,
+    });
     results.push({ id, status: 'active', label: d.label, version: d.version, authMethod: d.authMethod, ping });
   }
   syncLocalAgentSelection(connectedLocalAgents, ids, replace);
@@ -7391,7 +7425,10 @@ app.post('/api/agents/local/ping', async (req: Request, res: Response): Promise<
   if (!body.id) { res.status(400).json({ error: 'id required' }); return; }
   const r = await pingLocalAgent(body.id, body.prompt);
   const entry = connectedLocalAgents.get(body.id);
-  if (entry) entry.lastPing = r;
+  if (entry) {
+    entry.lastPing = r;
+    entry.lastHealthCheckAt = Date.now();
+  }
   res.json({ id: body.id, ...r });
 });
 
@@ -7411,8 +7448,10 @@ app.post('/api/agents/local/disconnect', (req: Request, res: Response): void =>
   res.json({ ok: true, connected: Array.from(connectedLocalAgents.keys()) });
 });
 
-// GET /api/agents/local/status — currently connected agents
-app.get('/api/agents/local/status', (_req: Request, res: Response): void => {
+// GET /api/agents/local/status — connected agents, optionally with a bounded live health check.
+app.get('/api/agents/local/status', async (req: Request, res: Response): Promise<void> => {
+  const check = /^(1|true|yes|on)$/i.test(String(req.query.check || ''));
+  if (check) await refreshConnectedLocalAgentHealth(false);
   res.json({ connected: Array.from(connectedLocalAgents.values()) });
 });