diff --git a/scripts/obsidivm-bench.mjs b/scripts/obsidivm-bench.mjs index 705c48b..a268a36 100644 --- a/scripts/obsidivm-bench.mjs +++ b/scripts/obsidivm-bench.mjs @@ -7,7 +7,9 @@ * 2. Dispatch a t3mp3st hunter against the target's URL. * - --hunter=live : direct LLM (OpenRouter / Anthropic / OpenAI auto-detect) * - --hunter=t3mp3st: drive t3mp3st's /api/general/auto (full platform) - * - --hunter=stub : synthesized transcript (no LLM, for plumbing tests) + * - --hunter=stub : synthesized transcript (no LLM, for plumbing tests). + * NB: only the HUNTER is offline — the OBSIDIVM service + * is still required for the spec (step 1) and scoring (3). * 3. Submit the agent transcript to OBSIDIVM's /api/score/text. * 4. Persist the run via OBSIDIVM's /api/runs ledger + per-target sessions. * 5. Print a per-target table + suite aggregate (weighted % + grade). @@ -105,9 +107,12 @@ function help() { Options: --target Target id (repeatable). Default: all 14 OBSIDIVM targets. --hunter - stub: synthesized transcript (no LLM) + stub: synthesized transcript (no LLM; the OBSIDIVM + service is still required for spec + scoring) live: direct LLM call t3mp3st: drive t3mp3st's /api/general/auto + All modes need the OBSIDIVM service (see --obsidivm / docs/OBSIDIVM.md); live + and t3mp3st additionally need an LLM key. --obsidivm OBSIDIVM base URL (default http://127.0.0.1:4200) --t3mp3st t3mp3st base URL (default http://127.0.0.1:3333) --model LLM model id (default claude-opus-4-7) diff --git a/scripts/obsidivm-bridge.d.mts b/scripts/obsidivm-bridge.d.mts new file mode 100644 index 0000000..bbc7272 --- /dev/null +++ b/scripts/obsidivm-bridge.d.mts @@ -0,0 +1,33 @@ +// Types for the OBSIDIVM bridge client (consumed by the vitest test). +// Runnable logic lives in the sibling .mjs (scripts run under bare `node`, no build step). + +/** A network-level failure (connection refused / DNS / timeout) carries this flag, + * distinguishing an unreachable service from an HTTP-status error (which sets `status`). */ +export interface ObsidivmError extends Error { + unreachable?: boolean; + status?: number; + body?: unknown; +} + +export interface ObsidivmClient { + readonly baseUrl: string; + health(): Promise; + getSpec(): Promise<{ version?: string; targets?: Array<{ id: string; name?: string; port?: number }> } & Record>; + status(): Promise; + deploy(): Promise; + destroy(): Promise; + startTarget(id: string): Promise; + stopTarget(id: string): Promise; + scoreText(targetId: string, text: string): Promise>; + createRun(payload?: Record): Promise>; + attachSession(runId: string, payload?: Record): Promise; + proposeGoalposts(payload?: Record): Promise; + evolveStart(payload?: Record): Promise; + evolveStop(): Promise; + cloudgoatInstall(): Promise; + cloudgoatCreate(scenario: string): Promise; + cloudgoatDestroy(scenario: string): Promise; + cloudgoatDestroyAll(): Promise; +} + +export function obsidivm(opts?: { baseUrl?: string; timeoutMs?: number }): ObsidivmClient; diff --git a/scripts/obsidivm-bridge.mjs b/scripts/obsidivm-bridge.mjs index 97bb765..11357a3 100644 --- a/scripts/obsidivm-bridge.mjs +++ b/scripts/obsidivm-bridge.mjs @@ -31,12 +31,40 @@ export function obsidivm({ baseUrl, timeoutMs } = {}) { const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), t); try { - const res = await fetch(url, { - method, - headers: body ? { 'content-type': 'application/json' } : {}, - body: body ? JSON.stringify(body) : undefined, - signal: ctrl.signal, - }); + let res; + try { + res = await fetch(url, { + method, + headers: body ? { 'content-type': 'application/json' } : {}, + body: body ? JSON.stringify(body) : undefined, + signal: ctrl.signal, + }); + } catch (e) { + // The fetch itself failed — distinct from an HTTP-status error below. A + // bare `fetch failed` here is the #156 dead-end; make it self-service. + // Three sub-cases, kept distinct so the message points at the real fix: + // AbortError → the request timed out + // ERR_INVALID_URL → OBSIDIVM_URL is malformed (a config error, NOT a + // down service — don't send the user to start one) + // otherwise → nothing is listening on the base URL + if (e?.name === 'AbortError') { + throw new Error(`OBSIDIVM ${method} ${path} timed out after ${t}ms (${base}).`); + } + if (e?.cause?.code === 'ERR_INVALID_URL') { + throw new Error( + `Invalid OBSIDIVM base URL ${JSON.stringify(base)} — set OBSIDIVM_URL (or --obsidivm) ` + + `to a full URL like http://127.0.0.1:4200.`, + ); + } + const err = new Error( + `OBSIDIVM service unreachable at ${base} — ${method} ${path} failed ` + + `(${e?.cause?.code || e?.message || 'network error'}). Is it running? ` + + `Start it per docs/OBSIDIVM.md (python3 range.py, listens on :4200), or set OBSIDIVM_URL.`, + ); + err.unreachable = true; + err.cause = e; + throw err; + } const text = await res.text(); let parsed; try { parsed = JSON.parse(text); } catch { parsed = { raw: text }; } diff --git a/src/__tests__/obsidivm-bridge.test.ts b/src/__tests__/obsidivm-bridge.test.ts new file mode 100644 index 0000000..ddfcf28 --- /dev/null +++ b/src/__tests__/obsidivm-bridge.test.ts @@ -0,0 +1,89 @@ +/** + * Diagnostics guard for issue #156 — "Benchmark showing error" / `FATAL: fetch failed`. + * + * The obsidivm:* benchmarks are a thin HTTP client over the separate OBSIDIVM + * range service (default http://127.0.0.1:4200, see docs/OBSIDIVM.md). Every + * request routes through the bridge's `call()`. When the service is down, the + * raw `fetch` throws a bare `TypeError: fetch failed`, which surfaced to the user + * as an opaque `FATAL: fetch failed` naming neither the service, the URL, nor how + * to start it — even in `--hunter stub` mode, whose grading step still needs it. + * + * This pins that a connection failure becomes an ACTIONABLE error: it says the + * OBSIDIVM service is unreachable, names the base URL, and points at the escape + * hatches (docs/OBSIDIVM.md / OBSIDIVM_URL). + */ +import { describe, it, expect } from 'vitest'; +import net from 'node:net'; +import { obsidivm } from '../../scripts/obsidivm-bridge.mjs'; + +/** + * A guaranteed-closed localhost port: bind an ephemeral listener, capture its + * port, then close it — a connection there now yields ECONNREFUSED, + * deterministically and with no external network (hermetic, no mocking). + */ +function closedPort(): Promise { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.once('error', reject); + srv.listen(0, '127.0.0.1', () => { + const { port } = srv.address() as net.AddressInfo; + srv.close(() => resolve(port)); + }); + }); +} + +describe('obsidivm bridge — unreachable-service diagnostics (#156)', () => { + it('turns a connection failure into an actionable error, not a bare "fetch failed"', async () => { + const port = await closedPort(); + const o = obsidivm({ baseUrl: `http://127.0.0.1:${port}`, timeoutMs: 2000 }); + + const err = (await o.getSpec().then(() => null, (e: unknown) => e)) as (Error & { unreachable?: boolean }) | null; + + expect(err, 'getSpec should reject when the service is down').toBeTruthy(); + // names the failure, the URL, and the self-service escape hatches + expect(err!.message).toMatch(/OBSIDIVM service unreachable/i); + expect(err!.message).toContain(`127.0.0.1:${port}`); + expect(err!.message).toMatch(/OBSIDIVM_URL|docs\/OBSIDIVM\.md/); + // and is flagged so callers can distinguish "down" from an HTTP-status error + expect(err!.unreachable).toBe(true); + }); + + it('reports a timeout distinctly (not as "unreachable")', async () => { + // A server that accepts the connection but never responds forces the + // AbortController timeout path — distinct from a refused connection. + const srv = net.createServer(() => { + /* accept, then hang: never write a response */ + }); + try { + const port: number = await new Promise((resolve) => + srv.listen(0, '127.0.0.1', () => resolve((srv.address() as net.AddressInfo).port)), + ); + const o = obsidivm({ baseUrl: `http://127.0.0.1:${port}`, timeoutMs: 200 }); + const err = (await o.getSpec().then(() => null, (e: unknown) => e)) as (Error & { unreachable?: boolean }) | null; + + expect(err, 'getSpec should reject on timeout').toBeTruthy(); + expect(err!.message).toMatch(/timed out after \d+ms/i); + expect(err!.unreachable).toBeUndefined(); // a timeout is not "service down" + } finally { + // Fire-and-forget: the aborted request leaves a half-open socket, so + // awaiting close() would hang. Destroy connections and close without + // waiting — the assertions are already done. (closeAllConnections lands in + // newer @types/node than this repo pins, hence the cast.) + (srv as net.Server & { closeAllConnections?: () => void }).closeAllConnections?.(); + srv.close(); + } + }); + + it('reports a malformed base URL as a config error, not "unreachable"', async () => { + // ERR_INVALID_URL is the operator's mistake, not a down service — the message + // must point at OBSIDIVM_URL, not at starting the range service. + const o = obsidivm({ baseUrl: 'not-a-valid-url' }); + const err = (await o.getSpec().then(() => null, (e: unknown) => e)) as (Error & { unreachable?: boolean }) | null; + + expect(err, 'getSpec should reject on a bad URL').toBeTruthy(); + expect(err!.message).toMatch(/invalid OBSIDIVM base URL/i); + expect(err!.message).toMatch(/OBSIDIVM_URL/); + expect(err!.message).not.toMatch(/unreachable|is it running/i); + expect(err!.unreachable).toBeUndefined(); + }); +});