From 41f0c140397ac41e09d03cb255d77540e61a9c00 Mon Sep 17 00:00:00 2001 From: Lyubomir Bozhinov <12250247+lyubomir-bozhinov@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:24:20 +0300 Subject: [PATCH] fix(obsidivm): actionable error when the range service is unreachable (#156) (#158) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(obsidivm): RED — bridge should give an actionable error when the service is down The obsidivm:* benchmarks fetch from the OBSIDIVM range service; when it is not running the bridge throws a bare 'fetch failed' (surfaced as 'FATAL: fetch failed'), naming neither the service, the URL, nor how to start it. Pins that a connection failure becomes an actionable, self-service error. Fails on main: the message is 'fetch failed'. Refs #156 * fix(obsidivm): actionable error when the range service is unreachable The obsidivm:* benchmarks fetch from the OBSIDIVM range service; when it is not running the bridge's shared call() let the raw 'fetch failed' propagate, which surfaced as an opaque 'FATAL: fetch failed' (#156) naming neither the service, the URL, nor how to start it — even in --hunter stub, whose grading step still needs it. Wrap the fetch in call() (the single choke point for every OBSIDIVM request) so a connection failure becomes: 'OBSIDIVM service unreachable at ... Start it per docs/OBSIDIVM.md (python3 range.py, listens on :4200), or set OBSIDIVM_URL', flagged err.unreachable to distinguish it from an HTTP-status error; a timeout is reported distinctly. Clarify in the bench --help/usage that every hunter mode (stub included) needs the OBSIDIVM service, live/t3mp3st also a key. Adds scripts/obsidivm-bridge.d.mts so the new test types cleanly. Closes #156 * fix(obsidivm): distinguish invalid-URL from unreachable; test timeout path Adversarial review of the diff found two gaps: - A malformed OBSIDIVM_URL (ERR_INVALID_URL) was folded into the 'service unreachable — is it running?' path, misdirecting the user to start a service when the real fix is the URL. Now branched to a distinct 'Invalid OBSIDIVM base URL … set OBSIDIVM_URL' error, no err.unreachable flag. - The timeout (AbortError) branch had no coverage. Added a test using a never-responding TCP server + short timeoutMs asserting 'timed out after Nms' and NOT unreachable; and a test for the invalid-URL branch. Also completed the .d.mts stub with the cloudgoat* methods it was missing. Refs #156 --- scripts/obsidivm-bench.mjs | 9 ++- scripts/obsidivm-bridge.d.mts | 33 ++++++++++ scripts/obsidivm-bridge.mjs | 40 ++++++++++-- src/__tests__/obsidivm-bridge.test.ts | 89 +++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 8 deletions(-) create mode 100644 scripts/obsidivm-bridge.d.mts create mode 100644 src/__tests__/obsidivm-bridge.test.ts diff --git a/scripts/obsidivm-bench.mjs b/scripts/obsidivm-bench.mjs index 705c48b..a268a36 100644 --- a/scripts/obsidivm-bench.mjs +++ b/scripts/obsidivm-bench.mjs @@ -7,7 +7,9 @@ * 2. Dispatch a t3mp3st hunter against the target's URL. * - --hunter=live : direct LLM (OpenRouter / Anthropic / OpenAI auto-detect) * - --hunter=t3mp3st: drive t3mp3st's /api/general/auto (full platform) - * - --hunter=stub : synthesized transcript (no LLM, for plumbing tests) + * - --hunter=stub : synthesized transcript (no LLM, for plumbing tests). + * NB: only the HUNTER is offline — the OBSIDIVM service + * is still required for the spec (step 1) and scoring (3). * 3. Submit the agent transcript to OBSIDIVM's /api/score/text. * 4. Persist the run via OBSIDIVM's /api/runs ledger + per-target sessions. * 5. Print a per-target table + suite aggregate (weighted % + grade). @@ -105,9 +107,12 @@ function help() { Options: --target Target id (repeatable). Default: all 14 OBSIDIVM targets. --hunter - stub: synthesized transcript (no LLM) + stub: synthesized transcript (no LLM; the OBSIDIVM + service is still required for spec + scoring) live: direct LLM call t3mp3st: drive t3mp3st's /api/general/auto + All modes need the OBSIDIVM service (see --obsidivm / docs/OBSIDIVM.md); live + and t3mp3st additionally need an LLM key. --obsidivm OBSIDIVM base URL (default http://127.0.0.1:4200) --t3mp3st t3mp3st base URL (default http://127.0.0.1:3333) --model LLM model id (default claude-opus-4-7) diff --git a/scripts/obsidivm-bridge.d.mts b/scripts/obsidivm-bridge.d.mts new file mode 100644 index 0000000..bbc7272 --- /dev/null +++ b/scripts/obsidivm-bridge.d.mts @@ -0,0 +1,33 @@ +// Types for the OBSIDIVM bridge client (consumed by the vitest test). +// Runnable logic lives in the sibling .mjs (scripts run under bare `node`, no build step). + +/** A network-level failure (connection refused / DNS / timeout) carries this flag, + * distinguishing an unreachable service from an HTTP-status error (which sets `status`). */ +export interface ObsidivmError extends Error { + unreachable?: boolean; + status?: number; + body?: unknown; +} + +export interface ObsidivmClient { + readonly baseUrl: string; + health(): Promise; + getSpec(): Promise<{ version?: string; targets?: Array<{ id: string; name?: string; port?: number }> } & Record>; + status(): Promise; + deploy(): Promise; + destroy(): Promise; + startTarget(id: string): Promise; + stopTarget(id: string): Promise; + scoreText(targetId: string, text: string): Promise>; + createRun(payload?: Record): Promise>; + attachSession(runId: string, payload?: Record): Promise; + proposeGoalposts(payload?: Record): Promise; + evolveStart(payload?: Record): Promise; + evolveStop(): Promise; + cloudgoatInstall(): Promise; + cloudgoatCreate(scenario: string): Promise; + cloudgoatDestroy(scenario: string): Promise; + cloudgoatDestroyAll(): Promise; +} + +export function obsidivm(opts?: { baseUrl?: string; timeoutMs?: number }): ObsidivmClient; diff --git a/scripts/obsidivm-bridge.mjs b/scripts/obsidivm-bridge.mjs index 97bb765..11357a3 100644 --- a/scripts/obsidivm-bridge.mjs +++ b/scripts/obsidivm-bridge.mjs @@ -31,12 +31,40 @@ export function obsidivm({ baseUrl, timeoutMs } = {}) { const ctrl = new AbortController(); const timer = setTimeout(() => ctrl.abort(), t); try { - const res = await fetch(url, { - method, - headers: body ? { 'content-type': 'application/json' } : {}, - body: body ? JSON.stringify(body) : undefined, - signal: ctrl.signal, - }); + let res; + try { + res = await fetch(url, { + method, + headers: body ? { 'content-type': 'application/json' } : {}, + body: body ? JSON.stringify(body) : undefined, + signal: ctrl.signal, + }); + } catch (e) { + // The fetch itself failed — distinct from an HTTP-status error below. A + // bare `fetch failed` here is the #156 dead-end; make it self-service. + // Three sub-cases, kept distinct so the message points at the real fix: + // AbortError → the request timed out + // ERR_INVALID_URL → OBSIDIVM_URL is malformed (a config error, NOT a + // down service — don't send the user to start one) + // otherwise → nothing is listening on the base URL + if (e?.name === 'AbortError') { + throw new Error(`OBSIDIVM ${method} ${path} timed out after ${t}ms (${base}).`); + } + if (e?.cause?.code === 'ERR_INVALID_URL') { + throw new Error( + `Invalid OBSIDIVM base URL ${JSON.stringify(base)} — set OBSIDIVM_URL (or --obsidivm) ` + + `to a full URL like http://127.0.0.1:4200.`, + ); + } + const err = new Error( + `OBSIDIVM service unreachable at ${base} — ${method} ${path} failed ` + + `(${e?.cause?.code || e?.message || 'network error'}). Is it running? ` + + `Start it per docs/OBSIDIVM.md (python3 range.py, listens on :4200), or set OBSIDIVM_URL.`, + ); + err.unreachable = true; + err.cause = e; + throw err; + } const text = await res.text(); let parsed; try { parsed = JSON.parse(text); } catch { parsed = { raw: text }; } diff --git a/src/__tests__/obsidivm-bridge.test.ts b/src/__tests__/obsidivm-bridge.test.ts new file mode 100644 index 0000000..ddfcf28 --- /dev/null +++ b/src/__tests__/obsidivm-bridge.test.ts @@ -0,0 +1,89 @@ +/** + * Diagnostics guard for issue #156 — "Benchmark showing error" / `FATAL: fetch failed`. + * + * The obsidivm:* benchmarks are a thin HTTP client over the separate OBSIDIVM + * range service (default http://127.0.0.1:4200, see docs/OBSIDIVM.md). Every + * request routes through the bridge's `call()`. When the service is down, the + * raw `fetch` throws a bare `TypeError: fetch failed`, which surfaced to the user + * as an opaque `FATAL: fetch failed` naming neither the service, the URL, nor how + * to start it — even in `--hunter stub` mode, whose grading step still needs it. + * + * This pins that a connection failure becomes an ACTIONABLE error: it says the + * OBSIDIVM service is unreachable, names the base URL, and points at the escape + * hatches (docs/OBSIDIVM.md / OBSIDIVM_URL). + */ +import { describe, it, expect } from 'vitest'; +import net from 'node:net'; +import { obsidivm } from '../../scripts/obsidivm-bridge.mjs'; + +/** + * A guaranteed-closed localhost port: bind an ephemeral listener, capture its + * port, then close it — a connection there now yields ECONNREFUSED, + * deterministically and with no external network (hermetic, no mocking). + */ +function closedPort(): Promise { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.once('error', reject); + srv.listen(0, '127.0.0.1', () => { + const { port } = srv.address() as net.AddressInfo; + srv.close(() => resolve(port)); + }); + }); +} + +describe('obsidivm bridge — unreachable-service diagnostics (#156)', () => { + it('turns a connection failure into an actionable error, not a bare "fetch failed"', async () => { + const port = await closedPort(); + const o = obsidivm({ baseUrl: `http://127.0.0.1:${port}`, timeoutMs: 2000 }); + + const err = (await o.getSpec().then(() => null, (e: unknown) => e)) as (Error & { unreachable?: boolean }) | null; + + expect(err, 'getSpec should reject when the service is down').toBeTruthy(); + // names the failure, the URL, and the self-service escape hatches + expect(err!.message).toMatch(/OBSIDIVM service unreachable/i); + expect(err!.message).toContain(`127.0.0.1:${port}`); + expect(err!.message).toMatch(/OBSIDIVM_URL|docs\/OBSIDIVM\.md/); + // and is flagged so callers can distinguish "down" from an HTTP-status error + expect(err!.unreachable).toBe(true); + }); + + it('reports a timeout distinctly (not as "unreachable")', async () => { + // A server that accepts the connection but never responds forces the + // AbortController timeout path — distinct from a refused connection. + const srv = net.createServer(() => { + /* accept, then hang: never write a response */ + }); + try { + const port: number = await new Promise((resolve) => + srv.listen(0, '127.0.0.1', () => resolve((srv.address() as net.AddressInfo).port)), + ); + const o = obsidivm({ baseUrl: `http://127.0.0.1:${port}`, timeoutMs: 200 }); + const err = (await o.getSpec().then(() => null, (e: unknown) => e)) as (Error & { unreachable?: boolean }) | null; + + expect(err, 'getSpec should reject on timeout').toBeTruthy(); + expect(err!.message).toMatch(/timed out after \d+ms/i); + expect(err!.unreachable).toBeUndefined(); // a timeout is not "service down" + } finally { + // Fire-and-forget: the aborted request leaves a half-open socket, so + // awaiting close() would hang. Destroy connections and close without + // waiting — the assertions are already done. (closeAllConnections lands in + // newer @types/node than this repo pins, hence the cast.) + (srv as net.Server & { closeAllConnections?: () => void }).closeAllConnections?.(); + srv.close(); + } + }); + + it('reports a malformed base URL as a config error, not "unreachable"', async () => { + // ERR_INVALID_URL is the operator's mistake, not a down service — the message + // must point at OBSIDIVM_URL, not at starting the range service. + const o = obsidivm({ baseUrl: 'not-a-valid-url' }); + const err = (await o.getSpec().then(() => null, (e: unknown) => e)) as (Error & { unreachable?: boolean }) | null; + + expect(err, 'getSpec should reject on a bad URL').toBeTruthy(); + expect(err!.message).toMatch(/invalid OBSIDIVM base URL/i); + expect(err!.message).toMatch(/OBSIDIVM_URL/); + expect(err!.message).not.toMatch(/unreachable|is it running/i); + expect(err!.unreachable).toBeUndefined(); + }); +});