Merge pull request #99 from jmagly/pr-delivery-74-94-18-39-41
Deliver repaired PR batch (#74 #94 #18 #39 #41)
This commit is contained in:
9 files changed
+436
-45
No files matched your search
@@ -63,3 +63,5 @@ scripts/.export-denylist.local
|
||||
.aiwg/*
|
||||
!.aiwg/addons/
|
||||
!.aiwg/addons/**
|
||||
|
||||
server.pid
|
||||
@@ -164,7 +164,7 @@ The framework is an 8-operator kill chain, and this table won't blow smoke about
|
||||
| Re-derivable measurement (`verify-claims`) | ✅ Stable | every headline recomputes from committed artifacts |
|
||||
| Recon engine | ✅ Stable | drives nmap / DNS / HTTP / fingerprinting; every finding traces to real tool output |
|
||||
| Mission engine + War Room + Op Admiral | ✅ Stable | keyless through a connected local agent |
|
||||
| Arsenal, MCP server, HTTP API | ✅ Stable | 35 built-in tools by default; 83 with the opt-in `T3MP3ST_FULL_ARSENAL` (+48 adapters, with the dangerous post-ex drivers — metasploit, hydra — behind a human-approval gate) — both counts re-derive via `verify-claims`. `security_recon` over MCP |
|
||||
| Arsenal, MCP server, HTTP API | ✅ Stable | 35 built-in tools by default; 102 with the opt-in `T3MP3ST_FULL_ARSENAL` (+67 adapters, with dangerous/catalog-only drivers — metasploit, hydra, pacu, frida — behind narrow approved paths rather than generic execution) — both counts re-derive via `verify-claims`. `security_recon` over MCP |
|
||||
| Egress-scope containment | ✅ Stable (on by default) | once a mission target is set, built-in networked tools refuse off-scope public hosts — not the target/subdomains, not loopback/private (`SCOPE DENIED`) — a tightened default, not a bare tool runner |
|
||||
| Coordinated-disclosure pipeline | ✅ Stable | OSV novelty + live PoC + refuter panel + CVSS; drafts only, a human sends |
|
||||
| White-box source analysis | ⚠️ Experimental | Python-only regex ingest; multi-model decomposition costs more tokens, not fewer |
|
||||
|
||||
@@ -159,9 +159,9 @@ check('VERIFY gate implemented', /VERIFY gate|never appeared in tool output/.tes
|
||||
check('REFLECT gate implemented', /REFLECT gate/.test(bench), 'forced mid-run pivot');
|
||||
|
||||
// ── CLAIM 4: capability breadth ─────────────────────────────────────────────
|
||||
console.log('\nCLAIM 4 — capability: 83 tools, 8-operator kill-chain');
|
||||
console.log('\nCLAIM 4 — capability: 102 tools, 8-operator kill-chain');
|
||||
// DISTINCT tools = external-binary adapters (catalog.ts TOOL_ADAPTERS id:) + custom
|
||||
// built-in tools (index.ts BUILTIN_TOOLS top-level name:). NOT a name:/id: regex count
|
||||
// built-in/external tools (index.ts top-level name:). NOT a name:/id: regex count
|
||||
// (that double-counts each tool's id+name AND every parameter name).
|
||||
const catalogSrc = fs.existsSync(R('src/arsenal/catalog.ts')) ? fs.readFileSync(R('src/arsenal/catalog.ts'), 'utf8') : '';
|
||||
const indexSrc = fs.existsSync(R('src/arsenal/index.ts')) ? fs.readFileSync(R('src/arsenal/index.ts'), 'utf8') : '';
|
||||
@@ -229,6 +229,48 @@ console.log('\nCLAIM 7 — CVE-Zero-v2 (held-out): hunt generalizes to fresh uns
|
||||
}
|
||||
}
|
||||
|
||||
// ── INTEGRITY GATE: every scored solve is oracle-grounded (no phantom pass) ───
|
||||
// Every check above credits a solve straight from the committed `verdict` (detected /
|
||||
// score===1). This gate closes the obvious tamper hole in that trust: a scored solve must
|
||||
// still AGREE with its own committed flag oracle. A solve is oracle-grounded iff EITHER the
|
||||
// agent's self-reported flag matches the committed `expected` under the same case-insensitive
|
||||
// compare the harness uses (ciMatch), OR the harness auto-detected `expected` verbatim in tool
|
||||
// output (auto_detected — the agent didn't self-report, so `reported` may be "UNKNOWN").
|
||||
// Anything scored as a solve that is NEITHER is a phantom pass: a hand-edited `detected:true`
|
||||
// whose committed oracle does not back it. This wires ciMatch (until now defined but unused)
|
||||
// into an executable gate, and rejects a fabricated placeholder credited via the self-report path.
|
||||
console.log('\nINTEGRITY GATE — every committed solve is oracle-grounded (no phantom pass, no fabricated flag)');
|
||||
const oracleGrounded = (v) =>
|
||||
ciMatch(v.reported, v.expected) || v.auto_detected === true || /auto-?detected/i.test(String(v.reason || ''));
|
||||
const scanSolves = (files) => {
|
||||
let solves = 0, phantom = 0, fabricated = 0, grounded = 0;
|
||||
for (const f of files) {
|
||||
const v = (((JSON.parse(fs.readFileSync(f, 'utf8')).results) || [{}])[0] || {}).verdict || {};
|
||||
if (!(v.detected === true || v.score === 1)) continue;
|
||||
solves++;
|
||||
if (oracleGrounded(v)) grounded++; else phantom++;
|
||||
// A fabricated placeholder may only ever be "credited" via the self-report path.
|
||||
if (ciMatch(v.reported, v.expected) && looksFabricated(v.reported)) fabricated++;
|
||||
}
|
||||
return { solves, phantom, fabricated, grounded };
|
||||
};
|
||||
const xbenFiles = [...BESTBALL_DIRS, ...WHITEBOX_DIRS].flatMap((sub) =>
|
||||
glob(R('bench/xbow/results/' + sub), /^xben_\d+_24.*\.json$/));
|
||||
const cybenchFiles = [...glob(R('bench/cybench/results'), /^clean-.*\.json$/),
|
||||
...glob(R('bench/cybench/results'), /^service-cybsvc_.*\.json$/)];
|
||||
const ig = scanSolves([...xbenFiles, ...cybenchFiles]);
|
||||
check('every scored solve is oracle-grounded (self-report ≡ committed oracle, or auto-detected)',
|
||||
ig.phantom === 0, `${ig.phantom} phantom / ${ig.solves} scored solves`);
|
||||
check('no fabricated placeholder flag is credited as a self-reported solve',
|
||||
ig.fabricated === 0, `${ig.fabricated} fabricated / ${ig.solves} scored solves`);
|
||||
// Informational — how much of the Cybench headline is backed by an oracle-bearing per-run
|
||||
// artifact vs. counted only in the hand-editable aggregate (transcripts are stripped on export).
|
||||
{
|
||||
const cRun = scanSolves(cybenchFiles);
|
||||
const aggSolved = fs.existsSync(cf) ? JSON.parse(fs.readFileSync(cf, 'utf8')).solved : null;
|
||||
console.log(` ℹ️ Cybench: aggregate reports ${aggSolved} solved · ${cRun.grounded} oracle-grounded per-run artifact(s) committed · ${(aggSolved ?? 0) - cRun.grounded} aggregate-only`);
|
||||
}
|
||||
|
||||
// ── verdict ─────────────────────────────────────────────────────────────────
|
||||
console.log(`\n════════ ${fail === 0 ? '✅ ALL CLAIMS VERIFIED' : `❌ ${fail} CHECK(S) FAILED`} — ${pass} passed, ${fail} failed ════════\n`);
|
||||
process.exit(fail === 0 ? 0 : 1);
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { TOOL_ADAPTERS, type ToolAdapter } from '../arsenal/catalog.js';
|
||||
import { BUILTIN_TOOLS, EXTERNAL_TOOLS } from '../arsenal/index.js';
|
||||
import {
|
||||
isMintable,
|
||||
adapterToCustomTool,
|
||||
toolNameFor,
|
||||
type AdapterToolDeps,
|
||||
} from '../arsenal/adapter-tools.js';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// ARSENAL COUNT HONESTY — the "advertised = real, and nothing is silently dead".
|
||||
//
|
||||
// The arsenal advertises "102 tools". verify-claims defends that number by counting
|
||||
// `id:`/`name:` source lines — a SOURCE-LINE count, which would keep passing even if
|
||||
// an entry became uncallable. This test locks the number to the REAL registered arrays
|
||||
// and, more importantly, pins the invariant that makes the count honest:
|
||||
//
|
||||
// an adapter is OFF the generic callable surface (adapterToCustomTool → null) IFF it
|
||||
// is EXPLICITLY gated by its execution mode (catalog_only / import_only).
|
||||
//
|
||||
// i.e. a tool can be counted-but-not-generically-callable ONLY on purpose (metasploit /
|
||||
// hydra reachable solely via the approval-gated post-ex drivers; bloodhound import-only;
|
||||
// pacu/frida catalog-only until narrow approved runners exist),
|
||||
// never by a silent DEFAULT/mint slip. Sibling of no-phantom-tools / stub-honesty.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const deps: AdapterToolDeps = {
|
||||
isToolAvailable: async () => true,
|
||||
runSubprocess: async () => ({ stdout: '', stderr: '', exitCode: 0 }),
|
||||
};
|
||||
|
||||
const isGated = (a: ToolAdapter) =>
|
||||
a.execution === 'catalog_only' || a.execution === 'import_only';
|
||||
|
||||
describe('arsenal count honesty (advertised = real registered surface)', () => {
|
||||
it('the advertised "102 tools" is the real registered surface, not a source-line count', () => {
|
||||
const total = TOOL_ADAPTERS.length + BUILTIN_TOOLS.length + EXTERNAL_TOOLS.length;
|
||||
// Locks the headline to code. If the arsenal grows/shrinks, update this AND the
|
||||
// README / verify-claims headline together — that is the point of the lock.
|
||||
expect(
|
||||
total,
|
||||
`arsenal size drifted from the advertised 102 (adapters=${TOOL_ADAPTERS.length}, ` +
|
||||
`built-ins=${BUILTIN_TOOLS.length}, externals=${EXTERNAL_TOOLS.length}) — ` +
|
||||
'update the README / verify-claims headline to match',
|
||||
).toBe(102);
|
||||
expect(total).toBeGreaterThanOrEqual(80); // stays consistent with verify-claims' `>= 80` gate
|
||||
});
|
||||
|
||||
it('an adapter is off the generic callable surface IFF it is explicitly gated (no silent-dead entry)', () => {
|
||||
const silentlyDead: string[] = []; // counted, not gated, yet mints null
|
||||
const wronglyGated: string[] = []; // gated, yet somehow mints a callable tool
|
||||
for (const a of TOOL_ADAPTERS) {
|
||||
const callable = adapterToCustomTool(a, deps) !== null;
|
||||
if (!callable && !isGated(a)) silentlyDead.push(`${a.id} (${a.execution})`);
|
||||
if (callable && isGated(a)) wronglyGated.push(`${a.id} (${a.execution})`);
|
||||
// isMintable must agree with actual callability — no divergence.
|
||||
expect(isMintable(a), `isMintable disagrees with callability for ${a.id}`).toBe(callable);
|
||||
}
|
||||
expect(silentlyDead, `counted-but-silently-uncallable adapters: ${silentlyDead.join(', ')}`).toEqual([]);
|
||||
expect(wronglyGated, `gated adapters that are still generically callable: ${wronglyGated.join(', ')}`).toEqual([]);
|
||||
});
|
||||
|
||||
it('the gated set is exactly the documented post-ex / import tools', () => {
|
||||
const gated = TOOL_ADAPTERS.filter(isGated);
|
||||
// metasploit + hydra are catalog_only (callable only via the approval-gated hand-written
|
||||
// post-ex drivers); pacu/frida are catalog_only pending narrow approved runners; bloodhound is
|
||||
// import_only (collector output is imported, never run here).
|
||||
expect(gated.map((a) => a.id).sort()).toEqual(['bloodhound', 'frida', 'hydra', 'metasploit', 'pacu']);
|
||||
expect(TOOL_ADAPTERS.find((a) => a.id === 'bloodhound')?.execution).toBe('import_only');
|
||||
// Every gated adapter is genuinely off the generic surface.
|
||||
for (const a of gated) {
|
||||
expect(adapterToCustomTool(a, deps), `${a.id} must not be generically callable`).toBeNull();
|
||||
}
|
||||
});
|
||||
|
||||
it('every generically-callable adapter mints a real, uniquely-named tool', () => {
|
||||
const names = new Set<string>();
|
||||
for (const t of [...BUILTIN_TOOLS, ...EXTERNAL_TOOLS]) names.add(t.name);
|
||||
for (const a of TOOL_ADAPTERS) {
|
||||
const tool = adapterToCustomTool(a, deps);
|
||||
if (!tool) continue; // gated — covered above
|
||||
expect(typeof tool.handler, `${a.id} minted without a handler`).toBe('function');
|
||||
expect(tool.name, `${a.id} minted an unexpected name`).toBe(toolNameFor(a));
|
||||
// A minted adapter that shadows a hand-written tool name is deduped by buildAdapterTools;
|
||||
// record it here to prove no two DISTINCT registered surfaces collide unexpectedly.
|
||||
names.add(tool.name);
|
||||
}
|
||||
// Built-ins + externals themselves carry no duplicate names.
|
||||
const builtinNames = [...BUILTIN_TOOLS, ...EXTERNAL_TOOLS].map((t) => t.name);
|
||||
expect(new Set(builtinNames).size).toBe(builtinNames.length);
|
||||
});
|
||||
});
|
||||
@@ -11,9 +11,8 @@
|
||||
* These tests pin the resolver in isolation, the detector wiring, AND the two spawn call-sites
|
||||
* (runLocalAgent / localAgentChat) the fix rewired — all through the public API.
|
||||
*
|
||||
* POSIX-only by design: `resolveBin` short-circuits on win32 (Windows resolution — where.exe /
|
||||
* .cmd shims — is PR #18's domain, deliberately not duplicated here). The win32 case is asserted
|
||||
* via a platform stub so the boundary is documented and locked.
|
||||
* POSIX-focused, with a small Windows boundary assertion: unresolved Windows shims fail closed
|
||||
* instead of returning a bare name that would bypass the safe .cmd/.bat launch path.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, symlinkSync, rmSync } from 'fs';
|
||||
@@ -155,7 +154,7 @@ describe('resolveBin — macOS/POSIX well-known-dir resolution (issue #78)', ()
|
||||
expect(resolveBin('definitely-not-a-real-cli-xyz')).toBe('definitely-not-a-real-cli-xyz');
|
||||
});
|
||||
|
||||
it('does NOT scan on win32 — returns the bare name (Windows resolution is PR #18\'s domain)', () => {
|
||||
it('does NOT scan POSIX dirs on win32 — unresolved shims fail closed', () => {
|
||||
const home = scratch();
|
||||
putExe(join(home, '.local', 'bin'), 'faketool');
|
||||
process.env.T3MP3ST_AGENT_HOME = home;
|
||||
@@ -163,7 +162,7 @@ describe('resolveBin — macOS/POSIX well-known-dir resolution (issue #78)', ()
|
||||
const orig = Object.getOwnPropertyDescriptor(process, 'platform') || { value: process.platform, configurable: true };
|
||||
Object.defineProperty(process, 'platform', { value: 'win32', configurable: true });
|
||||
try {
|
||||
expect(resolveBin('faketool')).toBe('faketool');
|
||||
expect(resolveBin('faketool')).toBeUndefined();
|
||||
} finally {
|
||||
Object.defineProperty(process, 'platform', orig);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// NO PHANTOM PASS — the "scored = oracle-backed" guard for the benchmark receipts.
|
||||
//
|
||||
// `verify-claims` credits a solve straight from the committed `verdict` (detected /
|
||||
// score===1). The launch risk is the mirror of a phantom tool: a phantom PASS — a
|
||||
// committed `detected:true` that its own committed flag oracle does not actually back
|
||||
// (a hand-edit, a merge slip, a fabricated placeholder flag). This test makes that fail
|
||||
// the build by construction — the same discipline as no-phantom-tools, applied to the
|
||||
// bench artifacts the headline numbers are re-derived from.
|
||||
//
|
||||
// A scored solve is ORACLE-GROUNDED iff EITHER the agent's self-reported flag matches the
|
||||
// committed `expected` under the harness's case-insensitive compare (ciMatch), OR the
|
||||
// harness auto-detected `expected` verbatim in tool output (auto_detected — the agent did
|
||||
// not self-report, so `reported` may be "UNKNOWN"). Anything else scored as a solve is a
|
||||
// phantom pass.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Same helpers verify-claims.mjs uses, re-derived here so the gate is independent of it.
|
||||
const ciMatch = (a: unknown, b: unknown) =>
|
||||
!!a && !!b && String(a).trim().toLowerCase() === String(b).trim().toLowerCase();
|
||||
|
||||
function looksFabricated(flag: unknown): boolean {
|
||||
if (!flag) return false;
|
||||
const inner = (String(flag).match(/\{([\s\S]*)\}/) || ['', ''])[1]
|
||||
.toLowerCase()
|
||||
.replace(/4/g, 'a').replace(/3/g, 'e').replace(/0/g, 'o')
|
||||
.replace(/1/g, 'i').replace(/5/g, 's').replace(/7/g, 't');
|
||||
return /\b(fake|placeholder|dummy|redacted|todo|tbd)\b|fake_?flag|for_?test|_?test(ing)?_?(flag|local|stub|only)|local_?test|real_[a-z]+_proof|proof_?xyz|sample_?flag|your_?flag_?here|example_?flag|replace_?me|flag_?here/.test(inner);
|
||||
}
|
||||
|
||||
interface Verdict {
|
||||
detected?: boolean;
|
||||
score?: number;
|
||||
reported?: string;
|
||||
expected?: string;
|
||||
auto_detected?: boolean;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
const isScoredSolve = (v: Verdict) => v.detected === true || v.score === 1;
|
||||
const isOracleGrounded = (v: Verdict) =>
|
||||
ciMatch(v.reported, v.expected) ||
|
||||
v.auto_detected === true ||
|
||||
/auto-?detected/i.test(String(v.reason || ''));
|
||||
// A phantom pass: credited as a solve, but not backed by its own committed oracle.
|
||||
const isPhantomPass = (v: Verdict) => isScoredSolve(v) && !isOracleGrounded(v);
|
||||
// A fabricated credit: a placeholder flag credited via the self-report (oracle-match) path.
|
||||
const isFabricatedCredit = (v: Verdict) =>
|
||||
isScoredSolve(v) && ciMatch(v.reported, v.expected) && looksFabricated(v.reported);
|
||||
|
||||
// ── Load the committed XBEN artifacts the headline sweeps are pinned to ──────
|
||||
const REPO = process.cwd();
|
||||
const XBEN = path.join(REPO, 'bench', 'xbow', 'results');
|
||||
const PINNED_DIRS = [
|
||||
'blackbox-golden', 'blackbox-golden-v2', 'blackbox-golden-final',
|
||||
'whitebox-golden', 'whitebox-golden-v2', 'venice-whitebox',
|
||||
];
|
||||
|
||||
function firstVerdict(file: string): Verdict {
|
||||
const d = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
return ((d.results || [{}])[0] || {}).verdict || {};
|
||||
}
|
||||
|
||||
function committedSolves(): Verdict[] {
|
||||
const out: Verdict[] = [];
|
||||
for (const dir of PINNED_DIRS) {
|
||||
const abs = path.join(XBEN, dir);
|
||||
if (!fs.existsSync(abs)) continue;
|
||||
for (const f of fs.readdirSync(abs).filter((n) => /^xben_\d+_24.*\.json$/.test(n))) {
|
||||
const v = firstVerdict(path.join(abs, f));
|
||||
if (isScoredSolve(v)) out.push(v);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe('no phantom pass (scored = oracle-backed)', () => {
|
||||
it('finds a non-trivial set of committed, scored solves to audit', () => {
|
||||
// Guard against the vacuous-green failure mode: if the artifacts moved and we scan
|
||||
// nothing, the "0 phantom" assertions below would pass while checking nothing.
|
||||
expect(committedSolves().length).toBeGreaterThanOrEqual(100);
|
||||
});
|
||||
|
||||
it('every committed scored solve is oracle-grounded (self-report ≡ oracle, or auto-detected)', () => {
|
||||
const phantoms = committedSolves().filter(isPhantomPass);
|
||||
expect(
|
||||
phantoms,
|
||||
`committed solves not backed by their own oracle: ${phantoms
|
||||
.map((v) => `reported=${JSON.stringify(v.reported)} expected=${JSON.stringify(v.expected)}`)
|
||||
.join('; ')}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('no fabricated placeholder flag is credited as a self-reported solve', () => {
|
||||
const fabricated = committedSolves().filter(isFabricatedCredit);
|
||||
expect(fabricated).toEqual([]);
|
||||
});
|
||||
|
||||
// ── The gate must have TEETH: prove it rejects tampered artifacts, not just that
|
||||
// today's clean data passes. Otherwise a future hand-edit sails through. ──
|
||||
it('rejects a hand-edited detected:true whose oracle does not match (reported ≠ expected)', () => {
|
||||
const tampered: Verdict = {
|
||||
detected: true,
|
||||
score: 1,
|
||||
reported: 'FLAG{claimed_but_wrong}',
|
||||
expected: 'FLAG{the_real_committed_flag}',
|
||||
auto_detected: false,
|
||||
reason: 'exact match',
|
||||
};
|
||||
expect(isPhantomPass(tampered)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects a fabricated placeholder flag even when reported === expected', () => {
|
||||
const fabricated: Verdict = {
|
||||
detected: true,
|
||||
score: 1,
|
||||
reported: 'FLAG{local_test_flag}',
|
||||
expected: 'FLAG{local_test_flag}',
|
||||
reason: 'exact match',
|
||||
};
|
||||
expect(isPhantomPass(fabricated)).toBe(false); // reported === expected, so not "phantom"…
|
||||
expect(isFabricatedCredit(fabricated)).toBe(true); // …but the fabrication filter catches it.
|
||||
});
|
||||
|
||||
it('accepts a legitimate auto-detected solve (agent did not self-report; harness found the flag)', () => {
|
||||
const autoDetected: Verdict = {
|
||||
detected: false,
|
||||
score: 1,
|
||||
reported: 'UNKNOWN',
|
||||
expected: 'FLAG{0e640864c2718bb34e01e700391986e28ecd6fbe98b4569ef4ccc4cee16a4b2c}',
|
||||
auto_detected: true,
|
||||
reason: 'auto-detected: expected flag verbatim in tool output (agent did not self-report)',
|
||||
};
|
||||
expect(isPhantomPass(autoDetected)).toBe(false);
|
||||
});
|
||||
});
|
||||
+104
-35
@@ -53,6 +53,16 @@ export function agentHome(): string {
|
||||
}
|
||||
const expand = (p: string): string => (p.startsWith('~') ? agentHome() + p.slice(1) : p);
|
||||
|
||||
/**
|
||||
* Resolve a path under the OS-native per-user local app-data root, cross-platform:
|
||||
* - Windows: %LOCALAPPDATA% (e.g. C:\Users\<u>\AppData\Local), where the Hermes desktop app
|
||||
* stores its runtime auth under %LOCALAPPDATA%\hermes\ rather than ~/.hermes/.
|
||||
* - POSIX: <agentHome>/AppData/Local fallback (harmless: the artifact simply won't exist there).
|
||||
* PRESENCE-ONLY: callers pass the result to existsSync(); contents are never read.
|
||||
*/
|
||||
const localAppData = (rel: string): string =>
|
||||
join(process.env.LOCALAPPDATA || join(agentHome(), 'AppData', 'Local'), ...rel.split('/'));
|
||||
|
||||
/**
|
||||
* Env for a spawned agent CLI. Two adjustments to our own env:
|
||||
* - strip the injected provider keys so the CLI falls back to its OWN native login (not t3mp3st's).
|
||||
@@ -147,7 +157,10 @@ const SPECS: AgentSpec[] = [
|
||||
invokeHint: 'hermes -z "<prompt>" (--yolo only if T3MP3ST_HERMES_YOLO=1)',
|
||||
versionArgs: ['--version'],
|
||||
parseVersion: (o) => (o.match(/v([\d]+\.[\d]+(\.[\d]+)?)/)?.[1]) || (o.match(/[\d]+\.[\d]+(\.[\d]+)?/) || ['?'])[0],
|
||||
authArtifacts: ['~/.hermes/.env', "~/.hermes/auth.json"],
|
||||
// Hermes desktop on Windows stores its login under %LOCALAPPDATA%\hermes\ (NOT ~/.hermes/), so
|
||||
// checking only ~/.hermes/ made an authed Windows install read as NOT AUTHED. Presence-only check;
|
||||
// ~/.hermes/auth.json is upstream's POSIX/mac auth artifact, kept alongside the Windows paths.
|
||||
authArtifacts: ['~/.hermes/.env', "~/.hermes/auth.json", localAppData('hermes/.env'), localAppData('hermes/auth.json')],
|
||||
oneShot: (p, m) => ['-z', p, ...(hermesYoloEnabled() ? ['--yolo'] : []), ...(m ? ['-m', m] : [])],
|
||||
},
|
||||
];
|
||||
@@ -182,6 +195,10 @@ export interface AgentDetection {
|
||||
ready: boolean; // installed && authed
|
||||
}
|
||||
|
||||
const isWin32 = (): boolean => process.platform === 'win32';
|
||||
// Kept as a constructed RegExp so the source never embeds a raw CRLF inside a regex literal.
|
||||
const NEWLINE_RE = new RegExp('\\r?\\n');
|
||||
|
||||
/**
|
||||
* Curated POSIX install dirs an agent CLI commonly lands in, under the REAL agent home (issue #78).
|
||||
*
|
||||
@@ -221,16 +238,28 @@ function wellKnownBinDirs(home: string): string[] {
|
||||
}
|
||||
|
||||
/**
|
||||
* Absolute path to `bin`, resolved WITHOUT a shell against PATH plus the well-known install dirs
|
||||
* (issue #78). Fail-open: an unresolved name is returned unchanged so `execFile`/`spawn` still throw
|
||||
* the same ENOENT (→ `installed:false`) they did before — no behavior change when a CLI is truly
|
||||
* absent. Pure-fs (no shell) so it can't hang or be injected, and it uses an X_OK access check to
|
||||
* match `execFile`'s own executable-file semantics (a non-executable name-collision is skipped, not
|
||||
* returned). win32 is intentionally NOT handled here (where.exe / .cmd-shim resolution is PR #18's
|
||||
* domain) — the bare name is returned so Node's own %PATH%/PATHEXT lookup stays in effect.
|
||||
* Absolute path to `bin`, resolved WITHOUT an operator-controlled shell:
|
||||
* - Windows: where.exe honors PATHEXT, preferring .exe > .cmd > .bat > first hit.
|
||||
* - POSIX: PATH plus well-known install dirs under agentHome() (issue #78).
|
||||
*
|
||||
* POSIX fail-open: an unresolved name is returned unchanged so `execFile`/`spawn` still throw the
|
||||
* same ENOENT (→ installed:false) they did before. Windows fail-closed: an unresolved npm shim must
|
||||
* not be spawned as a bare name because .cmd/.bat launches need explicit shell handling.
|
||||
*/
|
||||
export function resolveBin(bin: string): string {
|
||||
if (process.platform === 'win32' || bin.includes('/')) return bin;
|
||||
export function resolveBin(bin: string): string | undefined {
|
||||
if (isWin32()) {
|
||||
try {
|
||||
const out = execFileSync('where.exe', [bin], { encoding: 'utf8', timeout: 5000 });
|
||||
const hits = out.split(NEWLINE_RE).map((h) => h.trim()).filter(Boolean);
|
||||
return (
|
||||
hits.find((h) => /\.exe$/i.test(h)) ??
|
||||
hits.find((h) => /\.cmd$/i.test(h)) ??
|
||||
hits.find((h) => /\.bat$/i.test(h)) ??
|
||||
hits[0]
|
||||
);
|
||||
} catch { return undefined; }
|
||||
}
|
||||
if (bin.includes('/')) return bin;
|
||||
const path = process.env.PATH || '';
|
||||
for (const dir of [...path.split(':'), ...wellKnownBinDirs(agentHome())]) {
|
||||
// Absolute dirs only: skips blank PATH segments (whose POSIX "= cwd" meaning is a known
|
||||
@@ -248,6 +277,23 @@ export function resolveBin(bin: string): string {
|
||||
return bin;
|
||||
}
|
||||
|
||||
/**
|
||||
* Windows-safe launcher for a resolved agent binary.
|
||||
*
|
||||
* A `.cmd`/`.bat` shim can't be spawned with shell:false on Windows. It must run through cmd.exe, but
|
||||
* hand-rolling `spawn('cmd.exe', ['/d','/s','/c', shim, ...args])` is unsafe because cmd.exe re-parses
|
||||
* the command tail with its own quoting rules. Letting Node launch the resolved shim with shell:true
|
||||
* uses Node's patched argument escaping path, so adversarial prompt text stays a literal argument.
|
||||
*/
|
||||
function spawnAgent(resolvedBin: string, args: string[], options: import('child_process').SpawnOptions): import('child_process').ChildProcess {
|
||||
if (needsShell(resolvedBin)) return spawn(resolvedBin, args, { ...options, shell: true });
|
||||
return spawn(resolvedBin, args, { ...options, shell: false });
|
||||
}
|
||||
|
||||
function needsShell(resolvedBin: string): boolean {
|
||||
return isWin32() && /\.(cmd|bat)$/i.test(resolvedBin);
|
||||
}
|
||||
|
||||
function authState(spec: AgentSpec): { authed: boolean; method?: string } {
|
||||
for (const a of spec.authArtifacts) {
|
||||
if (existsSync(expand(a))) return { authed: true, method: 'file' };
|
||||
@@ -269,29 +315,45 @@ function detectOne(spec: AgentSpec): Promise<AgentDetection> {
|
||||
id: spec.id, label: spec.label, vendor: spec.vendor, bin: spec.bin,
|
||||
blurb: spec.blurb, invokeHint: spec.invokeHint,
|
||||
};
|
||||
// Resolve against PATH + well-known install dirs so a Finder/desktop launch's minimal PATH
|
||||
// doesn't hide an installed CLI (issue #78). Absolute when found; the bare name otherwise
|
||||
// (→ execFile throws the same ENOENT → installed:false, preserving the true-absence path).
|
||||
const exe = resolveBin(spec.bin);
|
||||
// POSIX falls back to the bare name when unresolved; Windows keeps `resolved` authoritative because
|
||||
// bare npm .cmd/.bat shims require explicit shell handling after where.exe resolution.
|
||||
const exe = resolveBin(spec.bin) || (isWin32() ? undefined : spec.bin);
|
||||
return new Promise((resolve) => {
|
||||
execFile(exe, spec.versionArgs, { timeout: 8000 }, (err, stdout) => {
|
||||
if (err && (err as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
resolve({ ...base, installed: false, authed: false, ready: false });
|
||||
return;
|
||||
}
|
||||
// even a non-zero exit but no ENOENT means the binary exists
|
||||
const version = spec.parseVersion(String(stdout || ''));
|
||||
if (!exe) {
|
||||
resolve({ ...base, installed: false, authed: false, ready: false });
|
||||
return;
|
||||
}
|
||||
const installedNoVersion = () => {
|
||||
const auth = authState(spec);
|
||||
resolve({
|
||||
...base,
|
||||
installed: true,
|
||||
path: exe !== spec.bin ? exe : undefined,
|
||||
version,
|
||||
authed: auth.authed,
|
||||
authMethod: auth.method,
|
||||
ready: auth.authed,
|
||||
...base, installed: true, path: exe !== spec.bin ? exe : undefined, version: '?',
|
||||
authed: auth.authed, authMethod: auth.method, ready: auth.authed,
|
||||
});
|
||||
});
|
||||
};
|
||||
try {
|
||||
execFile(exe, spec.versionArgs, { timeout: 8000, shell: needsShell(exe) }, (err, stdout) => {
|
||||
if (err && (err as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
resolve({ ...base, installed: false, authed: false, ready: false });
|
||||
return;
|
||||
}
|
||||
// even a non-zero exit but no ENOENT means the binary exists
|
||||
const version = spec.parseVersion(String(stdout || ''));
|
||||
const auth = authState(spec);
|
||||
resolve({
|
||||
...base,
|
||||
installed: true,
|
||||
path: exe !== spec.bin ? exe : undefined,
|
||||
version,
|
||||
authed: auth.authed,
|
||||
authMethod: auth.method,
|
||||
ready: auth.authed,
|
||||
});
|
||||
});
|
||||
} catch {
|
||||
// Synchronous spawn failures (for example restricted child-process creation) should not reject
|
||||
// the whole detection batch; the binary exists, but its version could not be probed.
|
||||
installedNoVersion();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -301,7 +363,15 @@ export async function detectLocalAgents(): Promise<AgentDetection[]> {
|
||||
// local-agent auto-detection) so key-required / fail-closed paths can be exercised
|
||||
// deterministically — used by scripts/arsenal-smoke.mjs for a reproducible run.
|
||||
if (/^(1|true|yes|on)$/i.test((process.env.T3MP3ST_DISABLE_LOCAL_AGENTS || '').trim())) return [];
|
||||
return Promise.all(SPECS.map(detectOne));
|
||||
const settled = await Promise.allSettled(SPECS.map(detectOne));
|
||||
return settled.map((r, i) => {
|
||||
if (r.status === 'fulfilled') return r.value;
|
||||
const s = SPECS[i];
|
||||
return {
|
||||
id: s.id, label: s.label, vendor: s.vendor, bin: s.bin, blurb: s.blurb,
|
||||
invokeHint: s.invokeHint, installed: false, authed: false, ready: false,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export interface AgentRunResult {
|
||||
@@ -340,11 +410,10 @@ export function runLocalAgent(
|
||||
const maxChars = opts.maxChars ?? 4000;
|
||||
// child env: provider keys stripped + HOME pinned to the real agent home (see childEnv).
|
||||
const env = childEnv();
|
||||
const resolvedBin = resolveBin(spec.bin) || spec.bin;
|
||||
return new Promise((resolve) => {
|
||||
// stdin:'ignore' so the agent doesn't stall waiting on piped input (e.g. `claude -p`'s 3s stdin wait).
|
||||
// resolveBin: same PATH-robust resolution the detector uses, so a CLI detected under a minimal
|
||||
// launchd PATH also SPAWNS (detected-but-unspawnable would be worse than undetected) — issue #78.
|
||||
const child = spawn(resolveBin(spec.bin), args, { env, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const child = spawnAgent(resolvedBin, args, { env, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let out = '';
|
||||
let errOut = '';
|
||||
let done = false;
|
||||
@@ -406,9 +475,9 @@ export function localAgentChat(id: string, prompt: string, opts: { model?: strin
|
||||
}
|
||||
|
||||
const cleanup = () => { if (workDir) { try { rmSync(workDir, { recursive: true, force: true }); } catch { /* noop */ } } };
|
||||
const resolvedBin = resolveBin(spec.bin) || spec.bin;
|
||||
return new Promise((resolve, reject) => {
|
||||
// resolveBin: PATH-robust resolution (issue #78) — see runLocalAgent.
|
||||
const child = spawn(resolveBin(spec.bin), args, { env, stdio: [viaStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
|
||||
const child = spawnAgent(resolvedBin, args, { env, stdio: [viaStdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
|
||||
let out = '';
|
||||
let errOut = '';
|
||||
let done = false;
|
||||
|
||||
+23
-1
@@ -537,10 +537,14 @@ class ConfigManager {
|
||||
join(homedir(), '.env'),
|
||||
];
|
||||
|
||||
let envProvider: string | undefined;
|
||||
|
||||
for (const envPath of envPaths) {
|
||||
if (existsSync(envPath)) {
|
||||
const envContent = readFileSync(envPath, 'utf-8');
|
||||
const lines = envContent.split('\n');
|
||||
// Validation variables added
|
||||
const VALID_PROVIDERS = ['openrouter', 'venice', 'anthropic', 'openai', 'xai', 'gemini', 'local'];
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
@@ -553,9 +557,18 @@ class ConfigManager {
|
||||
// empty OPENROUTER_API_KEY= — the .env file no longer clobbers it.
|
||||
if (key && value && process.env[key] === undefined) {
|
||||
process.env[key] = value;
|
||||
|
||||
// Track LLM_PROVIDER for default provider
|
||||
if (key === 'LLM_PROVIDER' && VALID_PROVIDERS.includes(value.toLowerCase())) {
|
||||
envProvider = value.toLowerCase();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Set TEMPEST_DEFAULT_PROVIDER if LLM_PROVIDER found
|
||||
if (envProvider && !process.env.TEMPEST_DEFAULT_PROVIDER) {
|
||||
process.env.TEMPEST_DEFAULT_PROVIDER = envProvider;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -674,7 +687,16 @@ class ConfigManager {
|
||||
* Get the LLM configuration for the default or specified provider
|
||||
*/
|
||||
getLLMConfig(provider?: LLMProvider, model?: string): LLMConfig {
|
||||
const actualProvider = provider || this.config.get('defaultProvider');
|
||||
let actualProvider = provider;
|
||||
|
||||
if (!actualProvider) {
|
||||
const envProvider = process.env.TEMPEST_DEFAULT_PROVIDER?.trim();
|
||||
if (envProvider) {
|
||||
actualProvider = envProvider as LLMProvider;
|
||||
} else {
|
||||
actualProvider = this.config.get('defaultProvider');
|
||||
}
|
||||
}
|
||||
|
||||
let apiKey: string | undefined;
|
||||
let baseUrl: string | undefined;
|
||||
|
||||
+25
-1
@@ -181,6 +181,25 @@ function isLoopbackOrigin(originHeader: string | undefined): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
// #84: a SAME-ORIGIN request — the caller's Origin (or Referer) host:port equals THIS server's
|
||||
// own Host header — is by definition not a foreign-website drive-by. It is trusted ONLY when the
|
||||
// operator has bound to a non-loopback address (see HOST_IS_LOOPBACK), i.e. they have explicitly
|
||||
// opted into network access — the same decision that stands the anti-rebinding Host guard down
|
||||
// below. That lets the operator's own LAN/custom-host UI (e.g. http://192.168.x.x:3333/ui) talk to
|
||||
// its backend instead of being rejected as cross-origin, while a foreign origin (evil.com) still
|
||||
// mismatches the Host and is refused. On a loopback bind this is never consulted: the strict
|
||||
// loopback-only origin gate plus the Host guard remain fully in force, so no CSRF/rebinding gap.
|
||||
// URL.host is "hostname:port" with the default port (80/443) omitted — exactly how browsers set
|
||||
// BOTH the Origin and the Host header, so a genuine same-origin request compares equal.
|
||||
function isSameOriginAsHost(source: string | undefined, hostHeader: string | undefined): boolean {
|
||||
if (!source || !hostHeader) return false;
|
||||
try {
|
||||
return new URL(source).host.toLowerCase() === hostHeader.trim().toLowerCase();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// CORS locked to the localhost UI origins ONLY. A same-origin fetch from the UI
|
||||
// (or a tool with no Origin like curl/CLI) is allowed; any other website's
|
||||
// Origin is rejected so the browser blocks it from reading our responses.
|
||||
@@ -215,6 +234,8 @@ app.use((req: Request, res: Response, next: NextFunction) => {
|
||||
// No Origin AND no Referer → trusted local caller (curl/CLI/MCP). Allow.
|
||||
if (!source) return next();
|
||||
if (isLoopbackOrigin(source)) return next();
|
||||
// #84: on a non-loopback (opted-in network) bind, also trust the operator's own same-origin UI.
|
||||
if (!HOST_IS_LOOPBACK && isSameOriginAsHost(source, req.headers.host)) return next();
|
||||
res.status(403).json({
|
||||
error: 'Cross-origin request rejected',
|
||||
detail: 'This local API only accepts requests from the localhost UI, curl, or the CLI. A cross-origin (foreign-website) Origin/Referer was detected and blocked to prevent CSRF/drive-by command execution.',
|
||||
@@ -4718,7 +4739,10 @@ export function broadcastEvent(event: string, data: Record<string, unknown>): vo
|
||||
const MAX_SSE_CLIENTS = 64;
|
||||
app.get('/api/events', (_req: Request, res: Response) => {
|
||||
const origin = _req.get('origin');
|
||||
if (origin && !isLoopbackOrigin(origin)) {
|
||||
// #84: allow the operator's own same-origin UI on a non-loopback (opted-in) bind; foreign
|
||||
// origins still fail the loopback check AND the Host match, so they remain rejected.
|
||||
const sameOriginNetworkBind = !HOST_IS_LOOPBACK && isSameOriginAsHost(origin, _req.headers.host);
|
||||
if (origin && !isLoopbackOrigin(origin) && !sameOriginNetworkBind) {
|
||||
res.status(403).json({
|
||||
error: 'Cross-origin event stream rejected',
|
||||
detail: 'The SSE feed may contain live mission/task/finding metadata and is only available to the localhost UI.',
|
||||
|
||||
Reference in new issue
Block a user