From f8c283377c76b1b00daeacd679505a7a0a72a458 Mon Sep 17 00:00:00 2001 From: Pazificateur69 <158148129+Pazificateur69@users.noreply.github.com> Date: Thu, 16 Jul 2026 01:43:51 +0200 Subject: [PATCH] feat(arsenal): parse scanner JSON into structured findings (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generic adapter factory returned raw stdout only (`adapter-tools.ts` success path), so 35 catalogued scanners with `parserStatus: 'planned'` never populated the structured `ToolResult.findings` channel — their output was dropped on the floor and the agent could neither verify nor chain on it. The whole downstream pipe already exists and is honest-by-construction: the agent loop stamps `provenance:'tool'` + the raw output, the operator materialises each finding with output-evidence, and the live gate (`evidence/gate.ts`) marks it verified. Add `src/arsenal/parsers.ts` — `parseToolOutput(toolId, rawStdout) → ToolFinding[]` for the five scanners that already emit machine-readable JSON via their existing arg templates (zero template changes): nuclei (-jsonl), httpx (-json), dalfox (--format json), ffuf (-of json), katana (-jsonl). Parsers are pure, dependency-free, and NEVER throw: empty/garbled/unknown input yields `[]`, never a fabricated finding (same discipline as stub-honesty / no-phantom-tools). The factory attaches parsed findings while ALWAYS keeping the raw stdout as `output` — the evidence of record the gate checks. - nuclei/dalfox emit CWE, which the tool channel dropped: add optional `cwe?: string[]` to `ToolFinding` and thread it through the operator's finding materialisation. - flip `parserStatus: 'planned' → 'structured'` for httpx, dalfox, katana (nuclei/ffuf were already structured). Adds `src/__tests__/parsers.test.ts` + fixtures: per-scanner field mapping, the honesty contract (empty/garbled/adversarial → [], no throw), a factory-wiring check that a minted adapter returns findings while preserving raw stdout, and an end-to-end assertion that every parsed finding PASSES `gateLiveFinding` with provenance='tool'. The minted-adapter path is gated behind T3MP3ST_FULL_ARSENAL, so the benchmarked default arsenal is untouched. Supply-chain JSON tools (semgrep/gitleaks/trivy/grype) and secret scanners (need redaction) are a clean follow-up. --- src/__tests__/fixtures/dalfox.json | 1 + src/__tests__/fixtures/ffuf.json | 1 + src/__tests__/fixtures/httpx.jsonl | 2 + src/__tests__/fixtures/katana.jsonl | 3 + src/__tests__/fixtures/nuclei.jsonl | 3 + src/__tests__/parsers.test.ts | 187 +++++++++++++++++++++++++ src/arsenal/adapter-tools.ts | 12 +- src/arsenal/catalog.ts | 6 +- src/arsenal/parsers.ts | 210 ++++++++++++++++++++++++++++ src/operators/index.ts | 1 + src/types/index.ts | 1 + 11 files changed, 423 insertions(+), 4 deletions(-) create mode 100644 src/__tests__/fixtures/dalfox.json create mode 100644 src/__tests__/fixtures/ffuf.json create mode 100644 src/__tests__/fixtures/httpx.jsonl create mode 100644 src/__tests__/fixtures/katana.jsonl create mode 100644 src/__tests__/fixtures/nuclei.jsonl create mode 100644 src/__tests__/parsers.test.ts create mode 100644 src/arsenal/parsers.ts diff --git a/src/__tests__/fixtures/dalfox.json b/src/__tests__/fixtures/dalfox.json new file mode 100644 index 0000000..0eb88fd --- /dev/null +++ b/src/__tests__/fixtures/dalfox.json @@ -0,0 +1 @@ +[{"type":"V","inject_type":"inHTML-URL","severity":"high","cwe":"CWE-79","method":"GET","param":"q","data":"https://target.example.com/?q=","evidence":"reflected in HTML body","poc":"","message_str":"reflected XSS"}] diff --git a/src/__tests__/fixtures/ffuf.json b/src/__tests__/fixtures/ffuf.json new file mode 100644 index 0000000..a346347 --- /dev/null +++ b/src/__tests__/fixtures/ffuf.json @@ -0,0 +1 @@ +{"commandline":"ffuf -u https://target.example.com/FUZZ -w common.txt","results":[{"input":{"FUZZ":"admin"},"status":200,"length":1234,"words":56,"url":"https://target.example.com/admin"},{"input":{"FUZZ":"login"},"status":301,"length":0,"words":1,"url":"https://target.example.com/login"}]} diff --git a/src/__tests__/fixtures/httpx.jsonl b/src/__tests__/fixtures/httpx.jsonl new file mode 100644 index 0000000..f44a408 --- /dev/null +++ b/src/__tests__/fixtures/httpx.jsonl @@ -0,0 +1,2 @@ +{"url":"https://target.example.com","status_code":200,"title":"Example Corp","tech":["Nginx","PHP","jQuery"],"webserver":"nginx"} +{"url":"https://api.example.com","status_code":403,"title":"Forbidden"} diff --git a/src/__tests__/fixtures/katana.jsonl b/src/__tests__/fixtures/katana.jsonl new file mode 100644 index 0000000..85dc59f --- /dev/null +++ b/src/__tests__/fixtures/katana.jsonl @@ -0,0 +1,3 @@ +{"timestamp":"2026-01-01T00:00:00Z","request":{"method":"GET","endpoint":"https://target.example.com/"}} +{"timestamp":"2026-01-01T00:00:01Z","request":{"method":"GET","endpoint":"https://target.example.com/about"}} +{"timestamp":"2026-01-01T00:00:02Z","request":{"method":"GET","endpoint":"https://target.example.com/contact"}} diff --git a/src/__tests__/fixtures/nuclei.jsonl b/src/__tests__/fixtures/nuclei.jsonl new file mode 100644 index 0000000..c27468e --- /dev/null +++ b/src/__tests__/fixtures/nuclei.jsonl @@ -0,0 +1,3 @@ +[INF] nuclei banner line that is not JSON and must be skipped +{"template-id":"CVE-2021-44228","info":{"name":"Apache Log4j RCE","severity":"critical","description":"Log4j JNDI remote code execution via crafted header","classification":{"cve-id":["CVE-2021-44228"],"cwe-id":["CWE-502"],"cvss-score":10.0},"remediation":"Upgrade log4j to 2.17.0 or later"},"host":"https://target.example.com","matched-at":"https://target.example.com/api"} +{"template-id":"nginx-version","info":{"name":"Nginx Version Disclosure","severity":"info"},"host":"https://target.example.com","matched-at":"https://target.example.com"} diff --git a/src/__tests__/parsers.test.ts b/src/__tests__/parsers.test.ts new file mode 100644 index 0000000..3915b88 --- /dev/null +++ b/src/__tests__/parsers.test.ts @@ -0,0 +1,187 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { parseToolOutput, hasParser, PARSED_TOOL_IDS } from '../arsenal/parsers.js'; +import { gateLiveFinding } from '../evidence/gate.js'; +import { adapterToCustomTool, type AdapterToolDeps } from '../arsenal/adapter-tools.js'; +import { TOOL_ADAPTERS } from '../arsenal/catalog.js'; +import { KillChainPhase } from '../types/index.js'; +import type { CustomTool, Finding, ToolFinding } from '../types/index.js'; + +// ───────────────────────────────────────────────────────────────────────────── +// STRUCTURED PARSERS — turn real scanner stdout into ToolFinding[], honestly. +// +// Two things are pinned here: +// 1) each JSON-emitting scanner's output maps to the right structured fields, and +// 2) the honesty contract holds: empty / garbled / unknown input yields [] (never a +// fabricated finding), and a parsed finding — carried through the SAME shape the +// agent loop + operator build — PASSES the live provenance gate. That last check +// is the point of the whole layer: structured output that the honesty spine accepts. +// ───────────────────────────────────────────────────────────────────────────── + +const fixture = (name: string): string => + readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8'); + +describe('parseToolOutput — field mapping per scanner', () => { + it('nuclei: one finding per JSONL match; skips non-JSON lines; maps cve/cwe/cvss/severity', () => { + const f = parseToolOutput('nuclei', fixture('nuclei.jsonl')); + expect(f).toHaveLength(2); // banner line skipped + const log4j = f[0]; + expect(log4j.title).toBe('Apache Log4j RCE'); + expect(log4j.severity).toBe('critical'); + expect(log4j.cve).toEqual(['CVE-2021-44228']); + expect(log4j.cwe).toEqual(['CWE-502']); + expect(log4j.cvss).toBe(10.0); + expect(log4j.remediation).toMatch(/2\.17\.0/); + expect(log4j.details).toContain('target.example.com'); + expect(f[1].severity).toBe('info'); // nginx version disclosure + }); + + it('httpx: one info finding per live host with url/status/tech in details', () => { + const f = parseToolOutput('httpx', fixture('httpx.jsonl')); + expect(f).toHaveLength(2); + expect(f[0].severity).toBe('info'); + expect(f[0].title).toContain('target.example.com'); + expect(f[0].details).toContain('status: 200'); + expect(f[0].details).toContain('Nginx'); + }); + + it('dalfox: one finding per XSS probe with severity + CWE', () => { + const f = parseToolOutput('dalfox', fixture('dalfox.json')); + expect(f).toHaveLength(1); + expect(f[0].severity).toBe('high'); + expect(f[0].cwe).toEqual(['CWE-79']); + expect(f[0].title).toContain('q'); // the param + expect(f[0].details).toContain('poc'); + }); + + it('ffuf: a single aggregate finding counting discovered paths (no per-path flood)', () => { + const f = parseToolOutput('ffuf', fixture('ffuf.json')); + expect(f).toHaveLength(1); + expect(f[0].title).toBe('ffuf: 2 path(s) discovered'); + expect(f[0].severity).toBe('info'); + expect(f[0].details).toContain('admin'); + expect(f[0].details).toContain('login'); + }); + + it('katana: a single aggregate finding counting discovered endpoints', () => { + const f = parseToolOutput('katana', fixture('katana.jsonl')); + expect(f).toHaveLength(1); + expect(f[0].title).toBe('katana: 3 endpoint(s) discovered'); + expect(f[0].details).toContain('/about'); + }); +}); + +describe('parseToolOutput — honesty contract (never fabricate, never throw)', () => { + it('returns [] for empty, whitespace, and unparseable output', () => { + for (const raw of ['', ' ', '\n\n', 'not json at all', '{bad json', 'error']) { + expect(parseToolOutput('nuclei', raw)).toEqual([]); + expect(parseToolOutput('ffuf', raw)).toEqual([]); + expect(parseToolOutput('dalfox', raw)).toEqual([]); + } + }); + + it('returns [] for a tool with no parser wired', () => { + expect(parseToolOutput('sqlmap', fixture('nuclei.jsonl'))).toEqual([]); + expect(parseToolOutput('metasploit', 'anything')).toEqual([]); + expect(hasParser('sqlmap')).toBe(false); + }); + + it('never throws on adversarial / deeply-nested / wrong-shape input', () => { + const evil = [ + JSON.stringify({ results: 'not-an-array' }), + '[1,2,3]', + '{"info":null}\n{"info":{"classification":"nope"}}', + '['.repeat(500), + JSON.stringify({ results: [{ input: null, status: {} }] }), + ]; + for (const id of PARSED_TOOL_IDS) { + for (const raw of evil) { + expect(() => parseToolOutput(id, raw)).not.toThrow(); + expect(Array.isArray(parseToolOutput(id, raw))).toBe(true); + } + } + }); + + it('exposes exactly the wired parser ids', () => { + expect([...PARSED_TOOL_IDS].sort()).toEqual(['dalfox', 'ffuf', 'httpx', 'katana', 'nuclei']); + }); +}); + +describe('end-to-end: a parsed finding passes the live provenance gate', () => { + // Mirror how the agent loop stamps provenance (src/agent/index.ts) and how the operator + // materialises a ToolFinding into a Finding with output-evidence (src/operators/index.ts), + // then run the real gate. This proves parsed output is accepted by the honesty spine. + const materialise = (tf: ToolFinding, rawOutput: string, toolName: string): Finding => { + const stamped = { ...tf, provenance: 'tool' as const, toolName, toolOutput: rawOutput.slice(0, 4000) }; + return { + id: 'finding-test', + title: stamped.title, + description: stamped.details, + severity: stamped.severity, + targetId: 'target-1', + operatorId: 'op-1', + phase: KillChainPhase.RECON, + cvss: stamped.cvss, + cve: stamped.cve, + cwe: stamped.cwe, + evidence: stamped.provenance === 'tool' + ? [{ type: 'output', content: stamped.toolOutput || stamped.details || '', timestamp: 1, metadata: { tool: stamped.toolName } }] + : [], + remediation: stamped.remediation, + discoveredAt: 1, + }; + }; + + it('a parsed nuclei critical finding is gated PASS with provenance=tool', () => { + const raw = fixture('nuclei.jsonl'); + const tf = parseToolOutput('nuclei', raw)[0]; + const gate = gateLiveFinding(materialise(tf, raw, 'nuclei_tool')); + expect(gate.passed).toBe(true); + expect(gate.provenance).toBe('tool'); + }); + + it('every parsed finding across all scanners is gate-passable (real tool evidence)', () => { + for (const [id, file] of [ + ['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'], + ['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'], + ] as const) { + const raw = fixture(file); + for (const tf of parseToolOutput(id, raw)) { + const gate = gateLiveFinding(materialise(tf, raw, `${id}_tool`)); + expect(gate.passed, `${id} finding "${tf.title}" should pass the gate`).toBe(true); + expect(gate.provenance).toBe('tool'); + } + } + }); +}); + +describe('factory wiring: a minted adapter returns structured findings from real stdout', () => { + // Proves the actual code path (adapter-tools.ts success return) populates ToolResult.findings — + // not just that the pure parser works. Deps are fakes; runSubprocess replays a fixture as stdout. + const depsReturning = (stdout: string): AdapterToolDeps => ({ + isToolAvailable: async () => true, + runSubprocess: async () => ({ stdout, stderr: '', exitCode: 0 }), + }); + const mint = (id: string, deps: AdapterToolDeps): CustomTool => { + const adapter = TOOL_ADAPTERS.find((a) => a.id === id); + if (!adapter) throw new Error(`fixture: adapter '${id}' not found`); + const tool = adapterToCustomTool(adapter, deps); + if (!tool) throw new Error(`fixture: adapter '${id}' is not mintable`); + return tool; + }; + + it('nuclei_tool surfaces parsed findings (with cwe/cve) AND keeps the raw stdout as evidence', async () => { + const raw = fixture('nuclei.jsonl'); + const res = await mint('nuclei', depsReturning(raw)).handler({ parameters: { url: 'https://target.example.com' } }); + expect(res.success).toBe(true); + expect(res.output).toBe(raw); // raw stdout preserved as the evidence of record + expect(res.findings).toHaveLength(2); + expect(res.findings?.[0].cwe).toEqual(['CWE-502']); + }); + + it('a tool whose scan yields no parseable output leaves findings unset (no fabrication)', async () => { + const res = await mint('nuclei', depsReturning('no findings here\n')).handler({ parameters: { url: 'https://target.example.com' } }); + expect(res.success).toBe(true); + expect(res.findings).toBeUndefined(); + }); +}); diff --git a/src/arsenal/adapter-tools.ts b/src/arsenal/adapter-tools.ts index 0b3029d..8223613 100644 --- a/src/arsenal/adapter-tools.ts +++ b/src/arsenal/adapter-tools.ts @@ -25,6 +25,7 @@ import type { ToolAdapter } from './catalog.js'; import type { CustomTool, ToolContext, ToolResult } from '../types/index.js'; +import { parseToolOutput } from './parsers.js'; // ============================================================================= // INJECTED DEPENDENCIES @@ -470,7 +471,16 @@ export function adapterToCustomTool(adapter: ToolAdapter, deps: AdapterToolDeps) }; } - return { success: true, output: result.stdout }; + // Populate the structured `findings` channel from the raw stdout when a parser is wired for + // this tool (parseToolOutput → [] otherwise). The raw stdout is ALWAYS kept as `output` — it + // is the evidence of record the agent loop stamps onto each finding and the live gate checks; + // the parser summarises it, never replaces it. A parse yielding nothing leaves findings unset. + const findings = parseToolOutput(adapter.id, result.stdout); + return { + success: true, + output: result.stdout, + ...(findings.length ? { findings } : {}), + }; }; return { diff --git a/src/arsenal/catalog.ts b/src/arsenal/catalog.ts index a04178b..5c94acf 100644 --- a/src/arsenal/catalog.ts +++ b/src/arsenal/catalog.ts @@ -179,7 +179,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['jsonl', 'text'], installHint: 'brew install projectdiscovery/tap/httpx', commandHint: 'httpx -u https://example.com -status-code -title -tech-detect -json', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'HTTP probing and technology detection for discovered assets.', }, { @@ -211,7 +211,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['jsonl', 'text'], installHint: 'brew install projectdiscovery/tap/katana', commandHint: 'katana -u https://example.com -jsonl -silent', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'Crawler for endpoint discovery and attack-surface mapping.', }, { @@ -307,7 +307,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['json', 'text'], installHint: 'brew install dalfox', commandHint: 'dalfox url https://example.com?q=test -o result.json --format json', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'XSS validation adapter for scoped endpoints.', }, { diff --git a/src/arsenal/parsers.ts b/src/arsenal/parsers.ts new file mode 100644 index 0000000..115ca90 --- /dev/null +++ b/src/arsenal/parsers.ts @@ -0,0 +1,210 @@ +/** + * T3MP3ST Arsenal — tool-output → structured evidence parsers. + * + * Turns the raw stdout of the JSON-emitting scanners into `ToolFinding[]` so the generic + * adapter factory (src/arsenal/adapter-tools.ts) can return STRUCTURED findings, not just + * text. Downstream the pipe already exists and is honest-by-construction: the agent loop + * stamps `provenance:'tool'` + the raw output onto each finding, the operator materialises + * it into a `Finding` with output-evidence, and the live gate (src/evidence/gate.ts) can + * then mark it verified. Until now these adapters (parserStatus:'planned') returned raw + * stdout only — the structured channel (`ToolResult.findings`) was never populated. + * + * HONESTY CONTRACT (same discipline as stub-honesty / no-phantom-tools): + * - Pure, dependency-free, and NEVER throws. Empty / garbled output → `[]`, never a + * fabricated finding. The caller keeps the raw stdout as the evidence of record; a + * parser SUMMARISES real output, it does not invent it. + * - Parsers do NOT set provenance/toolName/toolOutput — the agent loop stamps those from + * the real subprocess result, so a parser can never forge provenance. + */ + +import type { Severity, ToolFinding } from '../types/index.js'; + +// ── small, defensive helpers ──────────────────────────────────────────────── +const SEVERITIES = new Set(['critical', 'high', 'medium', 'low', 'info']); +const sev = (v: unknown, fallback: Severity = 'info'): Severity => { + const s = String(v ?? '').trim().toLowerCase(); + return SEVERITIES.has(s as Severity) ? (s as Severity) : fallback; +}; +const asObj = (v: unknown): Record => + v && typeof v === 'object' && !Array.isArray(v) ? (v as Record) : {}; +const asStrArray = (v: unknown): string[] => + Array.isArray(v) + ? v.map((x) => String(x)).filter((s) => s.trim().length > 0) + : v !== null && v !== undefined && v !== '' ? [String(v)] : []; +const num = (v: unknown): number | undefined => { + const n = typeof v === 'number' ? v : Number(v); + return Number.isFinite(n) ? n : undefined; +}; +const truncate = (s: string, n = 400): string => (s.length > n ? `${s.slice(0, n)}…` : s); + +/** Parse JSONL defensively: one object per non-empty line; any non-JSON line is skipped. */ +function jsonl(raw: string): Record[] { + const out: Record[] = []; + for (const line of String(raw).split('\n')) { + const t = line.trim(); + if (!t) continue; + try { + const parsed: unknown = JSON.parse(t); + if (parsed && typeof parsed === 'object') out.push(parsed as Record); + } catch { + /* not a JSON line (banner, log, truncated tail) — skip */ + } + } + return out; +} + +/** Parse a single JSON document defensively; returns undefined on any error. */ +function jsonDoc(raw: string): unknown { + try { + return JSON.parse(String(raw)); + } catch { + return undefined; + } +} + +// ── nuclei -jsonl : one finding per template match ─────────────────────────── +function parseNuclei(raw: string): ToolFinding[] { + const out: ToolFinding[] = []; + for (const e of jsonl(raw)) { + const info = asObj(e.info); + const cls = asObj(info.classification); + const templateId = String(e['template-id'] ?? e.templateID ?? ''); + const title = String(info.name || templateId || 'nuclei match'); + const bits = [ + e.host && `host: ${String(e.host)}`, + e['matched-at'] && `matched-at: ${String(e['matched-at'])}`, + info.description && `desc: ${truncate(String(info.description))}`, + ].filter(Boolean) as string[]; + const f: ToolFinding = { title, severity: sev(info.severity), details: bits.join(' | ') || title }; + const cvss = num(cls['cvss-score']); + if (cvss !== undefined) f.cvss = cvss; + const cve = asStrArray(cls['cve-id']); + if (cve.length) f.cve = cve; + const cwe = asStrArray(cls['cwe-id']); + if (cwe.length) f.cwe = cwe; + if (info.remediation) f.remediation = String(info.remediation); + out.push(f); + } + return out; +} + +// ── httpx -json : one info finding per live host (technology fingerprint) ───── +function parseHttpx(raw: string): ToolFinding[] { + const out: ToolFinding[] = []; + for (const e of jsonl(raw)) { + const url = String(e.url ?? e.input ?? e.host ?? ''); + if (!url) continue; + const status = e.status_code ?? e['status-code']; + const title = String(e.title ?? ''); + const tech = asStrArray(e.tech ?? e.technologies); + const server = String(e.webserver ?? e.server ?? ''); + const bits = [ + `url: ${url}`, + status !== null && status !== undefined && `status: ${String(status)}`, + title && `title: ${truncate(title, 120)}`, + server && `server: ${server}`, + tech.length && `tech: ${tech.join(', ')}`, + ].filter(Boolean) as string[]; + out.push({ title: `HTTP service ${url}`, severity: 'info', details: bits.join(' | ') }); + } + return out; +} + +// ── dalfox --format json : one finding per XSS probe ───────────────────────── +function parseDalfox(raw: string): ToolFinding[] { + const doc = jsonDoc(raw); + const arr: unknown[] = Array.isArray(doc) + ? doc + : Array.isArray(asObj(doc).pocs) ? (asObj(doc).pocs as unknown[]) : []; + const out: ToolFinding[] = []; + for (const item of arr) { + const e = asObj(item); + const param = String(e.param ?? ''); + const kind = String(e.type ?? e.inject_type ?? 'XSS'); + const bits = [ + e.method && `method: ${String(e.method)}`, + param && `param: ${param}`, + e.data && `data: ${truncate(String(e.data), 200)}`, + e.evidence && `evidence: ${truncate(String(e.evidence), 200)}`, + e.poc && `poc: ${truncate(String(e.poc), 300)}`, + ].filter(Boolean) as string[]; + const f: ToolFinding = { + title: `dalfox ${kind}${param ? ` @ ${param}` : ''}`, + severity: sev(e.severity, 'medium'), + details: bits.join(' | ') || String(e.message_str ?? e.message ?? 'XSS probe'), + }; + const cwe = asStrArray(e.cwe).map((c) => (/^CWE-/i.test(c) ? c.toUpperCase() : `CWE-${c}`)); + if (cwe.length) f.cwe = cwe; + out.push(f); + } + return out; +} + +// ── ffuf -of json : one aggregate info finding (N paths) ───────────────────── +function parseFfuf(raw: string): ToolFinding[] { + const results = ((): unknown[] => { + const r = asObj(jsonDoc(raw)).results; + return Array.isArray(r) ? r : []; + })(); + if (!results.length) return []; + const lines = results.slice(0, 50).map((r) => { + const e = asObj(r); + const path = String(asObj(e.input).FUZZ ?? e.input ?? e.url ?? ''); + return `${path} (status ${String(e.status ?? '?')}, len ${String(e.length ?? '?')})`; + }); + const more = results.length > 50 ? `\n… +${results.length - 50} more` : ''; + return [{ + title: `ffuf: ${results.length} path(s) discovered`, + severity: 'info', + details: lines.join('\n') + more, + }]; +} + +// ── katana -jsonl : one aggregate info finding (N endpoints) ────────────────── +function parseKatana(raw: string): ToolFinding[] { + const endpoints: string[] = []; + for (const e of jsonl(raw)) { + const ep = String(asObj(e.request).endpoint ?? e.endpoint ?? e.url ?? ''); + if (ep) endpoints.push(ep); + } + if (!endpoints.length) return []; + const more = endpoints.length > 50 ? `\n… +${endpoints.length - 50} more` : ''; + return [{ + title: `katana: ${endpoints.length} endpoint(s) discovered`, + severity: 'info', + details: endpoints.slice(0, 50).join('\n') + more, + }]; +} + +const PARSERS: Record ToolFinding[]> = { + nuclei: parseNuclei, + httpx: parseHttpx, + dalfox: parseDalfox, + ffuf: parseFfuf, + katana: parseKatana, +}; + +/** Adapter ids that have a structured output parser wired here. */ +export const PARSED_TOOL_IDS: readonly string[] = Object.keys(PARSERS); + +/** True iff a structured parser is wired for this adapter id. */ +export function hasParser(toolId: string): boolean { + return Object.prototype.hasOwnProperty.call(PARSERS, toolId); +} + +/** + * Parse a tool's raw stdout into structured `ToolFinding[]`. + * + * Returns `[]` for any tool with no parser, for empty output, and for output a parser cannot + * make sense of. NEVER throws — an unexpected parser failure degrades to `[]` so a malformed + * scan can never crash the agent loop or fabricate a finding. + */ +export function parseToolOutput(toolId: string, rawOutput: string): ToolFinding[] { + const parser = PARSERS[toolId]; + if (!parser || !rawOutput || !String(rawOutput).trim()) return []; + try { + return parser(rawOutput); + } catch { + return []; + } +} diff --git a/src/operators/index.ts b/src/operators/index.ts index c7743d4..305f5b4 100755 --- a/src/operators/index.ts +++ b/src/operators/index.ts @@ -476,6 +476,7 @@ export class OperatorAgent extends EventEmitter { phase: task.phase as KillChainPhase, cvss: finding.cvss, cve: finding.cve, + cwe: finding.cwe, evidence: toolBacked ? [{ type: 'output', diff --git a/src/types/index.ts b/src/types/index.ts index 7c92650..9417e4a 100755 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -426,6 +426,7 @@ export interface ToolFinding { details: string; cvss?: number; cve?: string[]; + cwe?: string[]; remediation?: string; /** * How this finding was produced — the provenance flag the honesty gate keys on: