From cf0090adf4e03ef34f960cde545790ed3cbff27c Mon Sep 17 00:00:00 2001 From: Joseph Magly <1159087+jmagly@users.noreply.github.com> Date: Wed, 15 Jul 2026 19:45:58 -0400 Subject: [PATCH] feat(arsenal): add source and supply-chain parsers (#97) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the JSON-scanner parser layer: wire semgrep, gitleaks, trivy and grype into parseToolOutput so their findings become structured, gate-verifiable evidence instead of raw text (they were parserStatus:'planned'). - semgrep (scan --json) → one finding per rule match; ERROR/WARNING/INFO mapped to severity; CWE token pulled from metadata (e.g. "CWE-78: …" → CWE-78); refs → remediation. - trivy (fs --format json) → one finding per package vuln; CVE + CweIDs + "upgrade to " remediation. - grype (dir:. -o json) → one finding per match; CVE + package coordinates. - gitleaks (detect --report json --redact) → one high finding per secret. Defence in depth: the details are run through redactString() so a raw secret can NEVER reach a finding / the vault, even though the adapter already passes --redact. Broadens the severity map to cover scanner vocabularies (ERROR/WARNING, CRITICAL… NEGLIGIBLE, moderate) onto the five canonical levels. Flips parserStatus to 'structured' for the four tools. Extends parsers.test.ts + fixtures: field mapping, the gitleaks redaction guarantee, and the end-to-end gate pass for every new scanner. Stacked on the parser layer (#42); pairs with the scanner-invocation fix (#40) that gives these tools the JSON-emitting arg templates. Until #40 lands the parsers simply degrade to [] (no JSON to parse) — no misbehaviour. Secret scanners beyond gitleaks and SARIF-style tools remain a follow-up. Co-authored-by: Pazificateur69 --- src/__tests__/fixtures/gitleaks.json | 1 + src/__tests__/fixtures/grype.json | 1 + src/__tests__/fixtures/semgrep.json | 1 + src/__tests__/fixtures/trivy.json | 1 + src/__tests__/parsers.test.ts | 44 +++++++++- src/arsenal/catalog.ts | 8 +- src/arsenal/parsers.ts | 121 ++++++++++++++++++++++++++- 7 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 src/__tests__/fixtures/gitleaks.json create mode 100644 src/__tests__/fixtures/grype.json create mode 100644 src/__tests__/fixtures/semgrep.json create mode 100644 src/__tests__/fixtures/trivy.json diff --git a/src/__tests__/fixtures/gitleaks.json b/src/__tests__/fixtures/gitleaks.json new file mode 100644 index 0000000..971cebc --- /dev/null +++ b/src/__tests__/fixtures/gitleaks.json @@ -0,0 +1 @@ +[{"RuleID":"aws-access-token","Description":"AWS Access Key AKIAIOSFODNN7EXAMPLE committed to source","File":"config/prod.env","StartLine":12,"Match":"REDACTED","Secret":"REDACTED","Commit":"abc1234","Author":"dev"}] diff --git a/src/__tests__/fixtures/grype.json b/src/__tests__/fixtures/grype.json new file mode 100644 index 0000000..a0cbeb9 --- /dev/null +++ b/src/__tests__/fixtures/grype.json @@ -0,0 +1 @@ +{"matches":[{"vulnerability":{"id":"CVE-2022-23529","severity":"High","description":"jsonwebtoken is vulnerable to signature verification bypass."},"artifact":{"name":"jsonwebtoken","version":"8.5.1","type":"npm"}}]} diff --git a/src/__tests__/fixtures/semgrep.json b/src/__tests__/fixtures/semgrep.json new file mode 100644 index 0000000..db724dd --- /dev/null +++ b/src/__tests__/fixtures/semgrep.json @@ -0,0 +1 @@ +{"results":[{"check_id":"python.lang.security.dangerous-subprocess-use","path":"app/run.py","start":{"line":42},"end":{"line":42},"extra":{"message":"Detected subprocess call with shell=True — command injection risk","severity":"ERROR","metadata":{"cwe":["CWE-78: OS Command Injection"],"owasp":["A03:2021 - Injection"],"references":["https://owasp.org/Top10/A03_2021-Injection/"]}}}],"errors":[]} diff --git a/src/__tests__/fixtures/trivy.json b/src/__tests__/fixtures/trivy.json new file mode 100644 index 0000000..8c0a1b9 --- /dev/null +++ b/src/__tests__/fixtures/trivy.json @@ -0,0 +1 @@ +{"SchemaVersion":2,"Results":[{"Target":"package-lock.json","Class":"lang-pkgs","Type":"npm","Vulnerabilities":[{"VulnerabilityID":"CVE-2021-3749","PkgName":"axios","InstalledVersion":"0.21.1","FixedVersion":"0.21.2","Severity":"HIGH","Title":"axios: Regular expression denial of service via trim","Description":"axios is vulnerable to Inefficient Regular Expression Complexity (ReDoS).","CweIDs":["CWE-400"]}]}]} diff --git a/src/__tests__/parsers.test.ts b/src/__tests__/parsers.test.ts index 3915b88..206bc42 100644 --- a/src/__tests__/parsers.test.ts +++ b/src/__tests__/parsers.test.ts @@ -103,7 +103,47 @@ describe('parseToolOutput — honesty contract (never fabricate, never throw)', }); it('exposes exactly the wired parser ids', () => { - expect([...PARSED_TOOL_IDS].sort()).toEqual(['dalfox', 'ffuf', 'httpx', 'katana', 'nuclei']); + expect([...PARSED_TOOL_IDS].sort()).toEqual( + ['dalfox', 'ffuf', 'gitleaks', 'grype', 'httpx', 'katana', 'nuclei', 'semgrep', 'trivy'], + ); + }); +}); + +describe('parseToolOutput — source / supply-chain scanners', () => { + it('semgrep: one finding per rule match with mapped severity + CWE token', () => { + const f = parseToolOutput('semgrep', fixture('semgrep.json')); + expect(f).toHaveLength(1); + expect(f[0].severity).toBe('high'); // ERROR → high + expect(f[0].cwe).toEqual(['CWE-78']); // extracted from "CWE-78: OS Command Injection" + expect(f[0].details).toContain('app/run.py:42'); + }); + + it('trivy: one finding per package vuln with CVE, CWE, and an upgrade remediation', () => { + const f = parseToolOutput('trivy', fixture('trivy.json')); + expect(f).toHaveLength(1); + expect(f[0].severity).toBe('high'); + expect(f[0].cve).toEqual(['CVE-2021-3749']); + expect(f[0].cwe).toEqual(['CWE-400']); + expect(f[0].remediation).toContain('0.21.2'); + expect(f[0].details).toContain('axios'); + }); + + it('grype: one finding per match with CVE + package coordinates', () => { + const f = parseToolOutput('grype', fixture('grype.json')); + expect(f).toHaveLength(1); + expect(f[0].severity).toBe('high'); // "High" → high + expect(f[0].cve).toEqual(['CVE-2022-23529']); + expect(f[0].details).toContain('jsonwebtoken'); + }); + + it('gitleaks: one high finding per secret, and the raw secret is REDACTED from details', () => { + const f = parseToolOutput('gitleaks', fixture('gitleaks.json')); + expect(f).toHaveLength(1); + expect(f[0].severity).toBe('high'); + expect(f[0].details).toContain('config/prod.env:12'); + // Defense in depth: the AWS key present in the raw report must never survive into the finding. + expect(f[0].details).not.toContain('AKIAIOSFODNN7EXAMPLE'); + expect(f[0].details).toContain('[redacted]'); }); }); @@ -144,6 +184,8 @@ describe('end-to-end: a parsed finding passes the live provenance gate', () => { for (const [id, file] of [ ['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'], ['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'], + ['semgrep', 'semgrep.json'], ['gitleaks', 'gitleaks.json'], + ['trivy', 'trivy.json'], ['grype', 'grype.json'], ] as const) { const raw = fixture(file); for (const tf of parseToolOutput(id, raw)) { diff --git a/src/arsenal/catalog.ts b/src/arsenal/catalog.ts index 5c94acf..663ad79 100644 --- a/src/arsenal/catalog.ts +++ b/src/arsenal/catalog.ts @@ -339,7 +339,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['json', 'sarif'], installHint: 'brew install semgrep or pipx install semgrep', commandHint: 'semgrep scan --config auto --json .', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'Static analysis backbone for repository hardening.', }, { @@ -355,7 +355,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['json', 'sarif'], installHint: 'brew install gitleaks', commandHint: 'gitleaks detect --source . --report-format json --redact', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'Secret detection must redact values before ledger insertion.', }, { @@ -387,7 +387,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['json', 'sarif'], installHint: 'brew install aquasecurity/trivy/trivy', commandHint: 'trivy fs --format json .', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'Vulnerability and IaC scanner for repos and images.', }, { @@ -419,7 +419,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [ outputFormats: ['json'], installHint: 'brew install grype', commandHint: 'grype dir:. -o json', - parserStatus: 'planned', + parserStatus: 'structured', notes: 'Pairs well with Syft SBOMs for dependency vulnerability triage.', }, { diff --git a/src/arsenal/parsers.ts b/src/arsenal/parsers.ts index 115ca90..6b58819 100644 --- a/src/arsenal/parsers.ts +++ b/src/arsenal/parsers.ts @@ -18,12 +18,20 @@ */ import type { Severity, ToolFinding } from '../types/index.js'; +import { redactString } from '../redact.js'; // ── small, defensive helpers ──────────────────────────────────────────────── const SEVERITIES = new Set(['critical', 'high', 'medium', 'low', 'info']); +// Map the many scanner severity vocabularies (semgrep ERROR/WARNING/INFO, trivy/grype +// CRITICAL…NEGLIGIBLE, npm-audit "moderate", …) onto the five canonical levels. +const SEVERITY_ALIAS: Record = { + error: 'high', warning: 'medium', warn: 'medium', inventory: 'info', experiment: 'info', + moderate: 'medium', negligible: 'info', unknown: 'info', none: 'info', informational: 'info', +}; const sev = (v: unknown, fallback: Severity = 'info'): Severity => { const s = String(v ?? '').trim().toLowerCase(); - return SEVERITIES.has(s as Severity) ? (s as Severity) : fallback; + if (SEVERITIES.has(s as Severity)) return s as Severity; + return SEVERITY_ALIAS[s] ?? fallback; }; const asObj = (v: unknown): Record => v && typeof v === 'object' && !Array.isArray(v) ? (v as Record) : {}; @@ -176,12 +184,123 @@ function parseKatana(raw: string): ToolFinding[] { }]; } +// ── semgrep scan --json : one finding per rule match (SAST) ────────────────── +function parseSemgrep(raw: string): ToolFinding[] { + const results = ((): unknown[] => { + const r = asObj(jsonDoc(raw)).results; + return Array.isArray(r) ? r : []; + })(); + const out: ToolFinding[] = []; + for (const item of results) { + const e = asObj(item); + const extra = asObj(e.extra); + const meta = asObj(extra.metadata); + const checkId = String(e.check_id ?? e.checkId ?? 'semgrep rule'); + const path = String(e.path ?? ''); + const line = num(asObj(e.start).line); + const where = path ? `${path}${line !== undefined ? `:${line}` : ''}` : ''; + const f: ToolFinding = { + title: checkId, + severity: sev(extra.severity), + details: [where && `at ${where}`, extra.message && truncate(String(extra.message))].filter(Boolean).join(' | ') || checkId, + }; + // semgrep CWE metadata is like ["CWE-89: SQL Injection"] — keep the CWE-NNN token. + const cwe = asStrArray(meta.cwe).map((c) => c.match(/CWE-\d+/i)?.[0] ?? '').filter(Boolean); + if (cwe.length) f.cwe = cwe.map((c) => c.toUpperCase()); + const refs = asStrArray(meta.references); + if (refs.length) f.remediation = `refs: ${refs.slice(0, 3).join(', ')}`; + out.push(f); + } + return out; +} + +// ── gitleaks detect --report-format json --redact : one finding per secret ──── +function parseGitleaks(raw: string): ToolFinding[] { + const arr: unknown[] = Array.isArray(jsonDoc(raw)) ? (jsonDoc(raw) as unknown[]) : []; + const out: ToolFinding[] = []; + for (const item of arr) { + const e = asObj(item); + const rule = String(e.RuleID ?? e.Description ?? 'secret'); + const file = String(e.File ?? ''); + const line = num(e.StartLine); + const where = file ? `${file}${line !== undefined ? `:${line}` : ''}` : ''; + // Defense in depth: the adapter already runs gitleaks with --redact, but scrub the details + // through redactString anyway so a raw secret can NEVER reach a finding or the evidence vault. + const details = redactString([where && `at ${where}`, String(e.Description ?? '')].filter(Boolean).join(' | ')); + out.push({ title: `secret: ${rule}`, severity: 'high', details: details || `secret: ${rule}` }); + } + return out; +} + +// ── trivy fs --format json : one finding per package vulnerability ──────────── +function parseTrivy(raw: string): ToolFinding[] { + const results = ((): unknown[] => { + const r = asObj(jsonDoc(raw)).Results; + return Array.isArray(r) ? r : []; + })(); + const out: ToolFinding[] = []; + for (const res of results) { + const target = String(asObj(res).Target ?? ''); + const vulns = asObj(res).Vulnerabilities; + if (!Array.isArray(vulns)) continue; + for (const item of vulns) { + const v = asObj(item); + const id = String(v.VulnerabilityID ?? ''); + const pkg = String(v.PkgName ?? ''); + const installed = String(v.InstalledVersion ?? ''); + const fixed = String(v.FixedVersion ?? ''); + const f: ToolFinding = { + title: String(v.Title || id || 'vulnerability'), + severity: sev(v.Severity), + details: [target && `target: ${target}`, pkg && `pkg: ${pkg}${installed ? ` ${installed}` : ''}`, + v.Description && truncate(String(v.Description), 200)].filter(Boolean).join(' | ') || id, + }; + if (/^CVE-/i.test(id)) f.cve = [id]; + const cwe = asStrArray(v.CweIDs); + if (cwe.length) f.cwe = cwe; + if (fixed) f.remediation = `upgrade ${pkg || 'package'} to ${fixed}`; + out.push(f); + } + } + return out; +} + +// ── grype dir:. -o json : one finding per matched vulnerability ─────────────── +function parseGrype(raw: string): ToolFinding[] { + const matches = ((): unknown[] => { + const m = asObj(jsonDoc(raw)).matches; + return Array.isArray(m) ? m : []; + })(); + const out: ToolFinding[] = []; + for (const item of matches) { + const m = asObj(item); + const vuln = asObj(m.vulnerability); + const art = asObj(m.artifact); + const id = String(vuln.id ?? ''); + const name = String(art.name ?? ''); + const version = String(art.version ?? ''); + const f: ToolFinding = { + title: id || 'vulnerability', + severity: sev(vuln.severity), + details: [name && `pkg: ${name}${version ? ` ${version}` : ''}`, + vuln.description && truncate(String(vuln.description), 200)].filter(Boolean).join(' | ') || id, + }; + if (/^CVE-/i.test(id)) f.cve = [id]; + out.push(f); + } + return out; +} + const PARSERS: Record ToolFinding[]> = { nuclei: parseNuclei, httpx: parseHttpx, dalfox: parseDalfox, ffuf: parseFfuf, katana: parseKatana, + semgrep: parseSemgrep, + gitleaks: parseGitleaks, + trivy: parseTrivy, + grype: parseGrype, }; /** Adapter ids that have a structured output parser wired here. */