feat(arsenal): add source and supply-chain parsers (#97)

Follow-up to the JSON-scanner parser layer: wire semgrep, gitleaks, trivy and
grype into parseToolOutput so their findings become structured, gate-verifiable
evidence instead of raw text (they were parserStatus:'planned').

- semgrep  (scan --json)          → one finding per rule match; ERROR/WARNING/INFO
                                     mapped to severity; CWE token pulled from
                                     metadata (e.g. "CWE-78: …" → CWE-78); refs → remediation.
- trivy    (fs --format json)     → one finding per package vuln; CVE + CweIDs +
                                     "upgrade <pkg> to <fixed>" remediation.
- grype    (dir:. -o json)        → one finding per match; CVE + package coordinates.
- gitleaks (detect --report json --redact) → one high finding per secret. Defence in
                                     depth: the details are run through redactString()
                                     so a raw secret can NEVER reach a finding / the vault,
                                     even though the adapter already passes --redact.

Broadens the severity map to cover scanner vocabularies (ERROR/WARNING, CRITICAL…
NEGLIGIBLE, moderate) onto the five canonical levels. Flips parserStatus to
'structured' for the four tools. Extends parsers.test.ts + fixtures: field mapping,
the gitleaks redaction guarantee, and the end-to-end gate pass for every new scanner.

Stacked on the parser layer (#42); pairs with the scanner-invocation fix (#40) that
gives these tools the JSON-emitting arg templates. Until #40 lands the parsers simply
degrade to [] (no JSON to parse) — no misbehaviour. Secret scanners beyond gitleaks
and SARIF-style tools remain a follow-up.

Co-authored-by: Pazificateur69 <agagliardi@guardiaschool.fr>
This commit is contained in:
Joseph MaglyandPazificateur69 authored and GitHub committed 2026-07-15 19:45:58 -04:00
1 parent f8c283377c
commit cf0090adf4
7 files changed
+171 -6

No files matched your search

+1
View File
@@ -0,0 +1 @@
[{"RuleID":"aws-access-token","Description":"AWS Access Key AKIAIOSFODNN7EXAMPLE committed to source","File":"config/prod.env","StartLine":12,"Match":"REDACTED","Secret":"REDACTED","Commit":"abc1234","Author":"dev"}]
+1
View File
@@ -0,0 +1 @@
{"matches":[{"vulnerability":{"id":"CVE-2022-23529","severity":"High","description":"jsonwebtoken is vulnerable to signature verification bypass."},"artifact":{"name":"jsonwebtoken","version":"8.5.1","type":"npm"}}]}
+1
View File
@@ -0,0 +1 @@
{"results":[{"check_id":"python.lang.security.dangerous-subprocess-use","path":"app/run.py","start":{"line":42},"end":{"line":42},"extra":{"message":"Detected subprocess call with shell=True — command injection risk","severity":"ERROR","metadata":{"cwe":["CWE-78: OS Command Injection"],"owasp":["A03:2021 - Injection"],"references":["https://owasp.org/Top10/A03_2021-Injection/"]}}}],"errors":[]}
+1
View File
@@ -0,0 +1 @@
{"SchemaVersion":2,"Results":[{"Target":"package-lock.json","Class":"lang-pkgs","Type":"npm","Vulnerabilities":[{"VulnerabilityID":"CVE-2021-3749","PkgName":"axios","InstalledVersion":"0.21.1","FixedVersion":"0.21.2","Severity":"HIGH","Title":"axios: Regular expression denial of service via trim","Description":"axios is vulnerable to Inefficient Regular Expression Complexity (ReDoS).","CweIDs":["CWE-400"]}]}]}
+43 -1
View File
@@ -103,7 +103,47 @@ describe('parseToolOutput — honesty contract (never fabricate, never throw)',
}); });
it('exposes exactly the wired parser ids', () => { it('exposes exactly the wired parser ids', () => {
expect([...PARSED_TOOL_IDS].sort()).toEqual(['dalfox', 'ffuf', 'httpx', 'katana', 'nuclei']); expect([...PARSED_TOOL_IDS].sort()).toEqual(
['dalfox', 'ffuf', 'gitleaks', 'grype', 'httpx', 'katana', 'nuclei', 'semgrep', 'trivy'],
);
});
});
describe('parseToolOutput — source / supply-chain scanners', () => {
it('semgrep: one finding per rule match with mapped severity + CWE token', () => {
const f = parseToolOutput('semgrep', fixture('semgrep.json'));
expect(f).toHaveLength(1);
expect(f[0].severity).toBe('high'); // ERROR → high
expect(f[0].cwe).toEqual(['CWE-78']); // extracted from "CWE-78: OS Command Injection"
expect(f[0].details).toContain('app/run.py:42');
});
it('trivy: one finding per package vuln with CVE, CWE, and an upgrade remediation', () => {
const f = parseToolOutput('trivy', fixture('trivy.json'));
expect(f).toHaveLength(1);
expect(f[0].severity).toBe('high');
expect(f[0].cve).toEqual(['CVE-2021-3749']);
expect(f[0].cwe).toEqual(['CWE-400']);
expect(f[0].remediation).toContain('0.21.2');
expect(f[0].details).toContain('axios');
});
it('grype: one finding per match with CVE + package coordinates', () => {
const f = parseToolOutput('grype', fixture('grype.json'));
expect(f).toHaveLength(1);
expect(f[0].severity).toBe('high'); // "High" → high
expect(f[0].cve).toEqual(['CVE-2022-23529']);
expect(f[0].details).toContain('jsonwebtoken');
});
it('gitleaks: one high finding per secret, and the raw secret is REDACTED from details', () => {
const f = parseToolOutput('gitleaks', fixture('gitleaks.json'));
expect(f).toHaveLength(1);
expect(f[0].severity).toBe('high');
expect(f[0].details).toContain('config/prod.env:12');
// Defense in depth: the AWS key present in the raw report must never survive into the finding.
expect(f[0].details).not.toContain('AKIAIOSFODNN7EXAMPLE');
expect(f[0].details).toContain('[redacted]');
}); });
}); });
@@ -144,6 +184,8 @@ describe('end-to-end: a parsed finding passes the live provenance gate', () => {
for (const [id, file] of [ for (const [id, file] of [
['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'], ['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'],
['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'], ['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'],
['semgrep', 'semgrep.json'], ['gitleaks', 'gitleaks.json'],
['trivy', 'trivy.json'], ['grype', 'grype.json'],
] as const) { ] as const) {
const raw = fixture(file); const raw = fixture(file);
for (const tf of parseToolOutput(id, raw)) { for (const tf of parseToolOutput(id, raw)) {
+4 -4
View File
@@ -339,7 +339,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
outputFormats: ['json', 'sarif'], outputFormats: ['json', 'sarif'],
installHint: 'brew install semgrep or pipx install semgrep', installHint: 'brew install semgrep or pipx install semgrep',
commandHint: 'semgrep scan --config auto --json .', commandHint: 'semgrep scan --config auto --json .',
parserStatus: 'planned', parserStatus: 'structured',
notes: 'Static analysis backbone for repository hardening.', notes: 'Static analysis backbone for repository hardening.',
}, },
{ {
@@ -355,7 +355,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
outputFormats: ['json', 'sarif'], outputFormats: ['json', 'sarif'],
installHint: 'brew install gitleaks', installHint: 'brew install gitleaks',
commandHint: 'gitleaks detect --source . --report-format json --redact', commandHint: 'gitleaks detect --source . --report-format json --redact',
parserStatus: 'planned', parserStatus: 'structured',
notes: 'Secret detection must redact values before ledger insertion.', notes: 'Secret detection must redact values before ledger insertion.',
}, },
{ {
@@ -387,7 +387,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
outputFormats: ['json', 'sarif'], outputFormats: ['json', 'sarif'],
installHint: 'brew install aquasecurity/trivy/trivy', installHint: 'brew install aquasecurity/trivy/trivy',
commandHint: 'trivy fs --format json .', commandHint: 'trivy fs --format json .',
parserStatus: 'planned', parserStatus: 'structured',
notes: 'Vulnerability and IaC scanner for repos and images.', notes: 'Vulnerability and IaC scanner for repos and images.',
}, },
{ {
@@ -419,7 +419,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
outputFormats: ['json'], outputFormats: ['json'],
installHint: 'brew install grype', installHint: 'brew install grype',
commandHint: 'grype dir:. -o json', commandHint: 'grype dir:. -o json',
parserStatus: 'planned', parserStatus: 'structured',
notes: 'Pairs well with Syft SBOMs for dependency vulnerability triage.', notes: 'Pairs well with Syft SBOMs for dependency vulnerability triage.',
}, },
{ {
+120 -1
View File
@@ -18,12 +18,20 @@
*/ */
import type { Severity, ToolFinding } from '../types/index.js'; import type { Severity, ToolFinding } from '../types/index.js';
import { redactString } from '../redact.js';
// ── small, defensive helpers ──────────────────────────────────────────────── // ── small, defensive helpers ────────────────────────────────────────────────
const SEVERITIES = new Set<Severity>(['critical', 'high', 'medium', 'low', 'info']); const SEVERITIES = new Set<Severity>(['critical', 'high', 'medium', 'low', 'info']);
// Map the many scanner severity vocabularies (semgrep ERROR/WARNING/INFO, trivy/grype
// CRITICAL…NEGLIGIBLE, npm-audit "moderate", …) onto the five canonical levels.
const SEVERITY_ALIAS: Record<string, Severity> = {
error: 'high', warning: 'medium', warn: 'medium', inventory: 'info', experiment: 'info',
moderate: 'medium', negligible: 'info', unknown: 'info', none: 'info', informational: 'info',
};
const sev = (v: unknown, fallback: Severity = 'info'): Severity => { const sev = (v: unknown, fallback: Severity = 'info'): Severity => {
const s = String(v ?? '').trim().toLowerCase(); const s = String(v ?? '').trim().toLowerCase();
return SEVERITIES.has(s as Severity) ? (s as Severity) : fallback; if (SEVERITIES.has(s as Severity)) return s as Severity;
return SEVERITY_ALIAS[s] ?? fallback;
}; };
const asObj = (v: unknown): Record<string, unknown> => const asObj = (v: unknown): Record<string, unknown> =>
v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {}; v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
@@ -176,12 +184,123 @@ function parseKatana(raw: string): ToolFinding[] {
}]; }];
} }
// ── semgrep scan --json : one finding per rule match (SAST) ──────────────────
function parseSemgrep(raw: string): ToolFinding[] {
const results = ((): unknown[] => {
const r = asObj(jsonDoc(raw)).results;
return Array.isArray(r) ? r : [];
})();
const out: ToolFinding[] = [];
for (const item of results) {
const e = asObj(item);
const extra = asObj(e.extra);
const meta = asObj(extra.metadata);
const checkId = String(e.check_id ?? e.checkId ?? 'semgrep rule');
const path = String(e.path ?? '');
const line = num(asObj(e.start).line);
const where = path ? `${path}${line !== undefined ? `:${line}` : ''}` : '';
const f: ToolFinding = {
title: checkId,
severity: sev(extra.severity),
details: [where && `at ${where}`, extra.message && truncate(String(extra.message))].filter(Boolean).join(' | ') || checkId,
};
// semgrep CWE metadata is like ["CWE-89: SQL Injection"] — keep the CWE-NNN token.
const cwe = asStrArray(meta.cwe).map((c) => c.match(/CWE-\d+/i)?.[0] ?? '').filter(Boolean);
if (cwe.length) f.cwe = cwe.map((c) => c.toUpperCase());
const refs = asStrArray(meta.references);
if (refs.length) f.remediation = `refs: ${refs.slice(0, 3).join(', ')}`;
out.push(f);
}
return out;
}
// ── gitleaks detect --report-format json --redact : one finding per secret ────
function parseGitleaks(raw: string): ToolFinding[] {
const arr: unknown[] = Array.isArray(jsonDoc(raw)) ? (jsonDoc(raw) as unknown[]) : [];
const out: ToolFinding[] = [];
for (const item of arr) {
const e = asObj(item);
const rule = String(e.RuleID ?? e.Description ?? 'secret');
const file = String(e.File ?? '');
const line = num(e.StartLine);
const where = file ? `${file}${line !== undefined ? `:${line}` : ''}` : '';
// Defense in depth: the adapter already runs gitleaks with --redact, but scrub the details
// through redactString anyway so a raw secret can NEVER reach a finding or the evidence vault.
const details = redactString([where && `at ${where}`, String(e.Description ?? '')].filter(Boolean).join(' | '));
out.push({ title: `secret: ${rule}`, severity: 'high', details: details || `secret: ${rule}` });
}
return out;
}
// ── trivy fs --format json : one finding per package vulnerability ────────────
function parseTrivy(raw: string): ToolFinding[] {
const results = ((): unknown[] => {
const r = asObj(jsonDoc(raw)).Results;
return Array.isArray(r) ? r : [];
})();
const out: ToolFinding[] = [];
for (const res of results) {
const target = String(asObj(res).Target ?? '');
const vulns = asObj(res).Vulnerabilities;
if (!Array.isArray(vulns)) continue;
for (const item of vulns) {
const v = asObj(item);
const id = String(v.VulnerabilityID ?? '');
const pkg = String(v.PkgName ?? '');
const installed = String(v.InstalledVersion ?? '');
const fixed = String(v.FixedVersion ?? '');
const f: ToolFinding = {
title: String(v.Title || id || 'vulnerability'),
severity: sev(v.Severity),
details: [target && `target: ${target}`, pkg && `pkg: ${pkg}${installed ? ` ${installed}` : ''}`,
v.Description && truncate(String(v.Description), 200)].filter(Boolean).join(' | ') || id,
};
if (/^CVE-/i.test(id)) f.cve = [id];
const cwe = asStrArray(v.CweIDs);
if (cwe.length) f.cwe = cwe;
if (fixed) f.remediation = `upgrade ${pkg || 'package'} to ${fixed}`;
out.push(f);
}
}
return out;
}
// ── grype dir:. -o json : one finding per matched vulnerability ───────────────
function parseGrype(raw: string): ToolFinding[] {
const matches = ((): unknown[] => {
const m = asObj(jsonDoc(raw)).matches;
return Array.isArray(m) ? m : [];
})();
const out: ToolFinding[] = [];
for (const item of matches) {
const m = asObj(item);
const vuln = asObj(m.vulnerability);
const art = asObj(m.artifact);
const id = String(vuln.id ?? '');
const name = String(art.name ?? '');
const version = String(art.version ?? '');
const f: ToolFinding = {
title: id || 'vulnerability',
severity: sev(vuln.severity),
details: [name && `pkg: ${name}${version ? ` ${version}` : ''}`,
vuln.description && truncate(String(vuln.description), 200)].filter(Boolean).join(' | ') || id,
};
if (/^CVE-/i.test(id)) f.cve = [id];
out.push(f);
}
return out;
}
const PARSERS: Record<string, (raw: string) => ToolFinding[]> = { const PARSERS: Record<string, (raw: string) => ToolFinding[]> = {
nuclei: parseNuclei, nuclei: parseNuclei,
httpx: parseHttpx, httpx: parseHttpx,
dalfox: parseDalfox, dalfox: parseDalfox,
ffuf: parseFfuf, ffuf: parseFfuf,
katana: parseKatana, katana: parseKatana,
semgrep: parseSemgrep,
gitleaks: parseGitleaks,
trivy: parseTrivy,
grype: parseGrype,
}; };
/** Adapter ids that have a structured output parser wired here. */ /** Adapter ids that have a structured output parser wired here. */