feat(arsenal): add source and supply-chain parsers (#97)
Follow-up to the JSON-scanner parser layer: wire semgrep, gitleaks, trivy and
grype into parseToolOutput so their findings become structured, gate-verifiable
evidence instead of raw text (they were parserStatus:'planned').
- semgrep (scan --json) → one finding per rule match; ERROR/WARNING/INFO
mapped to severity; CWE token pulled from
metadata (e.g. "CWE-78: …" → CWE-78); refs → remediation.
- trivy (fs --format json) → one finding per package vuln; CVE + CweIDs +
"upgrade <pkg> to <fixed>" remediation.
- grype (dir:. -o json) → one finding per match; CVE + package coordinates.
- gitleaks (detect --report json --redact) → one high finding per secret. Defence in
depth: the details are run through redactString()
so a raw secret can NEVER reach a finding / the vault,
even though the adapter already passes --redact.
Broadens the severity map to cover scanner vocabularies (ERROR/WARNING, CRITICAL…
NEGLIGIBLE, moderate) onto the five canonical levels. Flips parserStatus to
'structured' for the four tools. Extends parsers.test.ts + fixtures: field mapping,
the gitleaks redaction guarantee, and the end-to-end gate pass for every new scanner.
Stacked on the parser layer (#42); pairs with the scanner-invocation fix (#40) that
gives these tools the JSON-emitting arg templates. Until #40 lands the parsers simply
degrade to [] (no JSON to parse) — no misbehaviour. Secret scanners beyond gitleaks
and SARIF-style tools remain a follow-up.
Co-authored-by: Pazificateur69 <agagliardi@guardiaschool.fr>
This commit is contained in:
1 parent
f8c283377c
commit
cf0090adf4
7 files changed
+171
-6
No files matched your search
@@ -0,0 +1 @@
|
||||
[{"RuleID":"aws-access-token","Description":"AWS Access Key AKIAIOSFODNN7EXAMPLE committed to source","File":"config/prod.env","StartLine":12,"Match":"REDACTED","Secret":"REDACTED","Commit":"abc1234","Author":"dev"}]
|
||||
@@ -0,0 +1 @@
|
||||
{"matches":[{"vulnerability":{"id":"CVE-2022-23529","severity":"High","description":"jsonwebtoken is vulnerable to signature verification bypass."},"artifact":{"name":"jsonwebtoken","version":"8.5.1","type":"npm"}}]}
|
||||
@@ -0,0 +1 @@
|
||||
{"results":[{"check_id":"python.lang.security.dangerous-subprocess-use","path":"app/run.py","start":{"line":42},"end":{"line":42},"extra":{"message":"Detected subprocess call with shell=True — command injection risk","severity":"ERROR","metadata":{"cwe":["CWE-78: OS Command Injection"],"owasp":["A03:2021 - Injection"],"references":["https://owasp.org/Top10/A03_2021-Injection/"]}}}],"errors":[]}
|
||||
@@ -0,0 +1 @@
|
||||
{"SchemaVersion":2,"Results":[{"Target":"package-lock.json","Class":"lang-pkgs","Type":"npm","Vulnerabilities":[{"VulnerabilityID":"CVE-2021-3749","PkgName":"axios","InstalledVersion":"0.21.1","FixedVersion":"0.21.2","Severity":"HIGH","Title":"axios: Regular expression denial of service via trim","Description":"axios is vulnerable to Inefficient Regular Expression Complexity (ReDoS).","CweIDs":["CWE-400"]}]}]}
|
||||
@@ -103,7 +103,47 @@ describe('parseToolOutput — honesty contract (never fabricate, never throw)',
|
||||
});
|
||||
|
||||
it('exposes exactly the wired parser ids', () => {
|
||||
expect([...PARSED_TOOL_IDS].sort()).toEqual(['dalfox', 'ffuf', 'httpx', 'katana', 'nuclei']);
|
||||
expect([...PARSED_TOOL_IDS].sort()).toEqual(
|
||||
['dalfox', 'ffuf', 'gitleaks', 'grype', 'httpx', 'katana', 'nuclei', 'semgrep', 'trivy'],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseToolOutput — source / supply-chain scanners', () => {
|
||||
it('semgrep: one finding per rule match with mapped severity + CWE token', () => {
|
||||
const f = parseToolOutput('semgrep', fixture('semgrep.json'));
|
||||
expect(f).toHaveLength(1);
|
||||
expect(f[0].severity).toBe('high'); // ERROR → high
|
||||
expect(f[0].cwe).toEqual(['CWE-78']); // extracted from "CWE-78: OS Command Injection"
|
||||
expect(f[0].details).toContain('app/run.py:42');
|
||||
});
|
||||
|
||||
it('trivy: one finding per package vuln with CVE, CWE, and an upgrade remediation', () => {
|
||||
const f = parseToolOutput('trivy', fixture('trivy.json'));
|
||||
expect(f).toHaveLength(1);
|
||||
expect(f[0].severity).toBe('high');
|
||||
expect(f[0].cve).toEqual(['CVE-2021-3749']);
|
||||
expect(f[0].cwe).toEqual(['CWE-400']);
|
||||
expect(f[0].remediation).toContain('0.21.2');
|
||||
expect(f[0].details).toContain('axios');
|
||||
});
|
||||
|
||||
it('grype: one finding per match with CVE + package coordinates', () => {
|
||||
const f = parseToolOutput('grype', fixture('grype.json'));
|
||||
expect(f).toHaveLength(1);
|
||||
expect(f[0].severity).toBe('high'); // "High" → high
|
||||
expect(f[0].cve).toEqual(['CVE-2022-23529']);
|
||||
expect(f[0].details).toContain('jsonwebtoken');
|
||||
});
|
||||
|
||||
it('gitleaks: one high finding per secret, and the raw secret is REDACTED from details', () => {
|
||||
const f = parseToolOutput('gitleaks', fixture('gitleaks.json'));
|
||||
expect(f).toHaveLength(1);
|
||||
expect(f[0].severity).toBe('high');
|
||||
expect(f[0].details).toContain('config/prod.env:12');
|
||||
// Defense in depth: the AWS key present in the raw report must never survive into the finding.
|
||||
expect(f[0].details).not.toContain('AKIAIOSFODNN7EXAMPLE');
|
||||
expect(f[0].details).toContain('[redacted]');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -144,6 +184,8 @@ describe('end-to-end: a parsed finding passes the live provenance gate', () => {
|
||||
for (const [id, file] of [
|
||||
['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'],
|
||||
['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'],
|
||||
['semgrep', 'semgrep.json'], ['gitleaks', 'gitleaks.json'],
|
||||
['trivy', 'trivy.json'], ['grype', 'grype.json'],
|
||||
] as const) {
|
||||
const raw = fixture(file);
|
||||
for (const tf of parseToolOutput(id, raw)) {
|
||||
|
||||
@@ -339,7 +339,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
||||
outputFormats: ['json', 'sarif'],
|
||||
installHint: 'brew install semgrep or pipx install semgrep',
|
||||
commandHint: 'semgrep scan --config auto --json .',
|
||||
parserStatus: 'planned',
|
||||
parserStatus: 'structured',
|
||||
notes: 'Static analysis backbone for repository hardening.',
|
||||
},
|
||||
{
|
||||
@@ -355,7 +355,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
||||
outputFormats: ['json', 'sarif'],
|
||||
installHint: 'brew install gitleaks',
|
||||
commandHint: 'gitleaks detect --source . --report-format json --redact',
|
||||
parserStatus: 'planned',
|
||||
parserStatus: 'structured',
|
||||
notes: 'Secret detection must redact values before ledger insertion.',
|
||||
},
|
||||
{
|
||||
@@ -387,7 +387,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
||||
outputFormats: ['json', 'sarif'],
|
||||
installHint: 'brew install aquasecurity/trivy/trivy',
|
||||
commandHint: 'trivy fs --format json .',
|
||||
parserStatus: 'planned',
|
||||
parserStatus: 'structured',
|
||||
notes: 'Vulnerability and IaC scanner for repos and images.',
|
||||
},
|
||||
{
|
||||
@@ -419,7 +419,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
||||
outputFormats: ['json'],
|
||||
installHint: 'brew install grype',
|
||||
commandHint: 'grype dir:. -o json',
|
||||
parserStatus: 'planned',
|
||||
parserStatus: 'structured',
|
||||
notes: 'Pairs well with Syft SBOMs for dependency vulnerability triage.',
|
||||
},
|
||||
{
|
||||
|
||||
+120
-1
@@ -18,12 +18,20 @@
|
||||
*/
|
||||
|
||||
import type { Severity, ToolFinding } from '../types/index.js';
|
||||
import { redactString } from '../redact.js';
|
||||
|
||||
// ── small, defensive helpers ────────────────────────────────────────────────
|
||||
const SEVERITIES = new Set<Severity>(['critical', 'high', 'medium', 'low', 'info']);
|
||||
// Map the many scanner severity vocabularies (semgrep ERROR/WARNING/INFO, trivy/grype
|
||||
// CRITICAL…NEGLIGIBLE, npm-audit "moderate", …) onto the five canonical levels.
|
||||
const SEVERITY_ALIAS: Record<string, Severity> = {
|
||||
error: 'high', warning: 'medium', warn: 'medium', inventory: 'info', experiment: 'info',
|
||||
moderate: 'medium', negligible: 'info', unknown: 'info', none: 'info', informational: 'info',
|
||||
};
|
||||
const sev = (v: unknown, fallback: Severity = 'info'): Severity => {
|
||||
const s = String(v ?? '').trim().toLowerCase();
|
||||
return SEVERITIES.has(s as Severity) ? (s as Severity) : fallback;
|
||||
if (SEVERITIES.has(s as Severity)) return s as Severity;
|
||||
return SEVERITY_ALIAS[s] ?? fallback;
|
||||
};
|
||||
const asObj = (v: unknown): Record<string, unknown> =>
|
||||
v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
||||
@@ -176,12 +184,123 @@ function parseKatana(raw: string): ToolFinding[] {
|
||||
}];
|
||||
}
|
||||
|
||||
// ── semgrep scan --json : one finding per rule match (SAST) ──────────────────
|
||||
function parseSemgrep(raw: string): ToolFinding[] {
|
||||
const results = ((): unknown[] => {
|
||||
const r = asObj(jsonDoc(raw)).results;
|
||||
return Array.isArray(r) ? r : [];
|
||||
})();
|
||||
const out: ToolFinding[] = [];
|
||||
for (const item of results) {
|
||||
const e = asObj(item);
|
||||
const extra = asObj(e.extra);
|
||||
const meta = asObj(extra.metadata);
|
||||
const checkId = String(e.check_id ?? e.checkId ?? 'semgrep rule');
|
||||
const path = String(e.path ?? '');
|
||||
const line = num(asObj(e.start).line);
|
||||
const where = path ? `${path}${line !== undefined ? `:${line}` : ''}` : '';
|
||||
const f: ToolFinding = {
|
||||
title: checkId,
|
||||
severity: sev(extra.severity),
|
||||
details: [where && `at ${where}`, extra.message && truncate(String(extra.message))].filter(Boolean).join(' | ') || checkId,
|
||||
};
|
||||
// semgrep CWE metadata is like ["CWE-89: SQL Injection"] — keep the CWE-NNN token.
|
||||
const cwe = asStrArray(meta.cwe).map((c) => c.match(/CWE-\d+/i)?.[0] ?? '').filter(Boolean);
|
||||
if (cwe.length) f.cwe = cwe.map((c) => c.toUpperCase());
|
||||
const refs = asStrArray(meta.references);
|
||||
if (refs.length) f.remediation = `refs: ${refs.slice(0, 3).join(', ')}`;
|
||||
out.push(f);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── gitleaks detect --report-format json --redact : one finding per secret ────
|
||||
function parseGitleaks(raw: string): ToolFinding[] {
|
||||
const arr: unknown[] = Array.isArray(jsonDoc(raw)) ? (jsonDoc(raw) as unknown[]) : [];
|
||||
const out: ToolFinding[] = [];
|
||||
for (const item of arr) {
|
||||
const e = asObj(item);
|
||||
const rule = String(e.RuleID ?? e.Description ?? 'secret');
|
||||
const file = String(e.File ?? '');
|
||||
const line = num(e.StartLine);
|
||||
const where = file ? `${file}${line !== undefined ? `:${line}` : ''}` : '';
|
||||
// Defense in depth: the adapter already runs gitleaks with --redact, but scrub the details
|
||||
// through redactString anyway so a raw secret can NEVER reach a finding or the evidence vault.
|
||||
const details = redactString([where && `at ${where}`, String(e.Description ?? '')].filter(Boolean).join(' | '));
|
||||
out.push({ title: `secret: ${rule}`, severity: 'high', details: details || `secret: ${rule}` });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── trivy fs --format json : one finding per package vulnerability ────────────
|
||||
function parseTrivy(raw: string): ToolFinding[] {
|
||||
const results = ((): unknown[] => {
|
||||
const r = asObj(jsonDoc(raw)).Results;
|
||||
return Array.isArray(r) ? r : [];
|
||||
})();
|
||||
const out: ToolFinding[] = [];
|
||||
for (const res of results) {
|
||||
const target = String(asObj(res).Target ?? '');
|
||||
const vulns = asObj(res).Vulnerabilities;
|
||||
if (!Array.isArray(vulns)) continue;
|
||||
for (const item of vulns) {
|
||||
const v = asObj(item);
|
||||
const id = String(v.VulnerabilityID ?? '');
|
||||
const pkg = String(v.PkgName ?? '');
|
||||
const installed = String(v.InstalledVersion ?? '');
|
||||
const fixed = String(v.FixedVersion ?? '');
|
||||
const f: ToolFinding = {
|
||||
title: String(v.Title || id || 'vulnerability'),
|
||||
severity: sev(v.Severity),
|
||||
details: [target && `target: ${target}`, pkg && `pkg: ${pkg}${installed ? ` ${installed}` : ''}`,
|
||||
v.Description && truncate(String(v.Description), 200)].filter(Boolean).join(' | ') || id,
|
||||
};
|
||||
if (/^CVE-/i.test(id)) f.cve = [id];
|
||||
const cwe = asStrArray(v.CweIDs);
|
||||
if (cwe.length) f.cwe = cwe;
|
||||
if (fixed) f.remediation = `upgrade ${pkg || 'package'} to ${fixed}`;
|
||||
out.push(f);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── grype dir:. -o json : one finding per matched vulnerability ───────────────
|
||||
function parseGrype(raw: string): ToolFinding[] {
|
||||
const matches = ((): unknown[] => {
|
||||
const m = asObj(jsonDoc(raw)).matches;
|
||||
return Array.isArray(m) ? m : [];
|
||||
})();
|
||||
const out: ToolFinding[] = [];
|
||||
for (const item of matches) {
|
||||
const m = asObj(item);
|
||||
const vuln = asObj(m.vulnerability);
|
||||
const art = asObj(m.artifact);
|
||||
const id = String(vuln.id ?? '');
|
||||
const name = String(art.name ?? '');
|
||||
const version = String(art.version ?? '');
|
||||
const f: ToolFinding = {
|
||||
title: id || 'vulnerability',
|
||||
severity: sev(vuln.severity),
|
||||
details: [name && `pkg: ${name}${version ? ` ${version}` : ''}`,
|
||||
vuln.description && truncate(String(vuln.description), 200)].filter(Boolean).join(' | ') || id,
|
||||
};
|
||||
if (/^CVE-/i.test(id)) f.cve = [id];
|
||||
out.push(f);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const PARSERS: Record<string, (raw: string) => ToolFinding[]> = {
|
||||
nuclei: parseNuclei,
|
||||
httpx: parseHttpx,
|
||||
dalfox: parseDalfox,
|
||||
ffuf: parseFfuf,
|
||||
katana: parseKatana,
|
||||
semgrep: parseSemgrep,
|
||||
gitleaks: parseGitleaks,
|
||||
trivy: parseTrivy,
|
||||
grype: parseGrype,
|
||||
};
|
||||
|
||||
/** Adapter ids that have a structured output parser wired here. */
|
||||
|
||||
Reference in new issue
Block a user