feat(arsenal): add source and supply-chain parsers (#97)
Follow-up to the JSON-scanner parser layer: wire semgrep, gitleaks, trivy and
grype into parseToolOutput so their findings become structured, gate-verifiable
evidence instead of raw text (they were parserStatus:'planned').
- semgrep (scan --json) → one finding per rule match; ERROR/WARNING/INFO
mapped to severity; CWE token pulled from
metadata (e.g. "CWE-78: …" → CWE-78); refs → remediation.
- trivy (fs --format json) → one finding per package vuln; CVE + CweIDs +
"upgrade <pkg> to <fixed>" remediation.
- grype (dir:. -o json) → one finding per match; CVE + package coordinates.
- gitleaks (detect --report json --redact) → one high finding per secret. Defence in
depth: the details are run through redactString()
so a raw secret can NEVER reach a finding / the vault,
even though the adapter already passes --redact.
Broadens the severity map to cover scanner vocabularies (ERROR/WARNING, CRITICAL…
NEGLIGIBLE, moderate) onto the five canonical levels. Flips parserStatus to
'structured' for the four tools. Extends parsers.test.ts + fixtures: field mapping,
the gitleaks redaction guarantee, and the end-to-end gate pass for every new scanner.
Stacked on the parser layer (#42); pairs with the scanner-invocation fix (#40) that
gives these tools the JSON-emitting arg templates. Until #40 lands the parsers simply
degrade to [] (no JSON to parse) — no misbehaviour. Secret scanners beyond gitleaks
and SARIF-style tools remain a follow-up.
Co-authored-by: Pazificateur69 <agagliardi@guardiaschool.fr>
This commit is contained in:
1 parent
f8c283377c
commit
cf0090adf4
7 files changed
+171
-6
No files matched your search
@@ -0,0 +1 @@
|
|||||||
|
[{"RuleID":"aws-access-token","Description":"AWS Access Key AKIAIOSFODNN7EXAMPLE committed to source","File":"config/prod.env","StartLine":12,"Match":"REDACTED","Secret":"REDACTED","Commit":"abc1234","Author":"dev"}]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"matches":[{"vulnerability":{"id":"CVE-2022-23529","severity":"High","description":"jsonwebtoken is vulnerable to signature verification bypass."},"artifact":{"name":"jsonwebtoken","version":"8.5.1","type":"npm"}}]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"results":[{"check_id":"python.lang.security.dangerous-subprocess-use","path":"app/run.py","start":{"line":42},"end":{"line":42},"extra":{"message":"Detected subprocess call with shell=True — command injection risk","severity":"ERROR","metadata":{"cwe":["CWE-78: OS Command Injection"],"owasp":["A03:2021 - Injection"],"references":["https://owasp.org/Top10/A03_2021-Injection/"]}}}],"errors":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"SchemaVersion":2,"Results":[{"Target":"package-lock.json","Class":"lang-pkgs","Type":"npm","Vulnerabilities":[{"VulnerabilityID":"CVE-2021-3749","PkgName":"axios","InstalledVersion":"0.21.1","FixedVersion":"0.21.2","Severity":"HIGH","Title":"axios: Regular expression denial of service via trim","Description":"axios is vulnerable to Inefficient Regular Expression Complexity (ReDoS).","CweIDs":["CWE-400"]}]}]}
|
||||||
@@ -103,7 +103,47 @@ describe('parseToolOutput — honesty contract (never fabricate, never throw)',
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('exposes exactly the wired parser ids', () => {
|
it('exposes exactly the wired parser ids', () => {
|
||||||
expect([...PARSED_TOOL_IDS].sort()).toEqual(['dalfox', 'ffuf', 'httpx', 'katana', 'nuclei']);
|
expect([...PARSED_TOOL_IDS].sort()).toEqual(
|
||||||
|
['dalfox', 'ffuf', 'gitleaks', 'grype', 'httpx', 'katana', 'nuclei', 'semgrep', 'trivy'],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('parseToolOutput — source / supply-chain scanners', () => {
|
||||||
|
it('semgrep: one finding per rule match with mapped severity + CWE token', () => {
|
||||||
|
const f = parseToolOutput('semgrep', fixture('semgrep.json'));
|
||||||
|
expect(f).toHaveLength(1);
|
||||||
|
expect(f[0].severity).toBe('high'); // ERROR → high
|
||||||
|
expect(f[0].cwe).toEqual(['CWE-78']); // extracted from "CWE-78: OS Command Injection"
|
||||||
|
expect(f[0].details).toContain('app/run.py:42');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('trivy: one finding per package vuln with CVE, CWE, and an upgrade remediation', () => {
|
||||||
|
const f = parseToolOutput('trivy', fixture('trivy.json'));
|
||||||
|
expect(f).toHaveLength(1);
|
||||||
|
expect(f[0].severity).toBe('high');
|
||||||
|
expect(f[0].cve).toEqual(['CVE-2021-3749']);
|
||||||
|
expect(f[0].cwe).toEqual(['CWE-400']);
|
||||||
|
expect(f[0].remediation).toContain('0.21.2');
|
||||||
|
expect(f[0].details).toContain('axios');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('grype: one finding per match with CVE + package coordinates', () => {
|
||||||
|
const f = parseToolOutput('grype', fixture('grype.json'));
|
||||||
|
expect(f).toHaveLength(1);
|
||||||
|
expect(f[0].severity).toBe('high'); // "High" → high
|
||||||
|
expect(f[0].cve).toEqual(['CVE-2022-23529']);
|
||||||
|
expect(f[0].details).toContain('jsonwebtoken');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('gitleaks: one high finding per secret, and the raw secret is REDACTED from details', () => {
|
||||||
|
const f = parseToolOutput('gitleaks', fixture('gitleaks.json'));
|
||||||
|
expect(f).toHaveLength(1);
|
||||||
|
expect(f[0].severity).toBe('high');
|
||||||
|
expect(f[0].details).toContain('config/prod.env:12');
|
||||||
|
// Defense in depth: the AWS key present in the raw report must never survive into the finding.
|
||||||
|
expect(f[0].details).not.toContain('AKIAIOSFODNN7EXAMPLE');
|
||||||
|
expect(f[0].details).toContain('[redacted]');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -144,6 +184,8 @@ describe('end-to-end: a parsed finding passes the live provenance gate', () => {
|
|||||||
for (const [id, file] of [
|
for (const [id, file] of [
|
||||||
['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'],
|
['nuclei', 'nuclei.jsonl'], ['httpx', 'httpx.jsonl'], ['dalfox', 'dalfox.json'],
|
||||||
['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'],
|
['ffuf', 'ffuf.json'], ['katana', 'katana.jsonl'],
|
||||||
|
['semgrep', 'semgrep.json'], ['gitleaks', 'gitleaks.json'],
|
||||||
|
['trivy', 'trivy.json'], ['grype', 'grype.json'],
|
||||||
] as const) {
|
] as const) {
|
||||||
const raw = fixture(file);
|
const raw = fixture(file);
|
||||||
for (const tf of parseToolOutput(id, raw)) {
|
for (const tf of parseToolOutput(id, raw)) {
|
||||||
|
|||||||
@@ -339,7 +339,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
|||||||
outputFormats: ['json', 'sarif'],
|
outputFormats: ['json', 'sarif'],
|
||||||
installHint: 'brew install semgrep or pipx install semgrep',
|
installHint: 'brew install semgrep or pipx install semgrep',
|
||||||
commandHint: 'semgrep scan --config auto --json .',
|
commandHint: 'semgrep scan --config auto --json .',
|
||||||
parserStatus: 'planned',
|
parserStatus: 'structured',
|
||||||
notes: 'Static analysis backbone for repository hardening.',
|
notes: 'Static analysis backbone for repository hardening.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -355,7 +355,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
|||||||
outputFormats: ['json', 'sarif'],
|
outputFormats: ['json', 'sarif'],
|
||||||
installHint: 'brew install gitleaks',
|
installHint: 'brew install gitleaks',
|
||||||
commandHint: 'gitleaks detect --source . --report-format json --redact',
|
commandHint: 'gitleaks detect --source . --report-format json --redact',
|
||||||
parserStatus: 'planned',
|
parserStatus: 'structured',
|
||||||
notes: 'Secret detection must redact values before ledger insertion.',
|
notes: 'Secret detection must redact values before ledger insertion.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -387,7 +387,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
|||||||
outputFormats: ['json', 'sarif'],
|
outputFormats: ['json', 'sarif'],
|
||||||
installHint: 'brew install aquasecurity/trivy/trivy',
|
installHint: 'brew install aquasecurity/trivy/trivy',
|
||||||
commandHint: 'trivy fs --format json .',
|
commandHint: 'trivy fs --format json .',
|
||||||
parserStatus: 'planned',
|
parserStatus: 'structured',
|
||||||
notes: 'Vulnerability and IaC scanner for repos and images.',
|
notes: 'Vulnerability and IaC scanner for repos and images.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -419,7 +419,7 @@ export const TOOL_ADAPTERS: ToolAdapter[] = [
|
|||||||
outputFormats: ['json'],
|
outputFormats: ['json'],
|
||||||
installHint: 'brew install grype',
|
installHint: 'brew install grype',
|
||||||
commandHint: 'grype dir:. -o json',
|
commandHint: 'grype dir:. -o json',
|
||||||
parserStatus: 'planned',
|
parserStatus: 'structured',
|
||||||
notes: 'Pairs well with Syft SBOMs for dependency vulnerability triage.',
|
notes: 'Pairs well with Syft SBOMs for dependency vulnerability triage.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
+120
-1
@@ -18,12 +18,20 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import type { Severity, ToolFinding } from '../types/index.js';
|
import type { Severity, ToolFinding } from '../types/index.js';
|
||||||
|
import { redactString } from '../redact.js';
|
||||||
|
|
||||||
// ── small, defensive helpers ────────────────────────────────────────────────
|
// ── small, defensive helpers ────────────────────────────────────────────────
|
||||||
const SEVERITIES = new Set<Severity>(['critical', 'high', 'medium', 'low', 'info']);
|
const SEVERITIES = new Set<Severity>(['critical', 'high', 'medium', 'low', 'info']);
|
||||||
|
// Map the many scanner severity vocabularies (semgrep ERROR/WARNING/INFO, trivy/grype
|
||||||
|
// CRITICAL…NEGLIGIBLE, npm-audit "moderate", …) onto the five canonical levels.
|
||||||
|
const SEVERITY_ALIAS: Record<string, Severity> = {
|
||||||
|
error: 'high', warning: 'medium', warn: 'medium', inventory: 'info', experiment: 'info',
|
||||||
|
moderate: 'medium', negligible: 'info', unknown: 'info', none: 'info', informational: 'info',
|
||||||
|
};
|
||||||
const sev = (v: unknown, fallback: Severity = 'info'): Severity => {
|
const sev = (v: unknown, fallback: Severity = 'info'): Severity => {
|
||||||
const s = String(v ?? '').trim().toLowerCase();
|
const s = String(v ?? '').trim().toLowerCase();
|
||||||
return SEVERITIES.has(s as Severity) ? (s as Severity) : fallback;
|
if (SEVERITIES.has(s as Severity)) return s as Severity;
|
||||||
|
return SEVERITY_ALIAS[s] ?? fallback;
|
||||||
};
|
};
|
||||||
const asObj = (v: unknown): Record<string, unknown> =>
|
const asObj = (v: unknown): Record<string, unknown> =>
|
||||||
v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
v && typeof v === 'object' && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
|
||||||
@@ -176,12 +184,123 @@ function parseKatana(raw: string): ToolFinding[] {
|
|||||||
}];
|
}];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── semgrep scan --json : one finding per rule match (SAST) ──────────────────
|
||||||
|
function parseSemgrep(raw: string): ToolFinding[] {
|
||||||
|
const results = ((): unknown[] => {
|
||||||
|
const r = asObj(jsonDoc(raw)).results;
|
||||||
|
return Array.isArray(r) ? r : [];
|
||||||
|
})();
|
||||||
|
const out: ToolFinding[] = [];
|
||||||
|
for (const item of results) {
|
||||||
|
const e = asObj(item);
|
||||||
|
const extra = asObj(e.extra);
|
||||||
|
const meta = asObj(extra.metadata);
|
||||||
|
const checkId = String(e.check_id ?? e.checkId ?? 'semgrep rule');
|
||||||
|
const path = String(e.path ?? '');
|
||||||
|
const line = num(asObj(e.start).line);
|
||||||
|
const where = path ? `${path}${line !== undefined ? `:${line}` : ''}` : '';
|
||||||
|
const f: ToolFinding = {
|
||||||
|
title: checkId,
|
||||||
|
severity: sev(extra.severity),
|
||||||
|
details: [where && `at ${where}`, extra.message && truncate(String(extra.message))].filter(Boolean).join(' | ') || checkId,
|
||||||
|
};
|
||||||
|
// semgrep CWE metadata is like ["CWE-89: SQL Injection"] — keep the CWE-NNN token.
|
||||||
|
const cwe = asStrArray(meta.cwe).map((c) => c.match(/CWE-\d+/i)?.[0] ?? '').filter(Boolean);
|
||||||
|
if (cwe.length) f.cwe = cwe.map((c) => c.toUpperCase());
|
||||||
|
const refs = asStrArray(meta.references);
|
||||||
|
if (refs.length) f.remediation = `refs: ${refs.slice(0, 3).join(', ')}`;
|
||||||
|
out.push(f);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── gitleaks detect --report-format json --redact : one finding per secret ────
|
||||||
|
function parseGitleaks(raw: string): ToolFinding[] {
|
||||||
|
const arr: unknown[] = Array.isArray(jsonDoc(raw)) ? (jsonDoc(raw) as unknown[]) : [];
|
||||||
|
const out: ToolFinding[] = [];
|
||||||
|
for (const item of arr) {
|
||||||
|
const e = asObj(item);
|
||||||
|
const rule = String(e.RuleID ?? e.Description ?? 'secret');
|
||||||
|
const file = String(e.File ?? '');
|
||||||
|
const line = num(e.StartLine);
|
||||||
|
const where = file ? `${file}${line !== undefined ? `:${line}` : ''}` : '';
|
||||||
|
// Defense in depth: the adapter already runs gitleaks with --redact, but scrub the details
|
||||||
|
// through redactString anyway so a raw secret can NEVER reach a finding or the evidence vault.
|
||||||
|
const details = redactString([where && `at ${where}`, String(e.Description ?? '')].filter(Boolean).join(' | '));
|
||||||
|
out.push({ title: `secret: ${rule}`, severity: 'high', details: details || `secret: ${rule}` });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── trivy fs --format json : one finding per package vulnerability ────────────
|
||||||
|
function parseTrivy(raw: string): ToolFinding[] {
|
||||||
|
const results = ((): unknown[] => {
|
||||||
|
const r = asObj(jsonDoc(raw)).Results;
|
||||||
|
return Array.isArray(r) ? r : [];
|
||||||
|
})();
|
||||||
|
const out: ToolFinding[] = [];
|
||||||
|
for (const res of results) {
|
||||||
|
const target = String(asObj(res).Target ?? '');
|
||||||
|
const vulns = asObj(res).Vulnerabilities;
|
||||||
|
if (!Array.isArray(vulns)) continue;
|
||||||
|
for (const item of vulns) {
|
||||||
|
const v = asObj(item);
|
||||||
|
const id = String(v.VulnerabilityID ?? '');
|
||||||
|
const pkg = String(v.PkgName ?? '');
|
||||||
|
const installed = String(v.InstalledVersion ?? '');
|
||||||
|
const fixed = String(v.FixedVersion ?? '');
|
||||||
|
const f: ToolFinding = {
|
||||||
|
title: String(v.Title || id || 'vulnerability'),
|
||||||
|
severity: sev(v.Severity),
|
||||||
|
details: [target && `target: ${target}`, pkg && `pkg: ${pkg}${installed ? ` ${installed}` : ''}`,
|
||||||
|
v.Description && truncate(String(v.Description), 200)].filter(Boolean).join(' | ') || id,
|
||||||
|
};
|
||||||
|
if (/^CVE-/i.test(id)) f.cve = [id];
|
||||||
|
const cwe = asStrArray(v.CweIDs);
|
||||||
|
if (cwe.length) f.cwe = cwe;
|
||||||
|
if (fixed) f.remediation = `upgrade ${pkg || 'package'} to ${fixed}`;
|
||||||
|
out.push(f);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── grype dir:. -o json : one finding per matched vulnerability ───────────────
|
||||||
|
function parseGrype(raw: string): ToolFinding[] {
|
||||||
|
const matches = ((): unknown[] => {
|
||||||
|
const m = asObj(jsonDoc(raw)).matches;
|
||||||
|
return Array.isArray(m) ? m : [];
|
||||||
|
})();
|
||||||
|
const out: ToolFinding[] = [];
|
||||||
|
for (const item of matches) {
|
||||||
|
const m = asObj(item);
|
||||||
|
const vuln = asObj(m.vulnerability);
|
||||||
|
const art = asObj(m.artifact);
|
||||||
|
const id = String(vuln.id ?? '');
|
||||||
|
const name = String(art.name ?? '');
|
||||||
|
const version = String(art.version ?? '');
|
||||||
|
const f: ToolFinding = {
|
||||||
|
title: id || 'vulnerability',
|
||||||
|
severity: sev(vuln.severity),
|
||||||
|
details: [name && `pkg: ${name}${version ? ` ${version}` : ''}`,
|
||||||
|
vuln.description && truncate(String(vuln.description), 200)].filter(Boolean).join(' | ') || id,
|
||||||
|
};
|
||||||
|
if (/^CVE-/i.test(id)) f.cve = [id];
|
||||||
|
out.push(f);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
const PARSERS: Record<string, (raw: string) => ToolFinding[]> = {
|
const PARSERS: Record<string, (raw: string) => ToolFinding[]> = {
|
||||||
nuclei: parseNuclei,
|
nuclei: parseNuclei,
|
||||||
httpx: parseHttpx,
|
httpx: parseHttpx,
|
||||||
dalfox: parseDalfox,
|
dalfox: parseDalfox,
|
||||||
ffuf: parseFfuf,
|
ffuf: parseFfuf,
|
||||||
katana: parseKatana,
|
katana: parseKatana,
|
||||||
|
semgrep: parseSemgrep,
|
||||||
|
gitleaks: parseGitleaks,
|
||||||
|
trivy: parseTrivy,
|
||||||
|
grype: parseGrype,
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Adapter ids that have a structured output parser wired here. */
|
/** Adapter ids that have a structured output parser wired here. */
|
||||||
|
|||||||
Reference in new issue
Block a user