feat: pack-hunt safety + opt-in Kali+ arsenal + capability-approval gate + tester-feedback UX
Adds three coordinated capabilities and tightens two defaults on purpose — this is safety hardening, NOT a backward-compatible drop-in (see "Tightened defaults"). Fifteen independent adversarial review passes; findings fixed and test-pinned. - Pack-hunt safety: a strict-majority adjudicator with a comment/string/regex/ raw-string/heredoc/POD-aware cite-check (language-gated across C/C++/Py/Rust/Go/ Java/JS/TS/shell/SQL/PHP/Ruby/Perl) so a hallucinated killing-guard can't refute a real finding — the cited comparison's operator must match source, and a string/regex/heredoc/generic-arglist/template-interpolation is blanked before matching. Plus a rate-limit/degraded-health tracker, a global LLM semaphore, and a PackBoard with an atomic claim CAS + snapshot-on-emit/clone-on-ingest. - Opt-in Kali+ arsenal (T3MP3ST_FULL_ARSENAL, OFF by default): 35 default tools (31 built-in + 4 external-CLI wrappers) -> 83 with the catalog armed (verify-claims CLAIM 4). Most adapters need the underlying binary installed. metasploit / hydra / bloodhound are never minted through the generic factory. - Capability-approval + spicy-action gate for the NEW dangerous arsenal only (pre-existing built-ins keep running freely): approve-once, fail-safe deny when unattended, redacted audit trail over SSE. T3MP3ST_GATE_BUILTINS=1 extends it to the spicy built-in probes. Audit redaction is a pure, unit-tested module. Tightened defaults (intentional, ON by default — NOT backward-compatible): - Egress containment: once a mission target is set, built-in networked tools refuse off-scope public hosts (not the target/subdomains, not loopback/private) with SCOPE DENIED — keeps the agent on authorized scope. - Role-specialized operator toolkits: each archetype gets a curated default tool menu (scanner -> scan tools, exploiter -> sqli/xss/etc). Tools stay registered team-wide; the per-role menu is the operator's working set. UX (from live-tester feedback): a one-click "Approve & run" modal when a standard Hunt blocks on an external-target receipt (was a dead-end); bounded-parallel benchmark file ingestion; the deterministic mission-gate chip co-located with the subjective SITREP pace narration (no more "behind schedule" next to a clean gate); and click-through finding detail + evidence in the UI (was export-only). Docs: README + INTEGRITY_LEDGER made re-derivable — honest 35/83 tool count, the egress-containment default documented, and the Cybench number reconciled to the single verify-claims-enforced 23/40 = 58% (dropped the stale/uncited 21 and 26). README voice pass: killed the underselling `Successful Exploits: 0` disqualifier (read as "never popped anything" — false, given CVE-Zero + the live disclosure pipeline); swagger now rests on receipts, not vibes. Also corrected the operator status — the downstream operators run the SAME real tool-backed ReAct loop as recon (not "orchestration stubs"); the honest limit is that the coordinated swarm is unproven (single-agent is the benchmarked path), not that they're scaffolding. Gates: 292 tests, tsc + lint clean, verify-claims 20/20, arsenal:smoke 125/0, disclosure-CLI self-test 11/11. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
5255000cb6
commit
06ffe3ef66
35 files changed
+5019
-211
No files matched your search
@@ -0,0 +1,241 @@
|
||||
/**
|
||||
* Rate-limit / degraded-health + LLM semaphore — Phase-0 pack-hunt safety primitive (HT-3).
|
||||
* A rate-limited slot must NEVER read as "clean / found-nothing", and a single global semaphore
|
||||
* must cap ALL local-agent calls. Pins: isRateLimited on real quota strings, the 0.5 degraded
|
||||
* threshold, and that the Semaphore never exceeds max concurrent holders.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { isRateLimited, DegradedTracker, Semaphore } from '../pack/health.js';
|
||||
|
||||
describe('isRateLimited', () => {
|
||||
it('catches real usage-limit / 429 / quota signatures', () => {
|
||||
// real Claude/Anthropic banner
|
||||
expect(isRateLimited("ERROR: You've hit your usage limit. try again at 12:10 PM")).toBe(true);
|
||||
expect(isRateLimited('Claude: you have hit your usage limit')).toBe(true);
|
||||
expect(isRateLimited('429 Too Many Requests')).toBe(true);
|
||||
expect(isRateLimited('HTTP 429')).toBe(true);
|
||||
expect(isRateLimited('rate limit reached for this model')).toBe(true);
|
||||
expect(isRateLimited('rate-limited')).toBe(true);
|
||||
expect(isRateLimited('quota exceeded')).toBe(true);
|
||||
expect(isRateLimited('{"error":{"code":"insufficient_quota"}}')).toBe(true);
|
||||
expect(isRateLimited('Too Many Requests')).toBe(true);
|
||||
});
|
||||
|
||||
it('catches provider-specific throttle signals the original regex missed', () => {
|
||||
expect(isRateLimited('Error: Overloaded (overloaded_error)')).toBe(true); // Anthropic 529
|
||||
expect(isRateLimited('HTTP 529: overloaded')).toBe(true);
|
||||
expect(isRateLimited('TooManyRequestsException: Rate exceeded')).toBe(true); // AWS Bedrock
|
||||
expect(isRateLimited('grpc status RESOURCE_EXHAUSTED')).toBe(true); // gRPC / Vertex
|
||||
expect(isRateLimited('You exceeded your current quota, please check your plan')).toBe(true); // OpenAI reversed
|
||||
expect(isRateLimited('request was throttled, retry-after 30s')).toBe(true);
|
||||
expect(isRateLimited('503 Service Unavailable')).toBe(true);
|
||||
});
|
||||
|
||||
it('does NOT false-positive on a bare integer that merely looks like a status code', () => {
|
||||
expect(isRateLimited('see line 429 of the file')).toBe(false);
|
||||
expect(isRateLimited('CWE-529: exposure of sensitive information')).toBe(false);
|
||||
expect(isRateLimited('found 4290 candidate sinks')).toBe(false);
|
||||
});
|
||||
|
||||
it('does NOT false-positive on benign prose that incidentally contains throttle vocabulary', () => {
|
||||
// the signals must be the SPECIFIC provider tokens, not bare words in finding narratives
|
||||
expect(isRateLimited('resource exhausted state in the parser')).toBe(false); // not RESOURCE_EXHAUSTED
|
||||
expect(isRateLimited('a note about an overloaded operator')).toBe(false); // not overloaded_error
|
||||
expect(isRateLimited('the quota management module')).toBe(false); // not "quota exceeded"
|
||||
expect(isRateLimited('capacity planning document')).toBe(false); // not "at capacity"
|
||||
expect(isRateLimited('opened the throttle valve')).toBe(false); // "throttle" (noun) != "throttled"/"throttling"
|
||||
expect(isRateLimited('disk quota for the container')).toBe(false);
|
||||
});
|
||||
|
||||
it('does NOT flag a rate-limit / DoS FINDING body as a throttled slot (finding vs throttle)', () => {
|
||||
// a pack agent's OWN result describing a rate-limit/DoS vuln must read as a FINDING, not degraded
|
||||
expect(isRateLimited('Found: rate limiting is MISSING on /login (CWE-307)')).toBe(false);
|
||||
expect(isRateLimited('no rate-limit on password reset — brute force possible')).toBe(false);
|
||||
expect(isRateLimited('CWE-400: uncontrolled resource consumption leads to service unavailable')).toBe(false);
|
||||
expect(isRateLimited('recommend: slow down the attacker via a captcha')).toBe(false);
|
||||
expect(isRateLimited('concurrent request handling bug (race)')).toBe(false);
|
||||
expect(isRateLimited('the endpoint enforces no usage limit at all')).toBe(false);
|
||||
// but a limit actually being HIT (a genuine throttle) is still caught
|
||||
expect(isRateLimited('rate limit exceeded')).toBe(true);
|
||||
expect(isRateLimited('rate-limited')).toBe(true);
|
||||
});
|
||||
|
||||
it('still catches the canonical provider throttles the tightening must not lose (8th-pass regression)', () => {
|
||||
expect(isRateLimited('{"type":"error","error":{"type":"rate_limit_error"}}')).toBe(true); // Anthropic 429 body
|
||||
expect(isRateLimited('exceeded your per-minute rate limit')).toBe(true); // qualifier between verb and "rate limit"
|
||||
expect(isRateLimited('usage limit reached')).toBe(true); // Claude subscription banner (verb after)
|
||||
expect(isRateLimited("You've reached your usage limit")).toBe(true); // verb before
|
||||
expect(isRateLimited('Message limit reached')).toBe(true);
|
||||
// and the finding-vs-throttle discrimination still holds
|
||||
expect(isRateLimited('the endpoint enforces no usage limit at all')).toBe(false);
|
||||
expect(isRateLimited('rate limiting is MISSING on /login')).toBe(false);
|
||||
});
|
||||
|
||||
it('catches the provider throttles the tightening MUST NOT lose (9th-pass false-negatives)', () => {
|
||||
expect(isRateLimited('You have exceeded a secondary rate limit')).toBe(true); // GitHub/GitLab
|
||||
expect(isRateLimited('ServiceQuotaExceededException: rate exceeded')).toBe(true); // AWS Bedrock
|
||||
expect(isRateLimited('The engine is currently overloaded, please try again')).toBe(true); // OpenAI/Anthropic 529 prose
|
||||
expect(isRateLimited('Overloaded')).toBe(true); // Anthropic 529 bare body
|
||||
expect(isRateLimited('grpc error: Resource has been exhausted (e.g. check quota)')).toBe(true); // gRPC/Vertex prose
|
||||
// still NOT a false-positive on benign code-review vocabulary
|
||||
expect(isRateLimited('a note about an overloaded operator')).toBe(false);
|
||||
expect(isRateLimited('resource exhausted state in the parser')).toBe(false);
|
||||
});
|
||||
|
||||
it('catches the underscore 429 enum and future-tense throttle bodies (11th-pass false-negatives)', () => {
|
||||
expect(isRateLimited('{"reason":"too_many_requests"}')).toBe(true); // HTTP 429 reason enum
|
||||
expect(isRateLimited('TOO_MANY_REQUESTS')).toBe(true);
|
||||
expect(isRateLimited("This request would exceed your organization's rate limit")).toBe(true); // Anthropic future-tense
|
||||
expect(isRateLimited('you will exceed your per-minute rate limit')).toBe(true);
|
||||
// and still not a finding body
|
||||
expect(isRateLimited('rate limiting is MISSING on /login')).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores normal findings output and empty/undefined text', () => {
|
||||
expect(isRateLimited('[{"title":"x","file":"a.py","line":5,"cwe":"CWE-22"}]')).toBe(false);
|
||||
expect(isRateLimited('Clean — nothing exploitable in my specialty')).toBe(false);
|
||||
expect(isRateLimited('')).toBe(false);
|
||||
// guards against undefined leaking in despite the string type (matches cli-swarm's String(x||'') coercion)
|
||||
expect(isRateLimited(undefined as unknown as string)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DegradedTracker — 0.5 threshold', () => {
|
||||
it('is OK when all calls succeed', () => {
|
||||
const t = new DegradedTracker();
|
||||
t.record(true);
|
||||
t.record(true);
|
||||
t.record(true);
|
||||
expect(t.calls).toBe(3);
|
||||
expect(t.fails).toBe(0);
|
||||
expect(t.verdict()).toBe('OK');
|
||||
});
|
||||
|
||||
it('stays OK just below the threshold (2 of 5 failed = 0.4)', () => {
|
||||
const t = new DegradedTracker();
|
||||
[true, true, true, false, false].forEach((ok) => t.record(ok));
|
||||
expect(t.failRatio()).toBeCloseTo(0.4);
|
||||
expect(t.verdict()).toBe('OK');
|
||||
});
|
||||
|
||||
it('flips to INVALID_DEGRADED exactly AT the 0.5 threshold (2 of 4 failed)', () => {
|
||||
const t = new DegradedTracker();
|
||||
[true, true, false, false].forEach((ok) => t.record(ok));
|
||||
expect(t.failRatio()).toBe(0.5);
|
||||
expect(t.verdict()).toBe('INVALID_DEGRADED');
|
||||
});
|
||||
|
||||
it('is INVALID_DEGRADED when failures dominate', () => {
|
||||
const t = new DegradedTracker();
|
||||
[false, false, false, true].forEach((ok) => t.record(ok));
|
||||
expect(t.verdict()).toBe('INVALID_DEGRADED');
|
||||
});
|
||||
|
||||
it('a rate-limited slot poisons the verdict — never clean', () => {
|
||||
const t = new DegradedTracker();
|
||||
// two slots came back "successfully" but one is actually a quota banner
|
||||
t.recordResult(true, 'Clean — no findings');
|
||||
t.recordResult(true, "You've hit your usage limit");
|
||||
// the quota banner is booked as a failure => 1/2 = 0.5 => degraded
|
||||
expect(t.fails).toBe(1);
|
||||
expect(t.verdict()).toBe('INVALID_DEGRADED');
|
||||
});
|
||||
|
||||
it('an empty run (no calls) is INVALID_DEGRADED, not clean', () => {
|
||||
const t = new DegradedTracker();
|
||||
expect(t.verdict()).toBe('INVALID_DEGRADED');
|
||||
expect(t.failRatio()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Semaphore', () => {
|
||||
it('rejects a non-positive max', () => {
|
||||
expect(() => new Semaphore(0)).toThrow(RangeError);
|
||||
expect(() => new Semaphore(-1)).toThrow(RangeError);
|
||||
});
|
||||
|
||||
it('never exceeds max concurrent holders under a burst', async () => {
|
||||
const MAX = 3;
|
||||
const N = 40;
|
||||
const sem = new Semaphore(MAX);
|
||||
let live = 0;
|
||||
let peak = 0;
|
||||
|
||||
const task = async (): Promise<void> => {
|
||||
const release = await sem.acquire();
|
||||
live += 1;
|
||||
peak = Math.max(peak, live);
|
||||
expect(live).toBeLessThanOrEqual(MAX); // invariant checked WHILE holding
|
||||
// yield across a macrotask so overlap is real, not artificial
|
||||
await new Promise((r) => setTimeout(r, Math.floor(Math.random() * 5)));
|
||||
live -= 1;
|
||||
release();
|
||||
};
|
||||
|
||||
await Promise.all(Array.from({ length: N }, task));
|
||||
expect(peak).toBe(MAX); // saturates (N >> MAX) but never over
|
||||
expect(live).toBe(0);
|
||||
expect(sem.inFlight).toBe(0);
|
||||
expect(sem.queued).toBe(0);
|
||||
});
|
||||
|
||||
it('serializes fully at max=1', async () => {
|
||||
const sem = new Semaphore(1);
|
||||
let live = 0;
|
||||
let peak = 0;
|
||||
const task = async (): Promise<void> => {
|
||||
const release = await sem.acquire();
|
||||
live += 1;
|
||||
peak = Math.max(peak, live);
|
||||
await new Promise((r) => setTimeout(r, 1));
|
||||
live -= 1;
|
||||
release();
|
||||
};
|
||||
await Promise.all([task(), task(), task(), task()]);
|
||||
expect(peak).toBe(1);
|
||||
});
|
||||
|
||||
it('serves waiters FIFO', async () => {
|
||||
const sem = new Semaphore(1);
|
||||
const order: number[] = [];
|
||||
const first = await sem.acquire(); // occupy the only slot
|
||||
// three callers queue up in order
|
||||
const p2 = sem.acquire().then((rel) => { order.push(2); rel(); });
|
||||
const p3 = sem.acquire().then((rel) => { order.push(3); rel(); });
|
||||
const p4 = sem.acquire().then((rel) => { order.push(4); rel(); });
|
||||
expect(sem.queued).toBe(3);
|
||||
first(); // release => drains in enqueue order
|
||||
await Promise.all([p2, p3, p4]);
|
||||
expect(order).toEqual([2, 3, 4]);
|
||||
});
|
||||
|
||||
it('double-release is a no-op (cannot inflate the pool past max)', async () => {
|
||||
const sem = new Semaphore(1);
|
||||
const release = await sem.acquire();
|
||||
release();
|
||||
release(); // second call must NOT free a phantom extra slot
|
||||
|
||||
let live = 0;
|
||||
let peak = 0;
|
||||
const task = async (): Promise<void> => {
|
||||
const rel = await sem.acquire();
|
||||
live += 1;
|
||||
peak = Math.max(peak, live);
|
||||
await new Promise((r) => setTimeout(r, 1));
|
||||
live -= 1;
|
||||
rel();
|
||||
};
|
||||
await Promise.all([task(), task(), task()]);
|
||||
expect(peak).toBe(1); // still fully serialized
|
||||
});
|
||||
|
||||
it('run() releases even when the body throws', async () => {
|
||||
const sem = new Semaphore(1);
|
||||
await expect(sem.run(async () => { throw new Error('boom'); })).rejects.toThrow('boom');
|
||||
expect(sem.inFlight).toBe(0);
|
||||
// slot is free again — a subsequent acquire resolves immediately
|
||||
const release = await sem.acquire();
|
||||
expect(sem.inFlight).toBe(1);
|
||||
release();
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user