Add files via upload

This commit is contained in:
pliny authored and GitHub committed 2026-07-02 13:56:02 -04:00
commit 5bec530a89
52 files changed
+33300

No files matched your search

+49
View File
@@ -0,0 +1,49 @@
import { describe, it, expect } from 'vitest';
import {
AI_REDTEAM_PLAYBOOK,
AI_REDTEAM_TECHNIQUE_IDS,
aiRedTeamBriefing,
redTeamTechnique,
} from '../resources/ai-redteam-playbook.js';
describe('AI red-team playbook (distilled from L1B3RT4S / P4RS3LT0NGV3)', () => {
it('has the full distilled technique set', () => {
expect(AI_REDTEAM_PLAYBOOK.length).toBeGreaterThanOrEqual(15);
expect(AI_REDTEAM_TECHNIQUE_IDS.length).toBe(AI_REDTEAM_PLAYBOOK.length);
});
it('every technique is well-formed (all fields, kebab-case id)', () => {
for (const t of AI_REDTEAM_PLAYBOOK) {
expect(t.id).toMatch(/^[a-z0-9]+(-[a-z0-9]+)*$/);
for (const field of ['category', 'principle', 'redteamUse', 'defense'] as const) {
expect(typeof t[field]).toBe('string');
expect(t[field].length).toBeGreaterThan(10);
}
}
});
it('ids are unique', () => {
expect(new Set(AI_REDTEAM_TECHNIQUE_IDS).size).toBe(AI_REDTEAM_TECHNIQUE_IDS.length);
});
it('covers the load-bearing technique classes', () => {
for (const id of ['refusal-suppression-inversion', 'prefill-affirmation', 'format-scaffold-hijack', 'stacked-composition']) {
expect(AI_REDTEAM_TECHNIQUE_IDS).toContain(id);
}
});
it('lookup resolves a known id and rejects unknown', () => {
expect(redTeamTechnique('format-scaffold-hijack')?.category).toMatch(/format/i);
expect(redTeamTechnique('does-not-exist')).toBeUndefined();
});
it('briefing is a non-empty multi-line summary', () => {
const brief = aiRedTeamBriefing();
expect(brief.length).toBeGreaterThan(100);
expect(brief.split('\n').length).toBe(AI_REDTEAM_PLAYBOOK.length);
});
it('every technique carries a defensive countermeasure (this is defensive methodology)', () => {
expect(AI_REDTEAM_PLAYBOOK.every(t => t.defense.trim().length > 0)).toBe(true);
});
});
+352
View File
@@ -0,0 +1,352 @@
/**
* code-ingest tests
*
* Builds a tiny temp Python repo with mkdtempSync, runs the full
* ingestRepository pipeline over it, and asserts the SELECTION behavior:
* - blocks extracted with correct names + plausible line ranges
* - entry points detected from @app.route decorators
* - call graph resolves an intra-repo call (handler -> helper)
* - classification precedence: route -> exposed_externally,
* fetch_url -> attack_surface, validate_token -> security_control,
* format_name -> neutral
* - priority: the exposed handler outranks the neutral util
* - packAnalysisUnits preserves priority order and respects a small budget
*
* Reminder: code-ingest is a PYTHON-ONLY regex "AST-lite" prototype (see the
* module header) — these tests exercise the constructs it is designed to handle.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import {
ingestRepository,
createPythonIngestConfig,
parseFile,
buildCallGraph,
findEntryPoints,
packAnalysisUnits,
formatUnitForLLM,
crawl,
type IngestResult,
type AnalysisUnit,
} from '../recon/code-ingest.js';
import { estimateTokens } from '../orchestration/context-pack.js';
// ---------------------------------------------------------------------------
// Temp repo fixture
// ---------------------------------------------------------------------------
const APP_PY = `from flask import Flask, request
from util import fetch_url, validate_token, format_name
app = Flask(__name__)
@app.route("/proxy")
def proxy_view():
token = request.args.get("token")
if not validate_token(token):
return "no", 403
data = fetch_url(request.args.get("target"))
return data
@app.route("/health")
def health_view():
return "ok"
`;
const UTIL_PY = `import requests
def fetch_url(url):
resp = requests.get(url)
return resp.text
def validate_token(tok):
return tok == "secret"
def format_name(x):
return str(x).strip().title()
`;
// A file that must be skipped because it lives in a test dir.
const TEST_PY = `def test_something():
assert True
`;
let repoRoot: string;
let result: IngestResult;
beforeAll(() => {
repoRoot = mkdtempSync(join(tmpdir(), 'code-ingest-'));
writeFileSync(join(repoRoot, 'app.py'), APP_PY);
writeFileSync(join(repoRoot, 'util.py'), UTIL_PY);
// a dir that must be excluded by default
mkdirSync(join(repoRoot, 'tests'));
writeFileSync(join(repoRoot, 'tests', 'test_app.py'), TEST_PY);
// a non-python file that must be ignored by includeExts
writeFileSync(join(repoRoot, 'README.md'), '# not python\n');
const config = createPythonIngestConfig(repoRoot);
result = ingestRepository(config);
});
afterAll(() => {
rmSync(repoRoot, { recursive: true, force: true });
});
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function unitByName(name: string): AnalysisUnit {
const u = result.analysisUnits.find((au) => au.block.name === name);
if (!u) throw new Error(`no analysis unit for ${name}`);
return u;
}
// ---------------------------------------------------------------------------
// crawl
// ---------------------------------------------------------------------------
describe('crawl', () => {
it('keeps .py files and skips test dirs + non-python files', () => {
const files = crawl(createPythonIngestConfig(repoRoot));
const names = files.map((f) => f.split(/[\\/]/).pop());
expect(names).toContain('app.py');
expect(names).toContain('util.py');
// excluded by default test-dir rule
expect(names).not.toContain('test_app.py');
// excluded by includeExts (.py only)
expect(names).not.toContain('README.md');
});
});
// ---------------------------------------------------------------------------
// parseFile
// ---------------------------------------------------------------------------
describe('parseFile', () => {
it('extracts function blocks with names, params, and plausible line ranges', () => {
const blocks = parseFile('util.py', UTIL_PY);
const names = blocks.map((b) => b.name);
expect(names).toContain('fetch_url');
expect(names).toContain('validate_token');
expect(names).toContain('format_name');
const fetch = blocks.find((b) => b.name === 'fetch_url')!;
expect(fetch.kind).toBe('function');
expect(fetch.params).toEqual(['url']);
// def is on line 4 of UTIL_PY (1-indexed), body spans a couple lines
expect(fetch.lineStart).toBe(4);
expect(fetch.lineEnd).toBeGreaterThanOrEqual(fetch.lineStart);
expect(fetch.lineEnd).toBeLessThan(fetch.lineStart + 4);
expect(fetch.body).toContain('requests.get(url)');
// stable id shape
expect(fetch.id).toBe(`util.py::fetch_url@${fetch.lineStart}`);
});
it('captures decorators immediately preceding a def', () => {
const blocks = parseFile('app.py', APP_PY);
const proxy = blocks.find((b) => b.name === 'proxy_view')!;
expect(proxy.decorators.join('\n')).toContain('@app.route("/proxy")');
});
it('labels methods nested under a class as "method"', () => {
const src = [
'class Service:',
' def handle(self, req):',
' return req',
'',
'def free_function(a):',
' return a',
].join('\n');
const blocks = parseFile('svc.py', src);
const cls = blocks.find((b) => b.name === 'Service')!;
const method = blocks.find((b) => b.name === 'handle')!;
const fn = blocks.find((b) => b.name === 'free_function')!;
expect(cls.kind).toBe('class');
expect(method.kind).toBe('method');
// self dropped from params
expect(method.params).toEqual(['req']);
expect(fn.kind).toBe('function');
});
it('recovers a multiline def signature instead of dropping the function', () => {
// A security-relevant function whose params span lines must NOT vanish.
const src = [
'def transfer(',
' src_account,',
' dst_account,',
' amount=compute_fee(0),',
'):',
' do_wire(src_account, dst_account, amount)',
' return True',
'',
'def after(x):',
' return x',
].join('\n');
const blocks = parseFile('bank.py', src);
const transfer = blocks.find((b) => b.name === 'transfer');
expect(transfer, 'transfer() must be captured, not dropped').toBeTruthy();
// params gathered across lines, defaults/nested-call stripped to bare names
expect(transfer!.params).toEqual(['src_account', 'dst_account', 'amount']);
expect(transfer!.lineStart).toBe(1);
// body reaches past the ")" line to the real statements
expect(transfer!.body).toContain('do_wire(');
// the following def is still parsed (signature lines were skipped, not eaten)
expect(blocks.find((b) => b.name === 'after')).toBeTruthy();
});
});
// ---------------------------------------------------------------------------
// entry points + call graph
// ---------------------------------------------------------------------------
describe('findEntryPoints', () => {
it('detects @app.route handlers as entry points', () => {
const blocks = parseFile('app.py', APP_PY);
const eps = findEntryPoints(blocks);
const proxy = blocks.find((b) => b.name === 'proxy_view')!;
const health = blocks.find((b) => b.name === 'health_view')!;
expect(eps).toContain(proxy.id);
expect(eps).toContain(health.id);
});
});
describe('buildCallGraph', () => {
it('resolves an intra-repo call from a handler to a helper', () => {
const blocks = [
...parseFile('app.py', APP_PY),
...parseFile('util.py', UTIL_PY),
];
const graph = buildCallGraph(blocks);
const proxy = blocks.find((b) => b.name === 'proxy_view')!;
const fetch = blocks.find((b) => b.name === 'fetch_url')!;
// handler -> helper edge
expect(graph[proxy.id].callees).toContain(fetch.id);
// inverse: helper knows its caller
expect(graph[fetch.id].callers).toContain(proxy.id);
});
});
// ---------------------------------------------------------------------------
// full pipeline: classification, reachability, priority
// ---------------------------------------------------------------------------
describe('ingestRepository classification', () => {
it('classifies a @route handler as exposed_externally', () => {
expect(unitByName('proxy_view').exposure).toBe('exposed_externally');
expect(result.entryPoints).toContain(unitByName('proxy_view').block.id);
});
it('classifies fetch_url (outbound request sink) as attack_surface', () => {
const fetch = unitByName('fetch_url');
expect(fetch.exposure).toBe('attack_surface');
// evidence recorded
expect(fetch.riskSignals.some((s) => s.includes('requests.get'))).toBe(true);
// SSRF/IDOR shape: url param + outbound request
expect(fetch.riskSignals.some((s) => s.startsWith('ssrf-idor:'))).toBe(true);
});
it('classifies validate_token as security_control (by name)', () => {
expect(unitByName('validate_token').exposure).toBe('security_control');
});
it('classifies format_name as neutral', () => {
expect(unitByName('format_name').exposure).toBe('neutral');
});
it('reaches fetch_url from the proxy_view entry point', () => {
const fetch = unitByName('fetch_url');
expect(fetch.reachable).toBe(true);
expect(fetch.reachDepth).toBe(1);
// representative reachability path starts at the handler
expect(fetch.reachabilityPaths[0][0]).toBe(unitByName('proxy_view').block.id);
});
it('stats account for every block and every exposure class', () => {
const { stats } = result;
expect(stats.files).toBe(2); // app.py + util.py (tests dir excluded)
expect(stats.blocks).toBe(result.analysisUnits.length);
const sum =
stats.exposed_externally +
stats.exposed_internally +
stats.attack_surface +
stats.security_control +
stats.neutral;
expect(sum).toBe(stats.blocks);
});
});
describe('ingestRepository priority ordering', () => {
it('ranks the exposed handler above the neutral util', () => {
const handler = unitByName('proxy_view');
const neutral = unitByName('format_name');
expect(handler.priority).toBeGreaterThan(neutral.priority);
});
it('emits analysisUnits sorted by priority descending', () => {
const priorities = result.analysisUnits.map((u) => u.priority);
const sorted = [...priorities].sort((a, b) => b - a);
expect(priorities).toEqual(sorted);
});
it('never assigns a negative priority (unreachable just misses the bonus)', () => {
for (const u of result.analysisUnits) {
expect(u.priority).toBeGreaterThanOrEqual(0);
}
});
});
// ---------------------------------------------------------------------------
// composition with context-pack
// ---------------------------------------------------------------------------
describe('packAnalysisUnits', () => {
it('respects a small token budget and drops the lowest-priority units', () => {
const units = result.analysisUnits;
// Budget large enough for the top unit but not all of them.
const topCost = estimateTokens(formatUnitForLLM(units[0]));
const budget = topCost + 5;
const packed = packAnalysisUnits(units, budget);
expect(packed.tokensUsed).toBeLessThanOrEqual(budget);
expect(packed.includedUnits.length).toBeGreaterThan(0);
expect(packed.droppedUnits.length).toBeGreaterThan(0);
// nothing vanishes off the books
expect(packed.includedUnits.length + packed.droppedUnits.length).toBe(
units.length,
);
// the highest-priority unit is included, a lower one is dropped
expect(packed.includedUnits[0].block.name).toBe(units[0].block.name);
});
it('preserves the incoming (security) priority order among included units', () => {
const units = result.analysisUnits;
const packed = packAnalysisUnits(units, 100_000); // everything fits
const includedNames = packed.includedUnits.map((u) => u.block.name);
const originalNames = units.map((u) => u.block.name);
// same order, all included
expect(includedNames).toEqual(originalNames);
expect(packed.droppedUnits).toHaveLength(0);
});
it('formatUnitForLLM renders path:lines, exposure, risk signals, and body', () => {
const fetch = unitByName('fetch_url');
const text = formatUnitForLLM(fetch);
expect(text).toContain(`util.py:${fetch.block.lineStart}-${fetch.block.lineEnd}`);
expect(text).toContain('exposure: attack_surface');
expect(text).toContain('risk signals:');
expect(text).toContain('requests.get(url)'); // body included
});
});
+150
View File
@@ -0,0 +1,150 @@
/**
* context-pack tests
*
* Covers the token-budgeted, repo-map + relevance packer that replaced the old
* blind head-slice truncation:
* - packContext respects the token budget
* - the REPO MAP header is ALWAYS present
* - higher-relevance files are packed before lower ones when budget is tight
* - droppedFiles is reported when the bundle exceeds budget (no silent loss)
* - parseLooseSource splits a delimited blob and handles a plain blob
*/
import { describe, it, expect } from 'vitest';
import {
parseLooseSource,
packContext,
estimateTokens,
type SourceBundle,
} from '../orchestration/context-pack.js';
describe('estimateTokens', () => {
it('approximates chars / 4', () => {
expect(estimateTokens('')).toBe(0);
expect(estimateTokens('abcd')).toBe(1);
expect(estimateTokens('a'.repeat(40))).toBe(10);
});
});
describe('parseLooseSource', () => {
it('splits a delimited blob into files by delimiter path', () => {
const blob = [
'=== src/auth.py ===',
'def login(u, p):',
' return check(u, p)',
'--- FILE: src/util.js ---',
'function noop() {}',
'## handlers/route.ts',
'export const route = 1;',
].join('\n');
const bundle = parseLooseSource(blob);
const paths = bundle.map(f => f.path);
expect(paths).toContain('src/auth.py');
expect(paths).toContain('src/util.js');
expect(paths).toContain('handlers/route.ts');
const auth = bundle.find(f => f.path === 'src/auth.py');
expect(auth?.content).toContain('def login');
expect(auth?.content).not.toContain('function noop'); // stays in util.js
});
it('returns a single pseudo-file for a plain blob with no delimiters', () => {
const blob = 'just some prose about a struct\nwith no file markers at all';
const bundle = parseLooseSource(blob);
expect(bundle).toHaveLength(1);
expect(bundle[0].path).toBe('(source)');
expect(bundle[0].content).toBe(blob);
});
it('does not split on ordinary horizontal rules or comment sentences', () => {
// "---" with no path, and a "// a comment." that is not a file path
const blob = 'line one\n---\nline two\n// this is a normal comment here\nline three';
const bundle = parseLooseSource(blob);
expect(bundle).toHaveLength(1);
expect(bundle[0].path).toBe('(source)');
});
it('preserves preamble content before the first delimiter', () => {
const blob = 'README notes here\n=== a/b.c ===\nint main(){}';
const bundle = parseLooseSource(blob);
expect(bundle.map(f => f.path)).toContain('(preamble)');
expect(bundle.find(f => f.path === '(preamble)')?.content).toContain('README notes');
});
});
describe('packContext', () => {
it('always emits a REPO MAP header, even for a huge repo on a tiny budget', () => {
const bundle: SourceBundle = Array.from({ length: 200 }, (_, i) => ({
path: `src/file_${i}.ts`,
content: 'x'.repeat(500),
}));
const packed = packContext(bundle, { tokenBudget: 300 });
expect(packed.text).toContain('=== REPO MAP');
// Huge inventory on a tiny budget must cap the map, not silently drop it.
expect(packed.text).toMatch(/more files/);
});
it('respects the token budget (uses at most ~the budget)', () => {
const bundle: SourceBundle = Array.from({ length: 30 }, (_, i) => ({
path: `mod_${i}.ts`,
content: 'const x = 1;\n'.repeat(200),
}));
const budget = 2000;
const packed = packContext(bundle, { tokenBudget: budget });
// tokensUsed is the reported accounting; assert it never overruns the budget.
expect(packed.tokensUsed).toBeLessThanOrEqual(budget);
// and the real assembled text is in the same ballpark (small map/newline slack)
expect(estimateTokens(packed.text)).toBeLessThanOrEqual(budget + 50);
});
it('reports droppedFiles when the bundle exceeds the budget', () => {
const bundle: SourceBundle = Array.from({ length: 20 }, (_, i) => ({
path: `big_${i}.ts`,
content: 'y'.repeat(4000), // ~1000 tokens each — cannot all fit in 1500
}));
const packed = packContext(bundle, { tokenBudget: 1500 });
expect(packed.droppedFiles.length).toBeGreaterThan(0);
// included + dropped account for every file (nothing vanishes off the books)
expect(packed.includedFiles.length + packed.droppedFiles.length).toBe(bundle.length);
});
it('packs higher-relevance files before lower ones when budget is tight', () => {
const bundle: SourceBundle = [
{ path: 'unrelated/readme.md', content: 'z'.repeat(3000) },
{ path: 'src/deserialize_handler.py', content: 'pickle.loads(data)\n'.repeat(150) },
{ path: 'misc/notes.txt', content: 'w'.repeat(3000) },
];
// Budget fits roughly one file body below the map. The security-relevant /
// objective-matching file must win.
const packed = packContext(bundle, {
tokenBudget: 1200,
objective: 'find an unsafe deserialize sink using pickle',
});
expect(packed.includedFiles).toContain('src/deserialize_handler.py');
// the winner appears in the body, not just the map
const bodyStart = packed.text.indexOf('=== FILE:');
expect(packed.text.slice(bodyStart)).toContain('src/deserialize_handler.py');
expect(packed.droppedFiles.length).toBeGreaterThan(0);
});
it('head+tail elides a single oversized file instead of dropping it', () => {
const marker = 'UNIQUE_TAIL_MARKER_9f3';
const huge = 'A'.repeat(20000) + marker;
const bundle: SourceBundle = [{ path: 'one/giant.c', content: huge }];
const packed = packContext(bundle, { tokenBudget: 1000 });
expect(packed.includedFiles).toContain('one/giant.c');
expect(packed.text).toContain('…[middle elided');
expect(packed.text).toContain('A'); // head present
expect(packed.text).toContain(marker); // tail present
});
it('echoes the budget and never drops the map for a plain single-file bundle', () => {
const bundle = parseLooseSource('int main() { return 0; }');
const packed = packContext(bundle, { tokenBudget: 5000 });
expect(packed.tokenBudget).toBe(5000);
expect(packed.text).toContain('=== REPO MAP');
expect(packed.includedFiles).toContain('(source)');
expect(packed.droppedFiles).toHaveLength(0);
});
});
+455
View File
@@ -0,0 +1,455 @@
/**
* T3MP3ST Core Tests
*
* Basic test suite for core functionality.
*/
import { describe, it, expect, beforeEach } from 'vitest';
import { Arsenal, BUILTIN_TOOLS, successResult, failResult, createToolContext } from '../arsenal/index.js';
import { FRONTIER_ARSENAL_MILESTONE, SAFE_COMMANDS, TOOL_ADAPTERS, adaptersForFamily, summarizeToolCatalog } from '../arsenal/catalog.js';
import { createKnowledgeBase, createEvasionEngine, CVE_DATABASE, MITRE_TECHNIQUES } from '../stubs/index.js';
import { AGENT_PROMPT_PACKS, FOREFRONT_PRESSURE_LANES, OPERATOR_RUNBOOKS, forefrontPressureForFamily, promptPacksForFamily, runbookForFamily } from '../resources/index.js';
import { OpGeneral } from '../general/index.js';
import { LLMBackbone } from '../llm/index.js';
describe('Arsenal', () => {
let arsenal: Arsenal;
beforeEach(() => {
arsenal = new Arsenal();
});
describe('Tool Registration', () => {
it('should register built-in tools', () => {
arsenal.registerMany(BUILTIN_TOOLS);
expect(arsenal.getAllTools().length).toBe(BUILTIN_TOOLS.length);
});
it('should get tool by name', () => {
arsenal.registerMany(BUILTIN_TOOLS);
const tool = arsenal.getTool('dns_lookup');
expect(tool).toBeDefined();
expect(tool?.name).toBe('dns_lookup');
});
it('should filter tools by category', () => {
arsenal.registerMany(BUILTIN_TOOLS);
const reconTools = arsenal.getToolsByCategory('recon');
expect(reconTools.length).toBeGreaterThan(0);
reconTools.forEach(tool => expect(tool.category).toBe('recon'));
});
it('should return undefined for unknown tool', () => {
const tool = arsenal.getTool('nonexistent');
expect(tool).toBeUndefined();
});
});
describe('Tool Execution', () => {
it('should execute dns_lookup tool (may fail if no network)', async () => {
arsenal.registerMany(BUILTIN_TOOLS);
const result = await arsenal.execute('dns_lookup', createToolContext(undefined, { domain: 'example.com' }));
// DNS lookup now does real network requests - may succeed or fail depending on network
expect(result.output || result.error).toContain('example.com');
});
it('should execute base64_decode tool', async () => {
arsenal.registerMany(BUILTIN_TOOLS);
const result = await arsenal.execute('base64_decode', createToolContext(undefined, { data: 'SGVsbG8gV29ybGQ=' }));
expect(result.success).toBe(true);
expect(result.output).toContain('Hello World');
});
it('should execute jwt_decode tool', async () => {
arsenal.registerMany(BUILTIN_TOOLS);
// Valid JWT token structure (header.payload.signature)
const testJwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IlRlc3QiLCJpYXQiOjE1MTYyMzkwMjJ9.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c';
const result = await arsenal.execute('jwt_decode', createToolContext(undefined, { token: testJwt }));
expect(result.success).toBe(true);
expect(result.output).toContain('HS256');
});
it('should throw for unknown tool', async () => {
await expect(arsenal.execute('nonexistent', createToolContext())).rejects.toThrow();
});
});
describe('Helper Functions', () => {
it('should create success result', () => {
const result = successResult('Test output');
expect(result.success).toBe(true);
expect(result.output).toBe('Test output');
});
it('should create fail result', () => {
const result = failResult('Test error');
expect(result.success).toBe(false);
expect(result.error).toBe('Test error');
});
});
});
describe('KnowledgeBase', () => {
const kb = createKnowledgeBase();
describe('CVE Database', () => {
it('should have CVE entries', () => {
expect(CVE_DATABASE.length).toBeGreaterThan(0);
});
it('should query CVEs by ID', () => {
const result = kb.query({ type: 'cve', query: 'Log4Shell' });
expect(result.results.length).toBeGreaterThan(0);
});
it('should get CVE by exact ID', () => {
const cve = kb.getCVE('CVE-2021-44228');
expect(cve).toBeDefined();
expect(cve?.description).toContain('Log4Shell');
});
it('should filter critical CVEs by CVSS', () => {
const critical = kb.getCriticalCVEs(9.0);
expect(critical.length).toBeGreaterThan(0);
critical.forEach(c => expect(c.cvss).toBeGreaterThanOrEqual(9.0));
});
});
describe('MITRE Techniques', () => {
it('should have MITRE techniques', () => {
expect(MITRE_TECHNIQUES.length).toBeGreaterThan(0);
});
it('should get technique by ID', () => {
const technique = kb.getTechnique('T1190');
expect(technique).toBeDefined();
expect(technique?.name).toBe('Exploit Public-Facing Application');
});
it('should filter techniques by tactic', () => {
const initialAccess = kb.getTechniquesByTactic('initial-access');
expect(initialAccess.length).toBeGreaterThan(0);
initialAccess.forEach(t => expect(t.tactic).toBe('initial-access'));
});
});
describe('Pattern Matching', () => {
it('should detect SQL injection patterns', () => {
const matches = kb.matchPatterns("SELECT * FROM users WHERE id = '1' OR '1'='1'");
expect(matches.some(m => m.type === 'sqli')).toBe(true);
});
it('should detect XSS patterns', () => {
const matches = kb.matchPatterns('<script>alert(1)</script>');
expect(matches.some(m => m.type === 'xss')).toBe(true);
});
it('should detect path traversal patterns', () => {
const matches = kb.matchPatterns('../../etc/passwd');
expect(matches.some(m => m.type === 'path-traversal')).toBe(true);
});
});
});
describe('EvasionEngine', () => {
const evasion = createEvasionEngine();
describe('Encoding', () => {
it('should encode to base64', () => {
const result = evasion.encode('hello', 'base64');
expect(result.encoded).toBe('aGVsbG8=');
expect(result.encoding).toBe('base64');
});
it('should encode to hex', () => {
const result = evasion.encode('AB', 'hex');
expect(result.encoded).toBe('4142');
});
it('should encode to URL', () => {
const result = evasion.encode('<script>', 'url');
expect(result.encoded).toBe('%3Cscript%3E');
});
it('should handle unknown encoding', () => {
const result = evasion.encode('test', 'unknown');
expect(result.encoded).toBe('test');
expect(result.encoding).toBe('none');
});
});
describe('Multi-layer Encoding', () => {
it('should apply multiple encodings', () => {
const result = evasion.multiEncode('test', ['base64', 'url']);
expect(result.encoding).toBe('base64+url');
expect(result.encoded).not.toBe('test');
});
});
describe('Polymorphic Generation', () => {
it('should generate multiple payload variants', () => {
const variants = evasion.generatePolymorphic('<script>alert(1)</script>');
expect(variants.length).toBeGreaterThan(3);
// All variants should be different
const uniqueEncoded = new Set(variants.map(v => v.encoded));
expect(uniqueEncoded.size).toBe(variants.length);
});
});
describe('WAF Bypass', () => {
it('should generate WAF bypass variants', () => {
const variants = evasion.getWAFBypassVariants('SELECT * FROM users');
expect(variants.length).toBeGreaterThan(5);
expect(variants).toContain('SELECT * FROM users'); // Original included
});
});
describe('Code Obfuscation', () => {
it('should obfuscate strings', () => {
const result = evasion.obfuscate("var x = 'hello'", 'string_split');
expect(result.obfuscated).toContain('String.fromCharCode');
});
});
});
describe('Operator Contracts', () => {
it('should expose family-specific prompt packs with evidence contracts', () => {
expect(AGENT_PROMPT_PACKS.length).toBeGreaterThanOrEqual(5);
for (const pack of AGENT_PROMPT_PACKS) {
expect(pack.roleFrame.length).toBeGreaterThan(20);
expect(pack.evidenceContract.length).toBeGreaterThanOrEqual(3);
expect(pack.escalationRules.length).toBeGreaterThan(0);
}
});
it('should return the AI boundary prompt pack for AI red-team missions', () => {
const packs = promptPacksForFamily('ai_red_team');
expect(packs.map(pack => pack.id)).toContain('prompt-ai-boundary-cartographer');
});
it('should expose operator runbooks with phase risks and exit criteria', () => {
expect(OPERATOR_RUNBOOKS.length).toBeGreaterThanOrEqual(5);
for (const runbook of OPERATOR_RUNBOOKS) {
expect(runbook.operatorPromise.length).toBeGreaterThan(30);
expect(runbook.phases.length).toBeGreaterThanOrEqual(3);
for (const phase of runbook.phases) {
expect(phase.riskIfSkipped.length).toBeGreaterThan(20);
expect(phase.exitCriteria.length).toBeGreaterThan(0);
}
}
});
it('should return a runbook for the core routed mission families', () => {
expect(runbookForFamily('ai_red_team')?.title).toContain('AI Agent Boundary');
expect(runbookForFamily('web_api')?.title).toContain('Web/API');
expect(runbookForFamily('code_supply_chain')?.title).toContain('Repository Trust');
});
it('should expose forefront pressure lanes with defensive conversion artifacts', () => {
expect(FOREFRONT_PRESSURE_LANES.length).toBeGreaterThanOrEqual(6);
for (const lane of FOREFRONT_PRESSURE_LANES) {
expect(lane.frontierSignal.length).toBeGreaterThan(40);
expect(lane.pressureQuestion).toMatch(/\?$/);
expect(lane.operatorMove.length).toBeGreaterThan(40);
expect(lane.defensiveArtifact.length).toBeGreaterThan(30);
expect(lane.containment.length).toBeGreaterThan(30);
expect(lane.recommendedResources.length).toBeGreaterThan(0);
}
});
it('should return forefront lanes by mission family', () => {
expect(forefrontPressureForFamily('ai_red_team').map(lane => lane.id)).toContain('frontier-browser-tool-privilege');
expect(forefrontPressureForFamily('agent_warfare').map(lane => lane.id)).toContain('frontier-agent-command-injection');
});
});
describe('Tool Adapter Catalog', () => {
it('should expose a broad command-ready and catalog-only arsenal', () => {
const summary = summarizeToolCatalog();
expect(TOOL_ADAPTERS.length).toBeGreaterThanOrEqual(30);
expect(summary.commandReady).toBeGreaterThanOrEqual(20);
expect(summary.catalogOnly).toBeGreaterThanOrEqual(2);
expect(summary.unmodeled).toBe(false);
expect(TOOL_ADAPTERS.length).toBeLessThanOrEqual(FRONTIER_ARSENAL_MILESTONE);
expect(SAFE_COMMANDS).toContain('nuclei');
expect(SAFE_COMMANDS).toContain('semgrep');
expect(SAFE_COMMANDS).not.toContain('msfconsole');
});
it('should give core mission families first-class adapter coverage', () => {
expect(adaptersForFamily('web_api').map(adapter => adapter.id)).toEqual(expect.arrayContaining(['nmap', 'nuclei', 'ffuf', 'katana']));
expect(adaptersForFamily('code_supply_chain').map(adapter => adapter.id)).toEqual(expect.arrayContaining(['semgrep', 'gitleaks', 'trivy', 'syft']));
expect(adaptersForFamily('ai_red_team').map(adapter => adapter.id)).toEqual(expect.arrayContaining(['garak', 'promptfoo']));
expect(adaptersForFamily('smart_contract').map(adapter => adapter.id)).toEqual(expect.arrayContaining(['slither', 'mythril', 'foundry-forge', 'solhint']));
expect(adaptersForFamily('crypto_secrets').map(adapter => adapter.id)).toEqual(expect.arrayContaining(['openssl', 'john', 'hashcat', 'foundry-cast']));
expect(adaptersForFamily('reverse_binary').map(adapter => adapter.id)).toEqual(expect.arrayContaining(['binwalk', 'radare2', 'apktool', 'jadx']));
});
});
describe('OpGeneral v2 Campaign Control', () => {
function generalWithResponse(payload: Record<string, unknown>): OpGeneral {
const llm = {
prompt: async () => `\`\`\`json\n${JSON.stringify(payload)}\n\`\`\``,
} as unknown as LLMBackbone;
return new OpGeneral(llm);
}
it('should enrich sparse plans with hunt lanes, work orders, evidence contracts, and review gates', async () => {
const general = generalWithResponse({
codename: 'IRON CURRENT',
summary: 'Authorized local web/API weird-machine hunt.',
targets: [{ address: 'http://127.0.0.1:3333/ui/', expectedType: 'web_application', priority: 1, rationale: 'Local T3MP3ST UI' }],
objectives: [{ description: 'Find boundary failures in the local control plane', priority: 1, successCriteria: 'Evidence-backed hypotheses and retests', phase: 'reconnaissance' }],
operators: [{ archetype: 'scanner', count: 2, deployPhase: 'reconnaissance', briefing: 'Probe local UI/API edges safely' }],
roe: { scope: ['http://127.0.0.1:3333/ui/'], exclusions: ['external targets'], maxDetections: 1, destructiveAllowed: false, requireApproval: [] },
complexity: 'moderate',
rationale: 'Start local, preserve evidence, and require receipts before active external action.',
});
const plan = await general.planOperation({ objective: 'Run an authorized local web/API zero-day hunt against http://127.0.0.1:3333/ui/' });
const review = general.reviewPlan(plan);
expect(plan.missionFamily).toBe('web_api');
expect(plan.huntLanes.length).toBeGreaterThan(0);
expect(plan.workOrders.length).toBeGreaterThanOrEqual(2);
expect(plan.evidenceContract.requiredArtifacts.length).toBeGreaterThan(2);
expect(plan.authorityReceipts.map(receipt => receipt.action)).toContain('mission_execution');
expect(plan.missionGate.status).not.toBe('hold');
expect(review.score).toBeGreaterThanOrEqual(70);
});
it('should hold the gate when a directive has no explicit target or scope', async () => {
const general = generalWithResponse({
codename: 'VAGUE THUNDER',
summary: 'Interesting but under-scoped operation.',
objectives: [{ description: 'Hunt something somewhere', priority: 1, successCriteria: 'Unknown', phase: 'reconnaissance' }],
operators: [{ archetype: 'analyst', count: 1, deployPhase: 'reconnaissance', briefing: 'Clarify scope' }],
roe: { scope: [], exclusions: [], maxDetections: 0, destructiveAllowed: false, requireApproval: [] },
complexity: 'high',
rationale: 'Scope is not explicit enough for execution.',
});
const plan = await general.planOperation({ objective: 'Find weird machines wherever they may be' });
const review = general.reviewPlan(plan);
expect(review.status).toBe('hold');
expect(review.blockers.join(' ')).toMatch(/target|scope/i);
expect(plan.missionGate.status).toBe('hold');
});
it('should preserve specialist multiplicity and expose work-order assignments for execution', async () => {
const general = generalWithResponse({
codename: 'DOUBLE SCANNER',
summary: 'Authorized local scanner-heavy review.',
targets: [{ address: 'localhost', expectedType: 'web_application', priority: 1, rationale: 'Local lab' }],
objectives: [{ description: 'Compare two scanner routes', priority: 1, successCriteria: 'Two independent evidence paths', phase: 'reconnaissance' }],
operators: [
{ archetype: 'scanner', count: 2, deployPhase: 'reconnaissance', briefing: 'Route A and Route B scanner comparison' },
{ archetype: 'analyst', count: 1, deployPhase: 'actions_on_objectives', briefing: 'Synthesize proof and falsifiers' },
],
roe: { scope: ['localhost'], exclusions: ['external targets'], maxDetections: 1, destructiveAllowed: false, requireApproval: [] },
complexity: 'moderate',
rationale: 'Multiplicity matters for independent route comparison.',
});
const plan = await general.planOperation({ objective: 'Authorized local route comparison against localhost' });
const execConfig = general.executePlan(plan);
expect(execConfig.operators.filter(archetype => archetype === 'scanner').length).toBeGreaterThanOrEqual(2);
expect(execConfig.operatorAssignments.some(assignment => assignment.workOrderIds.length > 0)).toBe(true);
expect(execConfig.workOrders.every(order => order.falsifier && order.retest)).toBe(true);
});
});
describe('Wedged-dispatch timeout backstop', () => {
// A fake AgentLoop whose run() never settles — simulates a truly-hung LLM call
// (the wedge symptom: operator pinned in `executing` forever).
const makeWedgingLoop = () =>
({ run: () => new Promise(() => { /* never resolves */ }) } as unknown as import('../agent/index.js').AgentLoop);
it('OperatorAgent.abortActiveTask returns a wedged operator to idle so it can take new work', async () => {
const { createOperator } = await import('../operators/index.js');
const { LLMBackbone } = await import('../llm/index.js');
const { Arsenal } = await import('../arsenal/index.js');
const op = createOperator('Wedge-1', 'recon', undefined, new LLMBackbone({ provider: 'mock', model: 'mock-model' }));
op.attachArsenal(new Arsenal(), makeWedgingLoop());
const task = {
id: 't-wedge', missionId: 'm1', name: 'recon', description: 'scan example.com',
phase: 'reconnaissance' as any, operatorType: 'recon' as const, status: 'pending' as const,
priority: 1, dependencies: [], createdAt: Date.now(),
};
// Fire the task but DON'T await — the promise never settles (wedge).
void op.assignTask(task as any);
// Let the microtasks run so assignTask advances to 'executing'.
await Promise.resolve();
await Promise.resolve();
expect(op.status).toBe('executing');
expect(op.isAvailable()).toBe(false);
// Backstop reset.
op.abortActiveTask('test timeout');
expect(op.status).toBe('idle');
expect(op.isAvailable()).toBe(true);
expect(op.state.currentTask).toBeNull();
expect(op.getSummary().failedTasks).toBe(1);
});
it('a wedged mission reaches phase advancement once the per-dispatch backstop fires', async () => {
// Tiny backstop so the test is fast and deterministic.
const prev = process.env.T3MP3ST_TASK_TIMEOUT_MS;
process.env.T3MP3ST_TASK_TIMEOUT_MS = '30';
try {
// Re-import with the env override in place so resolveTaskTimeoutMs() reads it.
const mod = await import('../index.js');
const command = new mod.TempestCommand({
name: 'Wedge Op',
llm: { provider: 'mock', model: 'mock-model' },
});
// A target + a recon operator whose agent loop wedges forever.
command.targetEnv.addTarget({ name: 'example.com', address: 'example.com', type: 'web_application', zone: 'external' });
const op = command.spawnOperator('Recon-Wedge', 'recon');
op.attachArsenal(command.arsenal, makeWedgingLoop());
let phaseAdvanced = false;
command.on('mission:phase_changed', () => { phaseAdvanced = true; });
command.start();
// Poll (real timers): the 1s tick loop must dispatch, wedge, then reap the
// dispatch via the backstop and advance past RECON.
const deadline = Date.now() + 8000;
while (Date.now() < deadline && !phaseAdvanced) {
await new Promise(r => setTimeout(r, 100));
}
command.stop();
expect(phaseAdvanced).toBe(true);
// The wedged operator was reset back to idle (or re-tasked in a later phase),
// never left stuck in 'executing' with no current task.
const stuck = command.cell.getAllOperators().some(
o => (o.status === 'executing' || o.status === 'tasked') && !o.state.currentTask
);
expect(stuck).toBe(false);
} finally {
if (prev === undefined) delete process.env.T3MP3ST_TASK_TIMEOUT_MS;
else process.env.T3MP3ST_TASK_TIMEOUT_MS = prev;
}
}, 15000);
});
describe('Codex account provider', () => {
it('should expose Codex as an API-keyless planning backend', () => {
const llm = new LLMBackbone({
provider: 'codex',
model: 'codex-default',
});
expect(llm.getProvider()).toBe('codex');
expect(llm.getModel()).toBe('codex-default');
expect(llm.validateConfig()).toEqual({ valid: true });
});
});
+75
View File
@@ -0,0 +1,75 @@
/**
* ingest ceiling tests — the bounded-ingest hardening must FIRE only on pathological input
* and leave a normal ingest byte-identical (no truncated flag, no behavior change).
*/
import { describe, it, expect, afterAll } from 'vitest';
import { mkdtempSync, writeFileSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { ingestRepository, crawl, type IngestConfig } from '../recon/code-ingest.js';
function makeRepo(nFiles: number): string {
const dir = mkdtempSync(join(tmpdir(), 't3mp3st-ingest-limits-'));
for (let i = 0; i < nFiles; i++) {
writeFileSync(join(dir, `mod_${i}.py`), `def f_${i}(x):\n return x + ${i}\n`);
}
return dir;
}
describe('ingest ceilings (bounded ingest, non-regressing)', () => {
const dirs: string[] = [];
const cfg = (repoRoot: string, extra: Partial<IngestConfig> = {}): IngestConfig => ({
repoRoot, includeExts: ['.py'], excludeGlobs: [], ...extra,
});
afterAll(() => { for (const d of dirs) { try { rmSync(d, { recursive: true, force: true }); } catch { /* noop */ } } });
it('crawl caps at maxFiles and ingest flags stats.truncated', () => {
const dir = makeRepo(20); dirs.push(dir);
expect(crawl(cfg(dir, { maxFiles: 5 })).length).toBe(5);
const res = ingestRepository(cfg(dir, { maxFiles: 5 }));
expect(res.stats.files).toBe(5);
expect(res.stats.truncated).toBe(true);
});
it('ingest caps at maxTotalBytes and flags stats.truncated', () => {
const dir = makeRepo(20); dirs.push(dir);
const res = ingestRepository(cfg(dir, { maxTotalBytes: 50 })); // ~30 bytes/file → breaks after ~2
expect(res.stats.truncated).toBe(true);
expect(res.stats.blocks).toBeGreaterThan(0);
});
it('a NORMAL ingest under the ceilings is unchanged — no truncated key', () => {
const dir = makeRepo(10); dirs.push(dir);
const res = ingestRepository(cfg(dir, { maxFiles: 50000, maxTotalBytes: 1_000_000_000 }));
expect(res.stats.files).toBe(10);
expect(res.stats.blocks).toBe(10);
expect(res.stats.truncated).toBeUndefined(); // shape identical to pre-hardening
});
it('no ceilings configured = fully unbounded (back-compat)', () => {
const dir = makeRepo(8); dirs.push(dir);
const res = ingestRepository(cfg(dir)); // no maxFiles / maxTotalBytes at all
expect(res.stats.files).toBe(8);
expect(res.stats.truncated).toBeUndefined();
});
it('an empty / non-Python repo yields 0 files (the ONLY case the whitebox guard throws on)', () => {
const dir = mkdtempSync(join(tmpdir(), 't3mp3st-ingest-empty-')); dirs.push(dir);
writeFileSync(join(dir, 'README.md'), '# not python\n');
const res = ingestRepository(cfg(dir));
expect(res.stats.files).toBe(0);
expect(res.analysisUnits.length).toBe(0);
});
it('a Python repo with only module-level code has files>0 but 0 blocks (whitebox guard must NOT throw)', () => {
// A valid script (imports + top-level statements, no def/class) is legitimate input. The
// whitebox empty-guard gates on stats.files===0, so this files>0 case proceeds — it must
// NOT regress into a 500 (regression sweep finding #2).
const dir = mkdtempSync(join(tmpdir(), 't3mp3st-ingest-modlevel-')); dirs.push(dir);
writeFileSync(join(dir, 'app.py'), 'import os\nx = 1\nprint(os.getcwd(), x)\n');
const res = ingestRepository(cfg(dir));
expect(res.stats.files).toBe(1); // there IS source
expect(res.analysisUnits.length).toBe(0); // but 0 extractable def/class blocks
});
});
+55
View File
@@ -0,0 +1,55 @@
import { describe, it, expect } from 'vitest';
import { ARCHETYPE_PROFILES } from '../operators/index.js';
import { BUILTIN_TOOLS, EXTERNAL_TOOLS } from '../arsenal/index.js';
import { OPERATOR_SYSTEM_PROMPTS } from '../prompts/index.js';
// ─────────────────────────────────────────────────────────────────────────────
// NO PHANTOM TOOLS — the "advertised = wired" guard for the arsenal.
//
// An operator must never advertise a tool the harness can't call. Phantom names
// (metasploit/sqlmap/mimikatz, or a bare "nmap" when the real tool is "nmap_scan")
// burn the agent's iterations on calls that 404. This test makes that fail the
// build by construction — the same discipline as test:no-fitting, applied to tools.
// ─────────────────────────────────────────────────────────────────────────────
const REGISTERED = new Set([...BUILTIN_TOOLS, ...EXTERNAL_TOOLS].map((t) => t.name));
describe('no phantom tools (advertised = wired)', () => {
it('the arsenal registers a non-trivial, unique set of callable tools', () => {
const all = [...BUILTIN_TOOLS, ...EXTERNAL_TOOLS].map((t) => t.name);
expect(all.length).toBeGreaterThanOrEqual(20);
expect(new Set(all).size).toBe(all.length); // no duplicate tool names
});
it('every archetype defaultTool is a REGISTERED, callable tool', () => {
const phantoms: string[] = [];
for (const [archetype, profile] of Object.entries(ARCHETYPE_PROFILES)) {
for (const tool of profile.defaultTools) {
if (!REGISTERED.has(tool)) phantoms.push(`${archetype} → "${tool}"`);
}
}
expect(
phantoms,
`phantom tools advertised by archetypes but not registered in the arsenal: ${phantoms.join(', ')}`,
).toEqual([]);
});
it('operator system prompts reference no phantom binaries (advertise only what is wired)', () => {
// Classic phantoms we do NOT wrap. Backtick-bare forms (`nmap`) distinguish the phantom
// bare binary from the real registered tool (nmap_scan, which is `nmap_scan`).
const PHANTOMS = ['metasploit', 'mimikatz', 'sqlmap', 'bloodhound', 'crackmapexec', 'nikto', 'wpscan', 'psexec', '`nmap`', '`nuclei`', '`burp`'];
const hits: string[] = [];
for (const [archetype, prompt] of Object.entries(OPERATOR_SYSTEM_PROMPTS)) {
const low = String(prompt).toLowerCase();
for (const p of PHANTOMS) if (low.includes(p.toLowerCase())) hits.push(`${archetype}: ${p}`);
}
expect(hits, `phantom binaries named in operator prompts: ${hits.join(', ')}`).toEqual([]);
});
it('every registered tool has a real handler (no metadata-only stubs)', () => {
const noHandler = [...BUILTIN_TOOLS, ...EXTERNAL_TOOLS]
.filter((t) => typeof t.handler !== 'function')
.map((t) => t.name);
expect(noHandler, `registered tools missing an executable handler: ${noHandler.join(', ')}`).toEqual([]);
});
});
+44
View File
@@ -0,0 +1,44 @@
/**
* redactCredential — the API/output redactor that keeps raw harvested secrets from ever
* leaving the process (external-audit P0: /api/mission/findings previously returned raw
* Credential.secret). Pins the invariant so a refactor can't silently re-expose secrets.
*/
import { describe, it, expect } from 'vitest';
import { redactCredential } from '../evidence/index.js';
import type { Credential } from '../types/index.js';
describe('redactCredential — raw secrets never leave the process', () => {
const cred: Credential = {
id: 'cred-1',
type: 'password',
username: 'admin',
secret: 'sup3rs3cr3t!',
domain: 'example.com',
targetId: 't-1',
source: 'harvest',
discoveredAt: 1,
validatedAt: 2,
privilegeLevel: 'admin',
};
it('strips the raw secret entirely — no key, no value anywhere in the output', () => {
const r = redactCredential(cred);
expect('secret' in r).toBe(false);
expect(JSON.stringify(r)).not.toContain('sup3rs3cr3t');
});
it('keeps the non-sensitive metadata + a secretCaptured flag', () => {
const r = redactCredential(cred);
expect(r.id).toBe('cred-1');
expect(r.type).toBe('password');
expect(r.username).toBe('admin');
expect(r.domain).toBe('example.com');
expect(r.privilegeLevel).toBe('admin');
expect(r.secretCaptured).toBe(true);
});
it('secretCaptured is false when no secret was captured', () => {
expect(redactCredential({ ...cred, secret: '' }).secretCaptured).toBe(false);
});
});
+68
View File
@@ -0,0 +1,68 @@
import { describe, it, expect } from 'vitest';
import { createOperator } from '../operators/index.js';
import type { Finding } from '../types/index.js';
// ─────────────────────────────────────────────────────────────────────────────
// HONESTY SPINE — LOAD-BEARING IN THE LIVE PATH.
//
// The provenance gate (gateLiveFinding) must run where findings are CREATED, not as
// an opt-in verifyFinding(id) call nothing invokes. This test pins that invariant to
// the operator's recordFinding chokepoint — the same path executeTask() uses to record
// every agent finding. A model-asserted finding (no tool evidence) must be recorded
// UNVERIFIED; a tool-backed finding must verify. If this regresses, a model can assert a
// "critical" into the record for free again — the exact failure the whole project prevents.
// ─────────────────────────────────────────────────────────────────────────────
function baseFinding(over: Partial<Finding>): Finding {
return {
id: '',
title: 'x',
description: 'x',
severity: 'high',
targetId: 't',
operatorId: 'o',
phase: 'exploitation',
evidence: [],
discoveredAt: Date.now(),
...over,
} as Finding;
}
describe('honesty spine is load-bearing at finding creation (operator.recordFinding)', () => {
it('a model-asserted finding (no tool evidence) is recorded UNVERIFIED with gate reasons', () => {
const op = createOperator('test-recon', 'recon');
let blocked: { reasons: string[] } | undefined;
op.on('finding:gate-blocked', (e) => { blocked = e; });
const f = baseFinding({ title: 'model says critical', severity: 'critical', evidence: [] });
op.recordFinding(f);
expect(f.verifyGate?.passed).toBe(false);
expect(f.verifiedAt).toBeUndefined();
expect(f.verifyGate?.provenance).toBe('none');
expect(blocked, 'a gate-blocked event must fire for an unbacked finding').toBeDefined();
expect(blocked!.reasons.join(' ')).toMatch(/provenance|evidence/i);
});
it('a tool-backed finding (real tool-output evidence) verifies', () => {
const op = createOperator('test-recon2', 'recon');
const f = baseFinding({
title: 'nmap found open port',
severity: 'high',
evidence: [{ type: 'output', content: 'PORT 22/tcp open ssh\n...real nmap output...', timestamp: Date.now() }],
});
op.recordFinding(f);
expect(f.verifyGate?.passed).toBe(true);
expect(f.verifyGate?.provenance).toBe('tool');
expect(f.verifiedAt).toBeTypeOf('number');
});
it('getVerified-style filter only surfaces gate-passed findings', () => {
const op = createOperator('test-recon3', 'recon');
op.recordFinding(baseFinding({ title: 'prose claim', evidence: [] }));
op.recordFinding(baseFinding({ title: 'tool claim', evidence: [{ type: 'output', content: 'x'.repeat(20), timestamp: Date.now() }] }));
const verified = op.getFindings().filter((f) => f.verifyGate?.passed);
expect(verified.map((f) => f.title)).toEqual(['tool claim']);
});
});
+108
View File
@@ -0,0 +1,108 @@
import { describe, it, expect } from 'vitest';
import {
ExploitEngine,
ScannerOrchestrator,
BrowserAutomation,
BenchmarkRunner,
PersistenceController,
SwarmController,
CloudSecurityEngine,
ProtocolHandler,
WorkflowOrchestrator,
WorkflowBuilder,
} from '../stubs/index.js';
// ─────────────────────────────────────────────────────────────────────────────
// STUB HONESTY — placeholder engines must never fabricate success.
//
// Several "advanced modules" are interface-only stubs (see src/stubs/index.ts).
// The launch risk is a stub that RETURNS something that looks like a real result:
// a { success: true }, a deployed implant, an "Executed X" node result. A caller
// (or the UI) then believes work happened that never did. This test locks in the
// honest contract: stub methods return not-implemented / failure shapes, never a
// fabricated success or real-looking data.
// ─────────────────────────────────────────────────────────────────────────────
describe('stub honesty (no fabricated success)', () => {
it('ExploitEngine.exploit reports failure, not a fake exploited target', async () => {
const result = await new ExploitEngine().exploit('10.0.0.1', {
name: 'stub',
targetService: 'http',
});
expect(result.success).toBe(false);
expect(String(result.output || '')).toMatch(/not implemented/i);
});
it('ScannerOrchestrator.scan returns no fabricated findings', async () => {
const result = await new ScannerOrchestrator().scan('10.0.0.1');
expect(result.findings).toEqual([]);
});
it('BrowserAutomation.navigate does not fake a loaded page', async () => {
const state = await new BrowserAutomation().navigate('http://example.com');
// No real navigation happened: no live URL, an honest "Not implemented" title.
expect(state.url).toBe('');
expect(state.title).toMatch(/not implemented/i);
});
it('BenchmarkRunner.run does not fabricate a passing score', async () => {
const metrics = await new BenchmarkRunner().run([]);
expect(metrics.score).toBe(0);
});
it('PersistenceController.deploy returns an honest failure, not a fake implant', async () => {
const result = await new PersistenceController().deploy('10.0.0.1', {
type: 'scheduled_task',
location: 'stub',
});
const asRecord = result as unknown as Record<string, unknown>;
// Must NOT look like a real deployed implant (no fabricated id/implant object).
expect(asRecord.id).toBeUndefined();
expect(asRecord.implant).toBeUndefined();
// And it must never claim success.
expect(asRecord.success).not.toBe(true);
expect(result.success).toBe(false);
expect(result.error).toMatch(/not implemented/i);
});
it('SwarmController.initialize spawns no fabricated agents', async () => {
const agents = await new SwarmController().initialize({
agentCount: 5,
behavior: 'exploration',
});
expect(agents).toEqual([]);
});
it('CloudSecurityEngine.scan discovers no fabricated resources', async () => {
const resources = await new CloudSecurityEngine().scan({ provider: 'aws' });
expect(resources).toEqual([]);
});
it('ProtocolHandler.http does not fabricate a real HTTP response', async () => {
const response = await new ProtocolHandler().http({ method: 'GET', url: 'http://example.com' });
expect(response.status).toBe(0);
expect(response.body).toMatch(/not implemented/i);
});
it('WorkflowOrchestrator.execute never reports fabricated node successes', async () => {
const workflow = new WorkflowBuilder()
.addNode({ id: 'a', type: 'recon', action: { type: 'noop', params: {} } })
.addNode({ id: 'b', type: 'scan', action: { type: 'noop', params: {} } })
.addEdge({ from: 'a', to: 'b' })
.build();
const report = await new WorkflowOrchestrator(undefined).execute(workflow);
// The topological walk is real, but NO node actually executes.
expect(report.results.length).toBe(2);
for (const nodeResult of report.results) {
expect(nodeResult.success).toBe(false);
expect(nodeResult.notExecuted).toBe(true);
// No fabricated "Executed ..." claim.
expect(String(nodeResult.output || '')).not.toMatch(/^Executed /);
}
// The overall execution must NOT be reported as completed.
expect(report.execution.status).not.toBe('completed');
expect(report.execution.status).toBe('failed');
});
});
+112
View File
@@ -0,0 +1,112 @@
/**
* whitebox path-containment tests (B-03)
*
* resolveContainedRepoPath is the security chokepoint for the white-box path: it
* turns an operator-supplied repoPath into a canonical, contained absolute path
* or throws RepoPathError. These tests pin the containment invariants so a future
* refactor can't silently reopen the arbitrary-directory-read hole:
* - accepts real dirs inside the allowed root (root itself + nested)
* - rejects dirs outside the root, '..' escapes, and symlinks that escape
* - rejects non-existent paths, files (non-dirs), and empty/non-string input
*
* The root is pinned via T3MP3ST_REPO_ROOT to a realpath'd temp dir so the test
* is independent of the operator's real home.
*/
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import { mkdtempSync, writeFileSync, mkdirSync, rmSync, symlinkSync, realpathSync } from 'fs';
import { statSync } from 'node:fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { resolveContainedRepoPath, RepoPathError } from '../recon/whitebox.js';
// Mock node:fs (what whitebox.ts imports) so we can force statSync to throw AFTER realpath
// succeeds — the TOCTOU race path. Every other export keeps its real implementation, and
// statSync itself defaults to real (vi.fn wrapping the original), overridden per-test below.
vi.mock('node:fs', async (importOriginal) => {
const actual = await importOriginal<typeof import('node:fs')>();
return { ...actual, statSync: vi.fn(actual.statSync) };
});
describe('resolveContainedRepoPath (B-03 path containment)', () => {
let base: string; // realpath'd temp base holding both root/ and outside/
let root: string; // the allowed root (set as T3MP3ST_REPO_ROOT)
let outside: string; // a sibling dir OUTSIDE the root
let prevRoot: string | undefined;
beforeAll(() => {
prevRoot = process.env.T3MP3ST_REPO_ROOT;
// realpath the temp base up-front: on macOS /tmp -> /private/tmp is a symlink,
// which would otherwise make our own fixtures look like they escape the root.
base = realpathSync(mkdtempSync(join(tmpdir(), 't3mp3st-contain-')));
root = join(base, 'root');
outside = join(base, 'outside');
mkdirSync(join(root, 'repo', 'sub'), { recursive: true });
mkdirSync(outside, { recursive: true });
writeFileSync(join(root, 'afile.txt'), 'x');
process.env.T3MP3ST_REPO_ROOT = root;
});
afterAll(() => {
if (prevRoot === undefined) delete process.env.T3MP3ST_REPO_ROOT;
else process.env.T3MP3ST_REPO_ROOT = prevRoot;
try { rmSync(base, { recursive: true, force: true }); } catch { /* noop */ }
});
it('accepts a directory inside the allowed root (returns canonical path)', () => {
expect(resolveContainedRepoPath(join(root, 'repo'))).toBe(realpathSync(join(root, 'repo')));
});
it('accepts the root itself', () => {
expect(resolveContainedRepoPath(root)).toBe(realpathSync(root));
});
it('accepts a nested subdirectory', () => {
expect(resolveContainedRepoPath(join(root, 'repo', 'sub'))).toBe(realpathSync(join(root, 'repo', 'sub')));
});
it('rejects a directory OUTSIDE the root', () => {
expect(() => resolveContainedRepoPath(outside)).toThrow(RepoPathError);
});
it('rejects a traversal that escapes the root', () => {
expect(() => resolveContainedRepoPath(join(root, '..', 'outside'))).toThrow(/outside the allowed root/);
});
it('rejects a non-existent path', () => {
expect(() => resolveContainedRepoPath(join(root, 'nope'))).toThrow(/does not resolve/);
});
it('rejects a file (not a directory)', () => {
expect(() => resolveContainedRepoPath(join(root, 'afile.txt'))).toThrow(/must be a directory/);
});
it('rejects empty / whitespace / non-string input', () => {
expect(() => resolveContainedRepoPath('')).toThrow(RepoPathError);
expect(() => resolveContainedRepoPath(' ')).toThrow(RepoPathError);
expect(() => resolveContainedRepoPath(undefined as unknown as string)).toThrow(RepoPathError);
expect(() => resolveContainedRepoPath(42 as unknown as string)).toThrow(RepoPathError);
});
it('rejects a symlink whose target escapes the root (realpath resolves it)', () => {
const link = join(root, 'escape-link');
try {
symlinkSync(outside, link);
} catch {
return; // symlink creation not permitted in this env — skip rather than fail
}
expect(() => resolveContainedRepoPath(link)).toThrow(RepoPathError);
});
it('normalizes a statSync failure (TOCTOU delete after realpath) to RepoPathError', () => {
// The dir realpath-resolves (exists) but statSync then throws — simulating a deletion in
// the race window between realpath and stat. The wrap must convert it to a RepoPathError
// ('not accessible'), which the API layer turns into a clean 400 (never an unhandled fs
// error / unhandled rejection). Guards the statSync-wrap catch added this session.
const target = join(root, 'repo'); // real, in-root dir — realpath succeeds
vi.mocked(statSync).mockImplementationOnce(() => {
throw Object.assign(new Error('ENOENT: no such file or directory'), { code: 'ENOENT' });
});
expect(() => resolveContainedRepoPath(target)).toThrow(/not accessible/);
});
});