59 lines
2.7 KiB
JSON
59 lines
2.7 KiB
JSON
{
|
|
"schema": "t3mp3st.bench.model-matrix-manifest/v1",
|
|
"id": "cybench-opus-4.7-vs-4.8-pass1",
|
|
"generated_at": "2026-07-20T00:00:00.000Z",
|
|
"corpus": "Cybench 15-task common committed subset",
|
|
"attempt_policy": "pass@1; one committed run per task",
|
|
"scoring_policy": "oracle-graded strict verdict (detected=true or score=1)",
|
|
"comparison_limits": "Historical committed runs on the same 15 task IDs, live-tools harness label, artifact schema, and pass@1 policy. Run dates and model versions differ; raw transcripts were stripped for operator privacy. Results compare complete systems and must not be interpreted as an isolated model-quality ranking.",
|
|
"tasks": [
|
|
"cybsvc_hackthebox_cyber_apocalypse_2024_misc_easy_unbreak",
|
|
"cybsvc_hackthebox_cyber_apocalypse_2024_pwn_very_easy_del",
|
|
"cybsvc_hackthebox_cyber_apocalypse_2024_web_easy_labyrint",
|
|
"cybsvc_hackthebox_cyber_apocalypse_2024_web_medium_lockta",
|
|
"cybsvc_hackthebox_cyber_apocalypse_2024_web_very_easy_fla",
|
|
"cybsvc_hkcert_ctf_ctf_challenges_ctf_2022_05_motp",
|
|
"cybsvc_hkcert_ctf_ctf_challenges_ctf_2022_22_back_to_the_",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2022_crypto_ezmaze",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2022_crypto_failproof",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2022_crypto_robust_cbc",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2023_crypto_noisier_crc",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2023_crypto_noisy_crc",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2023_forensics_eval_me",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2023_pwn_network_tools",
|
|
"cybsvc_project_sekai_ctf_sekaictf_2023_web_chunky"
|
|
],
|
|
"subjects": [
|
|
{
|
|
"id": "anthropic-claude-opus-4.7",
|
|
"model": "claude-opus-4-7",
|
|
"provider": "Anthropic",
|
|
"model_version_or_date": "4.7 / artifacts dated 2026-05",
|
|
"harness": "live-tools",
|
|
"artifact_schema": "t3mp3st.bench.cybench/v1",
|
|
"agent_runtime": "T3MP3ST single-agent ReAct",
|
|
"tool_access": "tool_backed",
|
|
"target_class": "ctf_range",
|
|
"run_mode": "single_agent",
|
|
"source_dir": "bench/cybench/results/archive-opus-4.7",
|
|
"file_prefix": "service-"
|
|
},
|
|
{
|
|
"id": "anthropic-claude-opus-4.8",
|
|
"model": "claude-opus-4-8",
|
|
"provider": "Anthropic",
|
|
"model_version_or_date": "4.8 / artifacts dated 2026-05",
|
|
"harness": "live-tools",
|
|
"artifact_schema": "t3mp3st.bench.cybench/v1",
|
|
"agent_runtime": "T3MP3ST single-agent ReAct",
|
|
"tool_access": "tool_backed",
|
|
"target_class": "ctf_range",
|
|
"run_mode": "single_agent",
|
|
"source_dir": "bench/cybench/results",
|
|
"file_prefix": "service-"
|
|
}
|
|
],
|
|
"output_json": "bench/model-matrix/cybench-opus-pass1.json",
|
|
"output_markdown": "bench/model-matrix/cybench-opus-pass1.md"
|
|
}
|