{ "name": "AgentRanks coding test evidence", "url": "https://agentranks.uk/coding-tests.html", "dateModified": "2026-07-12", "currentScoreAudit": { "status": "provisional", "formula": "primary_score = strict_real_code_work first; legacy_ARscore remains proxy evidence only", "verdict": "70% on a strict benchmark is strong. Most models are still much lower under messy, long-horizon code work, so AgentRanks treats strict Real Code Work as the primary signal.", "issues": [ "Legacy ARscore mixes subjective agent architecture scores with model benchmark scores.", "DeepSWE-style long-horizon coding should outrank easier or more saturated benchmarks when available.", "It must include end-to-end terminal task completion, cost per completed task, retry count, latency, and tool failure rate.", "It must distinguish official, third-party, self-reported, preview, and inferred sources.", "It must penalize unavailable models or models with strong benchmark numbers but weak product access." ] }, "recommendedFormula": { "label": "AgentRanks Real Code Work Score", "formula": "0.40 * strict_real_code_work + 0.20 * SWE_bench_verified + 0.15 * terminal_bench + 0.10 * cost_to_finish + 0.10 * failure_risk_adjustment + 0.05 * openness_reproducibility", "note": "Use DeepSWE as the preferred primary input when available. SWE-bench, Terminal-Bench, cost, user failure votes, and openness are supporting signals." }, "benchmarkSources": [ { "name": "DeepSWE", "unit": "long-horizon real code work", "why": "Preferred primary AgentRanks coding benchmark because it targets original software-engineering tasks across real repositories and separates models under stricter pressure.", "source": "https://deepswe.datacurve.ai/", "trust": "primary strict benchmark" }, { "name": "SWE-bench Verified", "unit": "model or agent bug-fix success", "why": "Real GitHub issue resolution with tests; strong signal, but increasingly saturated and source quality varies by run.", "source": "https://www.swebench.com/", "trust": "official benchmark" }, { "name": "Terminal-Bench 2.0", "unit": "agent + model terminal completion", "why": "Closer to real coding-agent work because the system must inspect files, run commands, debug, and finish tasks.", "source": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "trust": "official benchmark" }, { "name": "SWE-bench / Vals AI", "unit": "model coding benchmark snapshot", "why": "Useful secondary snapshot with recent Fable 5, Opus 4.8, and GPT-5.5 figures, but should not replace primary benchmark links.", "source": "https://vals.ai/benchmarks/swebench", "trust": "third-party evaluation" } ], "externalSignals": [ { "stack": "DeepSWE strict coding benchmark", "type": "Real Code Work", "score": "Primary", "source": "https://deepswe.datacurve.ai/", "trust": "primary strict benchmark", "note": "Preferred AgentRanks primary input when available. A 70% score is strong; many models are far lower under strict real-code pressure." }, { "stack": "GPT-5.6 Sol / Terra / Luna", "type": "Preview claim", "score": "Unknown", "source": "https://openai.com/index/previewing-gpt-5-6-sol/", "trust": "unknown / preview", "note": "Unknown series for AgentRanks until public users can reproduce coding results. Apply a 50% hype haircut to any vendor or invite-only preview claim." }, { "stack": "Mythos-class claims", "type": "Persona / model-class claim", "score": "Unknown", "source": "https://agentranks.uk/coding-tests.html", "trust": "unknown / no reliable data", "note": "Mythos is not a guaranteed rank. Treat as an unverified frontier label unless community tests prove real-world stability." }, { "stack": "ChatGPT Codex + GPT-5.5", "type": "Terminal-Bench v2.1", "score": "83.4%", "source": "https://www.morphllm.com/best-ai-coding-agents-2026", "trust": "reported", "note": "Reported as leading Terminal-Bench v2.1 in June 2026 coverage; needs direct source verification before becoming an AgentRanks official score." }, { "stack": "Claude Code + Fable 5", "type": "Terminal-Bench v2.1", "score": "83.1%", "source": "https://www.morphllm.com/best-ai-coding-agents-2026", "trust": "reported / unavailable", "note": "Very strong reported score, but Fable 5 access was suspended by Anthropic on June 12, 2026." }, { "stack": "Claude Fable 5", "type": "SWE-bench Verified", "score": "95.0%", "source": "https://vals.ai/benchmarks/swebench", "trust": "third-party snapshot", "note": "Model-level coding signal, not an agent workflow score." }, { "stack": "Claude Opus 4.8", "type": "SWE-bench Verified", "score": "88.6%", "source": "https://vals.ai/benchmarks/swebench", "trust": "third-party snapshot", "note": "Useful model-level signal for Claude Code-style stacks." }, { "stack": "GPT-5.5", "type": "SWE-bench Verified", "score": "82.6%", "source": "https://vals.ai/benchmarks/swebench", "trust": "third-party snapshot", "note": "Useful model-level signal for Codex-style stacks." } ], "agentRanksTestPack": [ { "name": "Bug fix with hidden tests", "weight": 30, "task": "Patch a real failing issue in a small repo, add regression coverage, and pass hidden tests." }, { "name": "Terminal autonomy", "weight": 20, "task": "Inspect a repo, run commands, diagnose failures, and produce a working fix without manual file hints." }, { "name": "Feature build", "weight": 15, "task": "Implement a small UI/API feature from product requirements with tests and no unrelated churn." }, { "name": "Refactor safety", "weight": 15, "task": "Refactor a shared module while preserving behavior and avoiding over-broad edits." }, { "name": "Cost and latency", "weight": 10, "task": "Measure tokens, wall time, retries, and cost per accepted solution." }, { "name": "Maintainability review", "weight": 10, "task": "Score code clarity, test quality, minimalism, and ease of future modification." } ] }