{ "generated": "2026-09-16", "count": 54, "benchmarks": [ { "id": "aa-omniscience", "name": "AA-Omniscience", "full_name": "AA-Omniscience: Knowledge and Hallucination Benchmark", "layer": "model", "released": "2025-11", "maintainer": "Artificial Analysis", "description": "6,000 short-answer factual questions generated from authoritative academic and industry sources across 42 topics in six domains (business, humanities, health, law, software engineering, science and engineering). An LLM grader labels each answer correct, partial, incorrect or not attempted; the Omniscience Index (-100 to 100) awards correct answers, subtracts incorrect ones and leaves abstentions neutral, so it measures knowledge calibration rather than raw recall. Accuracy and hallucination rate are reported separately and feed the Artificial Analysis Intelligence Index.", "description_zh": "6,000 道由权威学术与行业来源生成的短答事实题,覆盖商业、人文社科、健康、法律、软件工程、科学工程数学六大领域 42 个主题。LLM 评分器将答案标为正确、部分正确、错误或未作答;Omniscience Index(-100 至 100)对正确加分、错误扣分、弃答不计,衡量知识校准而非单纯记忆。准确率与幻觉率另行报告并计入 Artificial Analysis Intelligence Index。", "domains": [ "knowledge", "factuality" ], "metric": { "name": "Omniscience Index", "unit": "score", "higher_is_better": true }, "task_count": 6000, "status": "active", "contamination_risk": "low", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2511.13029", "website": "https://artificialanalysis.ai/evaluations/omniscience", "leaderboard": "https://artificialanalysis.ai/evaluations/omniscience", "dataset": "https://huggingface.co/datasets/ArtificialAnalysis/AA-Omniscience-Public", "code": null }, "notes": "arXiv 2511.13029 verified (Jackson, Keating, Cameron, Hill-Smith; posted 2025-11-17). Only a public subset of questions is released; the full 6,000 are held privately, hence low contamination risk. Index of 0 means as many correct as incorrect answers. All rows are Artificial Analysis's own runs (independent-evaluation); the current leaderboard shows rounded integer index values for 31 tracked models, while the paper reports one decimal. Methodology: artificialanalysis.ai/methodology/intelligence-benchmarking#aa-omniscience (grader GPT-5.6 Luna medium as of 2026-09).", "sota": { "system": "GPT-6 Astra (high)", "developer": "OpenAI", "value": 44, "date": "2026-09-03", "source": { "url": "https://artificialanalysis.ai/evaluations/omniscience", "kind": "independent-evaluation", "accessed": "2026-09-04" }, "conditions": { "reasoning_effort": "high", "notes": "Top of the leaderboard on access date (integer-rounded). AA's 2026-09-03 article attributes the jump over GPT-5.6 Sol to hallucination rate falling from 92% to 51% at max effort." } }, "result_count": 4 }, { "id": "agentbench", "name": "AgentBench", "full_name": "AgentBench: Evaluating LLMs as Agents", "layer": "agent", "released": "2023-08", "maintainer": "Tsinghua University KEG / Zhipu AI (Liu et al.)", "description": "Eight interactive environments (operating system shell, SQL database, knowledge graph, digital card game, lateral-thinking puzzles, ALFWorld household, WebShop, Mind2Web browsing) in which an LLM acts over multiple turns. Each environment has its own success metric and the overall score is a weighted average normalised across environments, so it measures breadth of agentic decision-making rather than one skill. An early standard for comparing API and open-weight models as agents.", "description_zh": "包含八个交互式环境(操作系统 shell、SQL 数据库、知识图谱、数字卡牌游戏、横向思维谜题、ALFWorld 家居、WebShop、Mind2Web 网页浏览),LLM 需在其中进行多轮行动。每个环境有自己的成功指标,总分为跨环境归一化后的加权平均,因此衡量的是智能体决策的广度而非单一技能。是早期比较 API 模型与开放权重模型智能体能力的标准之一。", "domains": [ "tool-use", "reasoning", "web", "code" ], "metric": { "name": "overall score (weighted average)", "unit": "score", "higher_is_better": true }, "task_count": null, "status": "retired", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2308.03688", "website": "https://github.com/THUDM/AgentBench", "leaderboard": "https://docs.google.com/spreadsheets/d/e/2PACX-1vRR3Wl7wsCgHpwUw1_eUXW_fptAPLL3FkhnW_rua0O1Ji_GIVrpTjY5LaKAhwO-WeARjnY_KNw0SYNJ/pubhtml", "dataset": "https://github.com/THUDM/AgentBench", "code": "https://github.com/THUDM/AgentBench" }, "notes": "The original v0.2 leaderboard stopped being updated after the paper era; in October 2025 the repository was repurposed as 'AgentBench FC', a function-calling variant on five containerised tasks with its own leaderboard, and scores are not comparable to the original overall score. Marked retired because the maintainers no longer accept results for the original benchmark.", "sota": { "system": "gpt-4 (0613)", "developer": "OpenAI", "value": 4.01, "date": "2023-08-07", "source": { "url": "https://arxiv.org/abs/2308.03688", "kind": "paper", "accessed": "2026-09-04" }, "conditions": { "notes": "Overall AgentBench score (Table 3); weighted average, not a percentage." } }, "result_count": 3 }, { "id": "aider-polyglot", "name": "Aider Polyglot", "full_name": "Aider polyglot coding benchmark", "layer": "model", "released": "2024-12", "maintainer": "Paul Gauthier (Aider)", "description": "225 of the hardest Exercism exercises in C++, Go, Java, JavaScript, Python and Rust, selected because at most 3 of 7 reference models solved them. The model runs inside the aider CLI, must edit the files using a specified edit format, and gets one retry with the failing test output; the score is the percentage of exercises whose unit tests pass after two attempts. A widely quoted measure of instruction-following code editing that also exposes edit-format compliance and cost per run.", "description_zh": "从 C++、Go、Java、JavaScript、Python、Rust 的 Exercism 练习中选出 7 个参考模型至多 3 个能解的最难 225 题。模型在 aider CLI 中以指定编辑格式修改文件,失败后可凭测试输出重试一次;得分为两次尝试内通过单元测试的题目比例。被广泛引用的指令跟随式代码编辑指标,同时暴露编辑格式合规性与运行成本。", "domains": [ "code", "instruction-following" ], "metric": { "name": "percent correct (pass@2)", "unit": "percent", "higher_is_better": true }, "task_count": 225, "status": "saturating", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": null, "website": "https://aider.chat/2024/12/21/polyglot.html", "leaderboard": "https://aider.chat/docs/leaderboards/", "dataset": "https://github.com/Aider-AI/polyglot-benchmark", "code": "https://github.com/Aider-AI/aider" }, "notes": "Introduced 2024-12-21 to replace aider's saturated 133-problem Python edit benchmark. Rows below come from the maintainer's leaderboard data file (aider/website/_data/polyglot_leaderboard.yml), which records run dates, pass_rate_2 and total cost. The official leaderboard page was last updated 2025-11-20 and has not added 2026 models; Exercism problems are public, so newer models may have seen them.", "sota": { "system": "gpt-5 (high)", "developer": "OpenAI", "value": 88.0, "date": "2025-08-23", "source": { "url": "https://aider.chat/docs/leaderboards/", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "scaffold": "aider", "reasoning_effort": "high", "pass_k": 2, "cost_usd_per_task": 0.129, "notes": "Top of the official leaderboard as of its 2025-11-20 update; diff edit format." } }, "result_count": 5 }, { "id": "aime-2025", "name": "AIME 2025", "full_name": "American Invitational Mathematics Examination 2025 (I and II)", "layer": "model", "released": "2025-02", "maintainer": "Mathematical Association of America (exam); evaluated by MathArena and model developers", "description": "The 30 problems from the 2025 AIME I (6 February 2025) and AIME II (12 February 2025), each with an integer answer from 000 to 999. Scored as the fraction of problems answered exactly, usually averaged over several samples (mean pass@1). Because the exam is written fresh each year, it was a clean test of competition-math reasoning for models trained before February 2025; scores with a code interpreter are not comparable to no-tools runs.", "description_zh": "收录 2025 年 AIME I(2025 年 2 月 6 日)与 AIME II(2025 年 2 月 12 日)共 30 道题,每题答案为 000 到 999 的整数。得分为精确答对题目的比例,通常对多次采样取平均(平均 pass@1)。由于试题每年全新编写,对 2025 年 2 月之前训练的模型而言是一次干净的竞赛数学推理测试;使用代码解释器的成绩与无工具运行不可比。", "domains": [ "math", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 30, "status": "saturated", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": null, "website": "https://artofproblemsolving.com/wiki/index.php/2025_AIME_I", "leaderboard": "https://matharena.ai/", "dataset": null, "code": "https://github.com/eth-sri/matharena" }, "notes": "Scores are almost always reported on I+II combined (30 problems); a single exam has 15. Problems and solutions have been public since February 2025, so models released later have likely seen them. Several frontier systems report 100% (with or without tools), so the exam no longer separates them; MathArena marks AIME 2025 as deprecated in favour of newer competitions.", "sota": { "system": "GPT-5.2 (high)", "developer": "OpenAI", "value": 100.0, "date": "2025-12-11", "source": { "url": "https://matharena.ai/", "kind": "independent-evaluation", "accessed": "2026-09-04" }, "conditions": { "tools": false, "pass_k": 1, "notes": "MathArena AIME 2025 table, 100.00% +/- 0.00 for both high and xhigh effort. Date is model release; MathArena marks the competition deprecated." } }, "result_count": 4 }, { "id": "androidworld", "name": "AndroidWorld", "full_name": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents", "layer": "agent", "released": "2024-05", "maintainer": "Google Research / DeepMind (Rawles, Clinckemaillie, Riva et al.)", "description": "116 hand-crafted tasks across 20 real Android apps, executed in a live Android emulator. Each task is parameterised and instantiated with random values (names, dates, amounts), so the suite yields millions of unique variants; dedicated set-up, success-check and tear-down logic inspect device state to produce a durable reward. Agents observe screenshots and/or the accessibility tree and emit touch, type and navigation actions; the score is the task success rate. It is the reference benchmark for mobile GUI agents and also hosts MiniWoB++ web tasks.", "description_zh": "116 个人工设计的任务,覆盖 20 个真实 Android 应用,在实时 Android 模拟器中执行。任务带参数并随机实例化(姓名、日期、金额等),可生成数百万个变体;专用的初始化、成功检查和清理逻辑读取设备状态给出可靠奖励。智能体观察截图和/或无障碍树,输出点击、输入和导航动作,得分为任务成功率。是移动端 GUI 智能体的参考基准,并兼容 MiniWoB++ 网页任务。", "domains": [ "computer-use", "multimodal", "tool-use" ], "metric": { "name": "task success rate", "unit": "percent", "higher_is_better": true }, "task_count": 116, "status": "saturated", "contamination_risk": "medium", "human_baseline": { "value": 80.0, "population": "human annotators, 3 trials (paper / leaderboard sheet)", "source": "https://arxiv.org/abs/2405.14573" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2405.14573", "website": "https://google-research.github.io/android_world/", "leaderboard": "https://docs.google.com/spreadsheets/d/1cchzP9dlTZ3WXQTfYNhh3avxoLipqHN75v1Tb86uhHo/edit?gid=0#gid=0", "dataset": "https://google-research.github.io/android_world/task_list.html", "code": "https://github.com/google-research/android_world" }, "notes": "The leaderboard is a Google Sheet of community-submitted, self-reported pass@1 numbers with no independent verification; several entries have been disputed or revised. Since 2024-11-18 the per-task step budget is about 2x the human completion time. Task templates and success checkers are public and the top agents now exceed the 80% human baseline, with a 100% entry in August 2026, so the benchmark no longer separates frontier systems.", "sota": { "system": "FluizAI agent (gpt-4o / gpt-5.6-sol)", "developer": "FluizAI", "value": 100.0, "date": "2026-08-01", "source": { "url": "https://docs.google.com/spreadsheets/d/1cchzP9dlTZ3WXQTfYNhh3avxoLipqHN75v1Tb86uhHo/edit?gid=0#gid=0", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "pass_k": 1, "scaffold": "FluizAI", "notes": "Rank 1 on the community sheet as of access date; screenshot + a11y tree; code not open-sourced. Sheet gives only 08/2026, 1st used. No independent verification." } }, "result_count": 5 }, { "id": "arc-agi-1", "name": "ARC-AGI-1", "full_name": "Abstraction and Reasoning Corpus for Artificial General Intelligence, version 1", "layer": "model", "released": "2019-11", "maintainer": "ARC Prize Foundation (created by Francois Chollet)", "description": "Grid-based visual puzzles: given a few input-output example pairs, the system must infer the transformation rule and produce the exact output grid for a new input. 400 public training and 400 public evaluation tasks plus 100-task semi-private and private sets; scored as the percentage of test outputs exactly correct within two attempts (pass@2). Built to measure skill acquisition on novel problems rather than memorized knowledge; it resisted LLM scaling until test-time reasoning arrived in late 2024.", "description_zh": "基于网格的视觉谜题:给定少量输入-输出示例对,系统需推断变换规则并为新输入生成精确的输出网格。包含 400 个公开训练任务和 400 个公开评估任务,以及各 100 个任务的半私有和私有集;得分为两次尝试内(pass@2)完全正确的测试输出百分比。旨在衡量对新问题的技能获取而非记忆知识;在 2024 年底测试时推理出现之前,它一直抵御着 LLM 规模扩展。", "domains": [ "reasoning" ], "metric": { "name": "accuracy (pass@2)", "unit": "percent", "higher_is_better": true }, "task_count": 400, "splits": [ { "name": "Semi-Private", "task_count": 100, "note": "Held-out set used by the official ARC Prize leaderboard for API-tested models." }, { "name": "Public Eval", "task_count": 400, "note": "Public evaluation set; may be in training data." } ], "status": "saturated", "contamination_risk": "medium", "human_baseline": { "value": 98.0, "population": "ARC Prize human panel (STEM graduates); average MTurk worker scored 77%", "source": "https://arcprize.org/leaderboard" }, "supersedes": null, "superseded_by": "arc-agi-2", "links": { "paper": "https://arxiv.org/abs/1911.01547", "website": "https://arcprize.org/arc-agi/1", "leaderboard": "https://arcprize.org/leaderboard", "dataset": "https://github.com/fchollet/ARC-AGI", "code": "https://github.com/arcprize/arc-agi-benchmarking" }, "notes": "Official leaderboard scores are on the Semi-Private set and report cost per task alongside accuracy; only runs under $10k total are listed. The December 2024 o3-preview result (75.7% high-efficiency, 87.5% at 172x compute) was the first to approach the human panel; frontier models now score above 97%.", "sota": { "system": "GPT-6 Astra (XHigh)", "developer": "OpenAI", "value": 98.5, "date": "2026-09-02", "source": { "url": "https://arcprize.org/leaderboard", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "Semi-Private", "pass_k": 2, "cost_usd_per_task": 0.35, "notes": "Ties Claude Fable 5 (Max/XHigh, 98.5%, June 2026) at far lower cost; matches the 98% human panel." } }, "result_count": 6 }, { "id": "arc-agi-2", "name": "ARC-AGI-2", "full_name": "Abstraction and Reasoning Corpus for Artificial General Intelligence, version 2", "layer": "model", "released": "2025-03", "maintainer": "ARC Prize Foundation", "description": "Second-generation ARC grid puzzles designed to defeat brute-force search and test symbolic interpretation, compositional reasoning and context-dependent rule application. 1,000 public training tasks plus calibrated 120-task public, semi-private and private evaluation sets; every task was solved by at least two humans within two attempts. Scored as the percentage of test outputs exactly correct within two attempts, always reported together with cost per task. It launched with ARC Prize 2025 as the successor to ARC-AGI-1.", "description_zh": "第二代 ARC 网格谜题,旨在击败暴力搜索,并测试符号解释、组合推理和依赖上下文的规则应用。包含 1,000 个公开训练任务,以及经校准的各 120 个任务的公开、半私有和私有评估集;每个任务都至少被两名人类在两次尝试内解出。得分为两次尝试内完全正确的测试输出百分比,始终与每任务成本一并报告。作为 ARC-AGI-1 的继任者随 ARC Prize 2025 发布。", "domains": [ "reasoning" ], "metric": { "name": "accuracy (pass@2)", "unit": "percent", "higher_is_better": true }, "task_count": 120, "splits": [ { "name": "Semi-Private", "task_count": 120, "note": "Held-out set used by the official ARC Prize leaderboard for API-tested models." }, { "name": "Public Eval", "task_count": 120, "note": "Public evaluation set, statistically calibrated to match the private sets." } ], "status": "saturating", "contamination_risk": "medium", "human_baseline": { "value": 100.0, "population": "ARC Prize human panel (every task solved by at least two of 400+ general-public testers); 66% of individual attempts succeeded", "source": "https://arxiv.org/abs/2505.11831" }, "supersedes": "arc-agi-1", "superseded_by": "arc-agi-3", "links": { "paper": "https://arxiv.org/abs/2505.11831", "website": "https://arcprize.org/arc-agi/2", "leaderboard": "https://arcprize.org/leaderboard", "dataset": "https://github.com/arcprize/ARC-AGI-2", "code": "https://github.com/arcprize/arc-agi-benchmarking" }, "notes": "Announced 2025-03-24; the arXiv paper followed in May 2025. The ARC Prize grand-prize threshold is 85% on the private set under Kaggle compute limits, which no open solution has met. Frontier API models passed 90% on the Semi-Private set in mid-2026 at several dollars per task, so the benchmark is saturating for unconstrained systems while remaining open for efficient ones.", "sota": { "system": "GPT-6 Astra (Max)", "developer": "OpenAI", "value": 95.0, "date": "2026-09-02", "source": { "url": "https://arcprize.org/leaderboard", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "Semi-Private", "pass_k": 2, "cost_usd_per_task": 1.12, "notes": "ARC Prize verified; also republished by benchlm.ai and in OpenAI's GPT-6 Astra launch table (95.0%)." } }, "result_count": 5 }, { "id": "arc-agi-3", "name": "ARC-AGI-3", "full_name": "ARC-AGI-3: A New Challenge for Frontier Agentic Intelligence", "layer": "agent", "released": "2026-03", "maintainer": "ARC Prize Foundation", "description": "Interactive, turn-based game-like environments built only from Core Knowledge priors, with no instructions: an agent must explore, infer the goal, build a world model and plan. 25 public demo environments plus 55 semi-private (API-tested) and 55 fully private (competition) environments, each verified fully solvable by untrained humans. Scoring is efficiency-based against human action baselines, so 100% means solving every level as efficiently as humans. The first interactive generation of the ARC-AGI series.", "description_zh": "仅基于核心知识先验构建的交互式回合制类游戏环境,不提供任何说明:智能体必须探索、推断目标、构建世界模型并进行规划。包含 25 个公开演示环境、55 个半私有(API 测试)环境和 55 个完全私有(竞赛)环境,每个环境均经未受训人类验证可完全解决。评分基于相对人类行动基线的效率,100% 意味着以人类同等效率通关所有关卡。是 ARC-AGI 系列首个交互式版本。", "domains": [ "reasoning", "tool-use" ], "metric": { "name": "efficiency-weighted score", "unit": "percent", "higher_is_better": true }, "task_count": 135, "splits": [ { "name": "public-demo", "task_count": 25, "note": "Public demonstration environments; playable on arcprize.org." }, { "name": "semi-private", "task_count": 55, "note": "Held-out set used by ARC Prize to test models behind external APIs; the official leaderboard split." }, { "name": "fully-private", "task_count": 55, "note": "Competition-only set given to very few partners." } ], "status": "saturating", "contamination_risk": "low", "human_baseline": { "value": 100.0, "population": "untrained human test-takers (every environment solved by at least two of ten participants)", "source": "https://arxiv.org/abs/2603.24621" }, "supersedes": "arc-agi-2", "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2603.24621", "website": "https://arcprize.org/arc-agi/3", "leaderboard": "https://arcprize.org/leaderboard", "dataset": "https://arcprize.org/tasks?v=3", "code": "https://github.com/arcprize/arc-agi-3-benchmarking" }, "notes": "ARC Prize reports two harness conditions on the leaderboard: the Standard harness (model carries forward its own notes; provider-neutral) and the Provider Adapter harness (preserves opaque reasoning state and compacts context). Frontier models scored below 1% at the March 2026 launch; GPT-6 Astra reached 99.9% under the Provider Adapter harness in September 2026, so status is 'saturating' even though the Standard-harness score is 62.7%.", "sota": { "system": "GPT-6 Astra (high)", "developer": "OpenAI", "value": 99.9, "date": "2026-09-03", "source": { "url": "https://arcprize.org/blog/astra", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "semi-private", "scaffold": "Provider Adapter", "reasoning_effort": "high", "cost_usd_per_task": 342.13, "notes": "ARC Prize verified run; $18,817 total across 55 semi-private environments (cost per environment shown). Best Provider Adapter result; verified table shows 99.95%." } }, "result_count": 5 }, { "id": "automationbench", "name": "AutomationBench", "full_name": "AutomationBench: cross-application business workflow orchestration via REST APIs", "layer": "agent", "released": "2026-04", "maintainer": "Zapier (Daniel Shepard, Robin Salimans)", "description": "Business-workflow tasks across Sales, Marketing, Operations, Support, Finance and HR in which an agent, given one trigger message, must discover the right REST endpoints (BM25 search over ~500 endpoint schemas from 47 simulated SaaS apps), follow policy documents hidden in the environment, avoid decoy records, and mutate the state of a simulated company. Grading is deterministic end-state assertions, including negative ones; the official score is the strict fraction of tasks with every assertion passing on a private held-out set (600+ tasks); a 600-task public set is released for research.", "description_zh": "覆盖销售、市场、运营、客服、财务和 HR 的业务流程任务:智能体接收单条触发消息后,需在 47 个模拟 SaaS 应用的约 500 个 REST 端点中(BM25 检索)自行发现接口,遵循隐藏在环境中的政策文档,避开干扰记录,并修改模拟公司的状态。评分为确定性的最终状态断言(含负向断言);官方得分为所有断言全部通过的任务比例,在私有保留集(600+ 任务)上运行,另有 600 个公开任务供研究。", "domains": [ "tool-use", "general-assistant", "instruction-following" ], "metric": { "name": "task pass rate (all assertions)", "unit": "percent", "higher_is_better": true }, "task_count": 600, "splits": [ { "name": "private", "task_count": null, "note": "Held-out private set (600+ tasks, 100+ per domain) scored on the official zapier.com/benchmarks leaderboard; version 1.0.6 as of access date. Private tasks are periodically hardened." }, { "name": "public", "task_count": 600, "note": "Public research set in the GitHub repo (100 tasks per domain plus a 200-task 'simple' baseline domain that is not scored). Public scores run higher than private ones." } ], "status": "active", "contamination_risk": "low", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2604.18934", "website": "https://zapier.com/benchmarks", "leaderboard": "https://zapier.com/benchmarks", "dataset": "https://github.com/zapier/AutomationBench", "code": "https://github.com/zapier/AutomationBench" }, "notes": "Tasks were synthetically generated from the shape of real Zapier customer workflows and hardened; partial_credit (fraction of assertions) is reported as a diagnostic and RL reward but is not the headline score. Artificial Analysis runs an independent variant, AutomationBench-AA, whose headline is the share of objectives completed without guardrail violations, so AA numbers are on a different scale from Zapier's strict pass rate. Zapier's own leaderboard is a single run per model at its highest reasoning effort; the rank-1 Fable 5.1 entry uses an Opus 5 fallback for ~40% of tasks.", "sota": { "system": "Claude Opus 5 (max)", "developer": "Anthropic", "value": 50.3, "date": "2026-09-04", "source": { "url": "https://github.com/zapier/AutomationBench", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "public", "reasoning_effort": "max", "notes": "README table of pass rates on the 600-task public set; README undated, access date used. Public set is easier than the private leaderboard set." } }, "result_count": 6 }, { "id": "bbh", "name": "BIG-Bench Hard", "full_name": "BIG-Bench Hard (BBH): 23 challenging BIG-Bench tasks", "layer": "model", "released": "2022-10", "maintainer": "Google Research / Stanford (Suzgun et al.)", "description": "A suite of 23 BIG-Bench tasks (6,511 examples) on which 2022 language models scored below the average human rater: multi-step arithmetic, logical deduction, tracking shuffled objects, causal judgement, date understanding and others. Scored as exact-match accuracy averaged over tasks, usually 3-shot with chain-of-thought exemplars. It showed that CoT prompting unlocks emergent gains and became a standard reasoning check; frontier models now exceed the best human rater on most tasks.", "description_zh": "由 23 个 BIG-Bench 任务(6,511 个样例)组成的套件,2022 年的语言模型在这些任务上得分低于人类评分者平均水平:多步算术、逻辑推演、追踪被打乱的物体、因果判断、日期理解等。得分为各任务精确匹配准确率的平均值,通常采用 3-shot 并附带思维链示例。它证明了 CoT 提示能带来涌现式提升,并成为标准推理检验;前沿模型如今在多数任务上已超过最佳人类评分者。", "domains": [ "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 6511, "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 67.7, "population": "average BIG-Bench human rater (max rater: 94.4%)", "source": "https://arxiv.org/abs/2210.09261" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2210.09261", "website": "https://github.com/suzgunmirac/BIG-Bench-Hard", "leaderboard": null, "dataset": "https://huggingface.co/datasets/lukaemon/bbh", "code": "https://github.com/suzgunmirac/BIG-Bench-Hard" }, "notes": "Answer-only vs CoT prompting changes scores by 10-20 points; the Open LLM Leaderboard v2 reported a 3-shot answer-only variant, so numbers across harnesses are not comparable.", "sota": { "system": "DeepSeek-V4-Pro-Base", "developer": "DeepSeek", "value": 87.5, "date": "2026-04-22", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "shots": 3, "notes": "DeepSeek-V4 model card, base-model table (HF repo created 2026-04-22). Base model, exact match, 3-shot; V4-Flash-Base 86.9, V3.2-Base 87.6." } }, "result_count": 2 }, { "id": "benchcad", "name": "BenchCAD", "full_name": "BenchCAD: A Comprehensive, Industry-Standard Benchmark for Programmatic CAD", "layer": "model", "released": "2026-05", "maintainer": "BenchCAD team (Rice University, University of Virginia, UC San Diego; Zhang, Liu, Chen et al.)", "description": "17,900 execution-verified CadQuery programs across 106 industrial part families (gears, springs, drills, fittings), about half anchored to real ISO/DIN/EN/ASME/IEC specification tables. The prime task, Vision2Code, shows four orthographic renders and asks for a CadQuery program that is re-executed and scored by IoU-score (voxel IoU times execution rate); matched Vision QA, Code QA and Code Edit tasks isolate perception, parametric abstraction and program synthesis. An agentic variant adds a Python sandbox to render, measure and iterate. It is the yardstick vendors cite for AI-for-hardware.", "description_zh": "17,900 个经执行验证的 CadQuery 程序,覆盖 106 个工业零件族(齿轮、弹簧、钻头、管件等),约半数锚定真实 ISO/DIN/EN/ASME/IEC 规范表。主任务 Vision2Code 给出四视图渲染,要求生成 CadQuery 程序并重新执行,以 IoU-score(体素 IoU 乘执行率)评分;配套的 Vision QA、Code QA 和 Code Edit 任务分别隔离感知、参数抽象和程序合成能力。智能体变体提供 Python 沙箱用于渲染、测量和迭代。是各厂商引用的 AI for hardware 标尺。", "domains": [ "multimodal", "code", "reasoning" ], "metric": { "name": "Vision2Code IoU-score (voxel IoU x exec rate)", "unit": "score", "higher_is_better": true }, "task_count": 17900, "splits": [ { "name": "vision2code", "task_count": null, "note": "Prime task: image to CadQuery, no tools, scored on the full split by the official re-grader." }, { "name": "vision2code-tools", "task_count": null, "note": "Agentic setting with a Python sandbox ('IoU - tools'). Vendor numbers are self-reported voxel IoU, often on a random 1,000-file subset (Anthropic) or an unstated split (OpenAI), so not directly comparable to re-graded rows." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2605.10865", "website": "https://benchcad.com/", "leaderboard": "https://benchcad.com/leaderboard", "dataset": "https://huggingface.co/datasets/BenchCAD/BenchCAD", "code": "https://github.com/BenchCAD/BenchCAD-main" }, "notes": "Scores are 0-1 (IoU-score), not percentages. The official leaderboard (assets/data/leaderboard.json) marks self-reported vendor rows with an asterisk and re-grades submissions itself; OpenAI's GPT-6 Astra table quotes BenchCAD as 95.9% and notes Anthropic's Claude scores used three modifications to the eval, so launch-table percentages are on a different scale from the site's IoU-score. Dataset (CC-BY-4.0) is public on Hugging Face; a BenchCAD 2.0 with explicit parametric designs is in development at BenchCAD-org/benchcad-2.", "sota": { "system": "GPT-6 Astra", "developer": "OpenAI", "value": 0.959, "date": "2026-09-03", "source": { "url": "https://benchcad.com/leaderboard", "kind": "aggregator", "accessed": "2026-09-13" }, "conditions": { "split": "vision2code-tools", "notes": "Republished by the leaderboard, which labels it secondhand: OpenAI's own page was unreachable when that row was added. The split is inferred on that page from the same table's Anthropic figures (Fable 5.1 0.843, Opus 5 0.821, Fable 5 0.675), which match Anthropic's published with-tools numbers." } }, "result_count": 6 }, { "id": "bfcl", "name": "BFCL", "full_name": "Berkeley Function Calling Leaderboard (V4)", "layer": "model", "released": "2024-02", "maintainer": "UC Berkeley Gorilla team (Patil, Mao et al.)", "description": "Evaluates how accurately a model produces function (tool) calls from natural-language requests. V1 introduced abstract-syntax-tree matching of single, multiple and parallel calls in Python, Java, JavaScript and REST; V2 added user-contributed live functions; V3 added multi-turn, multi-step and state-tracking scenarios; V4 added web-search and memory agentic tasks and format-sensitivity checks. The overall score is an unweighted average of sub-category accuracies, reported with cost and latency. The de-facto standard for tool-calling ability.", "description_zh": "评估模型根据自然语言请求生成函数(工具)调用的准确度。V1 引入对 Python、Java、JavaScript 和 REST 中单个、多个及并行调用的抽象语法树匹配;V2 增加用户贡献的实时函数;V3 增加多轮、多步和状态跟踪场景;V4 增加网页搜索与记忆类智能体任务及格式敏感性检查。总分为各子类别准确率的无权重平均,并同时报告成本与延迟。是工具调用能力的事实标准。", "domains": [ "tool-use" ], "metric": { "name": "overall accuracy", "unit": "percent", "higher_is_better": true }, "task_count": null, "splits": [ { "name": "multi-turn", "note": "V3 multi-turn accuracy (base, missing function, missing parameter, long context)." }, { "name": "FC", "note": "Native function-calling mode; 'Prompt' rows use plain text generation instead." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://gorilla.cs.berkeley.edu/blogs/8_berkeley_function_calling_leaderboard.html", "website": "https://gorilla.cs.berkeley.edu/leaderboard.html", "leaderboard": "https://gorilla.cs.berkeley.edu/leaderboard.html", "dataset": "https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard", "code": "https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard" }, "notes": "The 'paper' link is the launch blog (February 2024); the ICML 2025 paper describes V4, and the related Gorilla paper is arXiv 2305.15334. Overall scores changed with each version (V1 to V4), so only compare rows from the same leaderboard version. The official leaderboard was last updated 2026-04-12 and models released after that are absent.", "sota": { "system": "Claude-Opus-4-5-20251101 (FC)", "developer": "Anthropic", "value": 77.47, "date": "2026-04-12", "source": { "url": "https://gorilla.cs.berkeley.edu/leaderboard.html", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "FC", "notes": "Rank 1 of 109 on BFCL V4 (multi-turn 68.38%); total run cost $86.55. Newer models (Opus 5, GPT-6 Astra) have not been added since the 2026-04-12 update." } }, "result_count": 5 }, { "id": "browsecomp", "name": "BrowseComp", "full_name": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "layer": "agent", "released": "2025-04", "maintainer": "OpenAI (Wei et al.)", "description": "1,266 human-written questions whose short, verifiable answers can only be found by persistently browsing many web pages and combining entangled constraints; human trainers verified that other people could not answer them within ten minutes and that then-current models failed. Scored as accuracy against the reference answer via a grader. It isolates deep web-research persistence rather than latent knowledge, so it separates browsing agents sharply.", "description_zh": "1,266 个人工编写的问题,其简短可验证的答案只能通过持续浏览大量网页并组合相互交织的约束才能找到;人类训练者验证了他人无法在十分钟内作答,且当时的模型均告失败。通过评分器对照参考答案计算准确率。它隔离考察深度网络研究的持久性而非潜在知识,因此能显著区分各浏览智能体。", "domains": [ "web", "tool-use", "factuality", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 1266, "status": "saturating", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2504.12516", "website": "https://openai.com/index/browsecomp/", "leaderboard": null, "dataset": "https://github.com/openai/simple-evals", "code": "https://github.com/openai/simple-evals" }, "notes": "No official leaderboard; numbers are vendor self-reports collected by aggregators, and browsing budget, search backend and parallel-sampling settings are rarely disclosed. Questions and encrypted answers are public in simple-evals. Top frontier scores are now clustered above 90%.", "sota": { "system": "GPT-5.6 Sol", "developer": "OpenAI", "value": 92.2, "date": "2026-09-03", "source": { "url": "https://benchlm.ai/benchmarks/browsecomp", "kind": "aggregator", "accessed": "2026-09-04" }, "conditions": { "notes": "Top of the aggregator table as of access date; developer-reported, scaffold and effort unspecified." } }, "result_count": 6 }, { "id": "cybench", "name": "Cybench", "full_name": "Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "layer": "agent", "released": "2024-08", "maintainer": "Stanford CRFM (Andy K. Zhang, Percy Liang et al.)", "description": "40 professional-level Capture-the-Flag tasks from four recent competitions (HackTheBox, SekaiCTF, Glacier, HKCert) across crypto, web, reverse engineering, forensics, misc and pwn. An agent works in a Kali Linux container, runs shell commands against local files and task servers, and submits a flag checked by an evaluator; tasks carry human first-solve times (minutes to 25 hours) and optional subtasks for graded partial credit. The headline metric is unguided % solved. It is the standard open cyber-offense eval used in AISI pre-deployment tests and frontier system cards.", "description_zh": "40 个专业级 CTF 任务,取自四个近期赛事(HackTheBox、SekaiCTF、Glacier、HKCert),涵盖 crypto、web、逆向、取证、misc 和 pwn。智能体在 Kali Linux 容器中执行 shell 命令,操作本地文件和任务服务器并提交 flag 由评测器校验;任务标注人类首解时间(数分钟至 25 小时),并可选子任务给出分级部分得分。主指标为无引导解题率。是 AISI 部署前测试和前沿模型系统卡采用的标准开源网络攻防能力评测。", "domains": [ "safety", "code", "tool-use" ], "metric": { "name": "unguided % solved", "unit": "percent", "higher_is_better": true }, "task_count": 40, "status": "saturated", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2408.08926", "website": "https://cybench.github.io", "leaderboard": "https://cybench.github.io/#leaderboard_title", "dataset": "https://github.com/andyzorigin/cybench", "code": "https://github.com/andyzorigin/cybench" }, "notes": "The official leaderboard (data/leaderboard.csv) mixes paper runs, HAL re-runs and numbers republished from vendor system cards, which often use a 35-39 task subset and pass@1 averaged over several trials; footnotes on the site give the provenance of each row. Two HAL rows (o3-mini, o1-mini) were adjusted downward after an Inspect port leaked an answer. Tasks are public 2023-2024 CTF challenges, so contamination is likely; the authors point to BountyBench as the successor for real-world tasks. Frontier models reach 96-100% on the subset, so it no longer discriminates.", "sota": { "system": "Claude Mythos Preview", "developer": "Anthropic", "value": 100.0, "date": "2026-04-07", "source": { "url": "https://cybench.github.io", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "notes": "Claude Mythos Preview System Card number republished on the official leaderboard (footnote 6): 35-task subset. Dated to the model's announcement day." } }, "result_count": 5 }, { "id": "deepsearchqa", "name": "DeepSearchQA", "full_name": "DeepSearchQA: Bridging the Comprehensiveness Gap for Deep Research Agents", "layer": "agent", "released": "2025-12", "maintainer": "Google DeepMind (Gupta, Chatterjee, Haas, Tao et al.); leaderboard run by Kaggle", "description": "900 hand-crafted, time-anchored web-research prompts across 17 fields whose answers are exhaustive sets rather than single facts (65% set answers). Each prompt is a causal chain of dependent look-ups, so agents must plan multi-step searches, collate fragments from many sources, de-duplicate entities and decide when to stop. Answers are judged for semantic set match by a fixed Gemini 2.5 Flash autorater; the primary metric is mean per-prompt F1, with fully-correct and fully-incorrect rates as diagnostics. It targets the comprehensiveness gap that single-answer benchmarks like BrowseComp miss.", "description_zh": "900 个人工构造、时间锚定的网页研究问题,覆盖 17 个领域,答案为完整集合而非单一事实(65% 为集合型答案)。每题是一条依赖链式检索,智能体需规划多步搜索、汇总多来源碎片、消歧去重并判断何时停止。由固定的 Gemini 2.5 Flash 自动评审判定语义集合匹配;主指标为逐题平均 F1(精确率对召回率),并报告完全正确、完全错误和多答比例作为诊断。针对 BrowseComp 等单答案基准忽视的全面性缺口。", "domains": [ "web", "research", "factuality", "tool-use" ], "metric": { "name": "F1 (mean over prompts)", "unit": "percent", "higher_is_better": true }, "task_count": 900, "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2601.20975", "website": "https://deepmind.google/research/evals/", "leaderboard": "https://www.kaggle.com/benchmarks/google/dsqa/leaderboard", "dataset": "https://huggingface.co/datasets/google/deepsearchqa", "code": "https://www.kaggle.com/code/andrewmingwang/deepsearchqa-starter-code" }, "notes": "Announced 2025-12-11 with the Gemini Deep Research agent; arXiv paper followed 2026-01-28. The official Kaggle leaderboard (last updated 2025-12-11, 13 systems) is independently run by Kaggle and reports F1 with 95% CIs; the dataset card warns that a different autorater or grading prompt gives statistically different scores, so vendor-reported numbers (Meta, Anthropic, Moonshot via aggregators) may not be comparable. Prompts and gold answers are public on Hugging Face/Kaggle, and ground truth can drift as web sources change.", "sota": { "system": "Claude Opus 5", "developer": "Anthropic", "value": 95.0, "date": "2026-09-03", "source": { "url": "https://benchlm.ai/benchmarks/deepsearchqa", "kind": "aggregator", "accessed": "2026-09-04" }, "conditions": { "tools": true, "notes": "Vendor-published number republished by aggregator (tied with Kimi K3 at 95.0); harness and autorater unspecified, so not comparable to the Kaggle-run board." } }, "result_count": 5 }, { "id": "drop", "name": "DROP", "full_name": "DROP: Discrete Reasoning Over the content of Paragraphs", "layer": "model", "released": "2019-03", "maintainer": "Allen Institute for AI / UC Irvine (Dua et al.)", "description": "96,000 crowd-sourced, adversarially created reading-comprehension questions over Wikipedia paragraphs that require discrete operations such as addition, counting, sorting and date arithmetic, with span, number or date answers. Scored by exact match and a numerically-aware F1 (the headline metric), usually 3-shot on the 9,535-question dev set. Once a hard test of grounded numerical reasoning; frontier models now approach expert human F1.", "description_zh": "96,000 个众包、对抗式构造的阅读理解问题,基于维基百科段落,需要加法、计数、排序和日期运算等离散操作,答案为文本片段、数字或日期。以精确匹配和数值感知的 F1(主要指标)评分,通常在 9,535 题的开发集上采用 3-shot。曾是有依据的数值推理的严苛测试;前沿模型如今已接近专家人类的 F1。", "domains": [ "reasoning", "knowledge" ], "metric": { "name": "F1", "unit": "percent", "higher_is_better": true }, "task_count": 9535, "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 96.4, "population": "expert annotators (paper Table 4, test F1)", "source": "https://arxiv.org/abs/1903.00161" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/1903.00161", "website": "https://github.com/allenai/allennlp-reading-comprehension", "leaderboard": null, "dataset": "https://huggingface.co/datasets/ucinlp/drop", "code": "https://github.com/allenai/allennlp-reading-comprehension" }, "notes": "Model-vendor reports use the dev split (test labels are hidden) and vary in shots and answer normalization; DROP was dropped from most vendor cards after 2025.", "sota": { "system": "o1", "developer": "OpenAI", "value": 90.2, "date": "2024-12-17", "source": { "url": "https://github.com/openai/simple-evals", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "shots": 3, "notes": "simple-evals README, DROP F1 3-shot; o3-high 89.8 (April 2025)." } }, "result_count": 3 }, { "id": "fieldworkarena", "name": "FieldWorkArena", "full_name": "FieldWorkArena: Agentic AI Benchmark for Real Field Work Tasks", "layer": "agent", "released": "2025-05", "maintainer": "Fujitsu Research (with Carnegie Mellon University)", "description": "886 tasks over on-site images, videos and documents captured in real factories, warehouses and retail stores (711 perception, 121 decision-making, 54 combination tasks), written from interviews with site workers and managers. An agent must extract information, detect safety or procedural violations and produce reports; answers are scored against ground truth with a weighted mix of exact and near-match scoring on a 0-1 scale (reported here as percent). It tests multimodal agents on physical-world field operations rather than digital environments.", "description_zh": "886 个任务,基于真实工厂、仓库和零售店现场采集的图像、视频和文档(711 个感知任务、121 个决策任务、54 个组合任务),依据对现场工人和管理者的访谈编写。智能体需提取信息、检测安全或流程违规并生成报告;答案对照真值,以精确匹配与近似匹配的加权组合按 0-1 评分(此处以百分比报告)。它测试多模态智能体在物理世界现场作业而非数字环境中的表现。", "domains": [ "multimodal", "general-assistant", "safety", "reasoning" ], "metric": { "name": "accuracy rate (total)", "unit": "percent", "higher_is_better": true }, "task_count": 886, "splits": [ { "name": "perception", "task_count": 711 }, { "name": "decision-making", "task_count": 121 }, { "name": "combination", "task_count": 54 } ], "status": "active", "contamination_risk": "low", "human_baseline": { "value": 74.0, "population": "human evaluators on a random sample of perception tasks (paper, score 0.74)", "source": "https://arxiv.org/abs/2505.19662" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2505.19662", "website": "https://en-documents.research.global.fujitsu.com/fieldworkarena/", "leaderboard": null, "dataset": "https://en-documents.research.global.fujitsu.com/fieldworkarena/", "code": "https://github.com/FujitsuResearch/FieldWorkArena" }, "notes": "Factory dataset (V1.0) was released on the Fujitsu site in February 2025, before the May 2025 arXiv paper. Dataset access requires an application form (gated HuggingFace), and the site lists its leaderboard as 'coming soon', so the only public numbers are the authors' own MLLM evaluations in the paper (v4, June 2026); task counts and the human figure are from that revision. The paper's 0-1 scores are multiplied by 100 here.", "sota": { "system": "GPT-5.2 (2025-12-11)", "developer": "OpenAI", "value": 52.0, "date": "2026-06-07", "source": { "url": "https://arxiv.org/abs/2505.19662", "kind": "paper", "accessed": "2026-09-04" }, "conditions": { "notes": "Table 2 total score 0.52 in arXiv v4; best model in the paper." } }, "result_count": 3 }, { "id": "frontiermath", "name": "FrontierMath", "full_name": "FrontierMath: A Benchmark for Evaluating Advanced Mathematical Reasoning in AI (Tiers 1-4)", "layer": "model", "released": "2024-11", "maintainer": "Epoch AI", "description": "Hundreds of original, unpublished research-grade mathematics problems written and vetted by expert mathematicians, from hard undergraduate (Tier 1) through advanced graduate (Tier 3) to research-level Tier 4, each with an automatically verifiable answer. Models may reason and run Python within a token budget and submit an answer function; scored as the fraction of problems solved. Kept private to avoid contamination, it is the main measure of frontier mathematical reasoning, and Epoch runs all evaluations itself.", "description_zh": "数百道由专家数学家编写并审核的原创、未发表的研究级数学问题,难度从高难本科(Tier 1)经高阶研究生(Tier 3)到研究水平的 Tier 4,每题答案均可自动验证。模型可在 token 预算内推理并运行 Python,提交答案函数;得分为解出问题的比例。为避免污染而保持私有,是前沿数学推理的主要衡量标准,所有评估均由 Epoch 自行运行。", "domains": [ "math", "reasoning", "research" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 338, "splits": [ { "name": "Tiers 1-3 (v2)", "task_count": 295, "note": "Core set after the June 2026 v2 correction; the headline FrontierMath number." }, { "name": "Tier 4 (v2)", "task_count": 43, "note": "Research-level expansion set released June 2025, corrected June 2026." }, { "name": "Tiers 1-3 (v1)", "task_count": 300, "note": "Original problem set used for all runs before 2026-06-12; not comparable with v2." } ], "status": "saturating", "contamination_risk": "low", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2411.04872", "website": "https://epoch.ai/frontiermath", "leaderboard": "https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2", "dataset": "https://epoch.ai/frontiermath/tiers-1-4/benchmark-problems", "code": null }, "notes": "OpenAI funded the benchmark and holds exclusive access to a subset (Epoch's conflict-of-interest statement). On 2026-06-12 Epoch released v2, correcting errors in 42% of problems and removing 12; v1 and v2 scores are separate series. Epoch also runs FrontierMath Open Problems and FrontierMath Erdos, which are different benchmarks.", "sota": { "system": "gpt-6-astra (max)", "developer": "OpenAI", "value": 93.7, "date": "2026-09-03", "source": { "url": "https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2", "kind": "independent-evaluation", "accessed": "2026-09-04" }, "conditions": { "split": "Tiers 1-3 (v2)", "tools": true, "notes": "Epoch AI Benchmarking Hub (run started 2026-08-30, published with the model on 2026-09-03), stderr 1.4; Tier 4 (v2): 95.1 at max, 97.6 at medium effort." } }, "result_count": 5 }, { "id": "gaia", "name": "GAIA", "full_name": "GAIA: a benchmark for General AI Assistants", "layer": "agent", "released": "2023-11", "maintainer": "Meta FAIR / Hugging Face / AutoGPT (Mialon et al.)", "description": "466 real-world questions (166 public validation, 300 private test) that need web browsing, file handling, multimodal reading and tool use to reach a short, unambiguous answer; scored by quasi-exact match after normalisation. Three difficulty levels, with Level 3 requiring long tool-use chains. Questions are easy for humans, so the gap to human accuracy is the headline signal for general assistants.", "description_zh": "466 个真实世界问题(166 个公开验证、300 个私有测试),需要网页浏览、文件处理、多模态阅读和工具使用才能得出简短、无歧义的答案;归一化后按准精确匹配评分。分三个难度等级,Level 3 需要长工具使用链。问题对人类来说很容易,因此与人类准确率的差距是通用助手的主要信号。", "domains": [ "general-assistant", "tool-use", "web", "reasoning", "multimodal" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 466, "splits": [ { "name": "test", "task_count": 300, "note": "Private-answer test set scored on the Hugging Face leaderboard; the only leaderboard still open." }, { "name": "validation", "task_count": 166, "note": "Public dev set with answers; the validation leaderboard was closed by the maintainers." } ], "status": "saturating", "contamination_risk": "medium", "human_baseline": { "value": 92.0, "population": "human annotators (validation phase, average over levels)", "source": "https://arxiv.org/abs/2311.12983" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2311.12983", "website": "https://huggingface.co/gaia-benchmark", "leaderboard": "https://huggingface.co/spaces/gaia-benchmark/leaderboard", "dataset": "https://huggingface.co/datasets/gaia-benchmark/GAIA", "code": "https://huggingface.co/spaces/gaia-benchmark/leaderboard/blob/main/scorer.py" }, "notes": "Test answers are private but the leaderboard is self-submitted and dominated by multi-model ensembles; top scores are within a few points of the 92% human figure. HAL (hal.cs.princeton.edu/gaia) re-runs open scaffolds on the validation set for independent numbers.", "sota": { "system": "CustomGPT.ai Research Lab v44", "developer": "CustomGPT.ai", "value": 93.36, "date": "2026-06-03", "source": { "url": "https://huggingface.co/spaces/gaia-benchmark/leaderboard", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "test", "scaffold": "CustomGPT.ai enterprise agent", "tools": true, "notes": "Top of the test leaderboard as of access date; ensemble of Claude, Gemini and GPT models (self-submitted, auto-scored)." } }, "result_count": 5 }, { "id": "gdpval", "name": "GDPval", "full_name": "GDPval: Evaluating AI Model Performance on Real-World Economically Valuable Tasks", "layer": "model", "released": "2025-09", "maintainer": "OpenAI", "description": "Real work deliverables (legal briefs, spreadsheets, slide decks, CAD, schedules, video edits) drawn from 44 occupations in the 9 US sectors that contribute most to GDP, written and reviewed by professionals averaging 14 years of experience. A model receives the request plus reference files and produces the deliverable in one shot; occupational experts blindly compare it with the human expert's work and the score is the share of comparisons rated better than or as good as the expert. The main public yardstick for economically valuable knowledge work.", "description_zh": "取自美国 GDP 贡献最大的 9 个行业、44 个职业的真实工作交付物(法律文书、电子表格、幻灯片、CAD、排班、视频剪辑),由平均 14 年经验的专业人士编写与审核。模型一次性根据任务描述和参考文件生成交付物,由同职业专家盲评与人类专家作品比较,得分为被评为优于或不逊于专家的比例。是衡量有经济价值的知识工作的主要公开标尺。", "domains": [ "general-assistant", "knowledge", "instruction-following" ], "metric": { "name": "win-or-tie rate vs industry experts", "unit": "percent", "higher_is_better": true }, "task_count": 220, "splits": [ { "name": "gold", "task_count": 220, "note": "Open-sourced subset (5 tasks per occupation) used for all public results." }, { "name": "full", "task_count": 1320, "note": "Private full set, 30 tasks per occupation; OpenAI-internal." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2510.04374", "website": "https://openai.com/index/gdpval/", "leaderboard": null, "dataset": "https://huggingface.co/datasets/openai/gdpval", "code": null }, "notes": "Announced 2025-09-25; arXiv 2510.04374 verified (Patwardhan et al., posted 2025-10-05). Grading is pairwise by human experts (3 samples x 3 graders per task); an experimental automated grader is offered at evals.openai.com but is not a substitute. 50% is expert parity by construction. Artificial Analysis runs the same 220 tasks agentically as GDPval-AA with Elo scoring, which is a different metric and is not recorded here. Vendor-reported 2025-12 and later numbers use newer ChatGPT tools not available to earlier models.", "sota": { "system": "GPT-5.2 Pro", "developer": "OpenAI", "value": 74.1, "date": "2025-12-11", "source": { "url": "https://openai.com/index/introducing-gpt-5-2/", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "split": "gold", "tools": true, "notes": "Wins-or-ties, from the appendix table of the GPT-5.2 announcement." } }, "result_count": 6 }, { "id": "gpqa-diamond", "name": "GPQA Diamond", "full_name": "Graduate-Level Google-Proof Q&A Benchmark, Diamond subset", "layer": "model", "released": "2023-11", "maintainer": "NYU / Cohere / Anthropic (Rein et al.)", "description": "198 four-option multiple-choice questions in biology, physics and chemistry written by domain PhDs and validated so that experts agree while skilled non-experts with unrestricted web access fail. The Diamond subset keeps only questions both experts answered correctly and most non-experts missed. The standard frontier science-reasoning benchmark; noise floor is roughly plus or minus 3 points at n=198.", "description_zh": "198 道生物、物理和化学领域的四选一多项选择题,由领域博士编写并经验证:专家意见一致,而可无限制访问网络的熟练非专家却会答错。Diamond 子集仅保留两位专家均答对且多数非专家答错的题目。是前沿科学推理的标准基准;在 n=198 时噪声下限约为正负 3 分。", "domains": [ "science", "reasoning", "knowledge" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 198, "status": "saturating", "contamination_risk": "medium", "human_baseline": { "value": 69.7, "population": "domain PhD experts (per-question, in-domain)", "source": "https://arxiv.org/abs/2311.12022" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2311.12022", "website": "https://github.com/idavidrein/gpqa", "leaderboard": null, "dataset": "https://huggingface.co/datasets/Idavidrein/gpqa", "code": "https://github.com/idavidrein/gpqa" }, "notes": "Answer options are public; the maintainers ask that the dataset never be posted in plain text. Scores above ~92% are within noise of each other.", "sota": { "system": "GPT-6 Astra", "developer": "OpenAI", "value": 96.0, "date": "2026-09-03", "source": { "url": "https://benchlm.ai/benchmarks/gpqa-diamond", "kind": "aggregator", "accessed": "2026-09-04" } }, "result_count": 7 }, { "id": "gsm8k", "name": "GSM8K", "full_name": "Grade School Math 8K", "layer": "model", "released": "2021-10", "maintainer": "OpenAI (Cobbe et al.)", "description": "8,500 linguistically diverse grade-school math word problems (7,473 train, 1,319 test) that take two to eight elementary arithmetic steps to solve, with natural-language step-by-step solutions. Scored as exact match of the final numeric answer on the test split. It drove chain-of-thought research and was the standard math-reasoning check from 2021 to 2024; frontier models now exceed 95%, and errors are dominated by label noise.", "description_zh": "8,500 道语言多样的小学数学应用题(7,473 训练、1,319 测试),需要两到八步基础算术求解,并附自然语言分步解答。以测试集上最终数值答案的精确匹配评分。它推动了思维链研究,是 2021 至 2024 年的标准数学推理检验;前沿模型如今已超过 95%,错误主要来自标注噪声。", "domains": [ "math", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 1319, "status": "saturated", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2110.14168", "website": "https://github.com/openai/grade-school-math", "leaderboard": null, "dataset": "https://huggingface.co/datasets/openai/gsm8k", "code": "https://github.com/openai/grade-school-math" }, "notes": "Roughly 1-2% of test items are believed mislabeled, so scores above ~97% are within noise. Few-shot count (0, 5 or 8) and use of a calculator or code affect comparability.", "sota": { "system": "DeepSeek-V4-Pro-Base", "developer": "DeepSeek", "value": 92.6, "date": "2026-04-22", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "shots": 8, "notes": "DeepSeek-V4 model card, base-model table (HF repo created 2026-04-22). Base model, exact match; chat models are no longer reported on GSM8K by most vendors." } }, "result_count": 2 }, { "id": "healthbench", "name": "HealthBench", "full_name": "HealthBench: Evaluating Large Language Models Towards Improved Human Health", "layer": "model", "released": "2025-05", "maintainer": "OpenAI", "description": "5,000 realistic multi-turn, multilingual health conversations between a model and a layperson or clinician, spanning seven themes such as emergency referrals, context seeking and global health. Each conversation carries a physician-written rubric (48,562 criteria from 262 physicians in 60 countries); a GPT-4.1 grader checks which criteria the final response meets and the score is points earned over the maximum. Hard (1,000) and Consensus (3,671) subsets isolate unsaturated and physician-validated criteria. The reference evaluation for medical helpfulness and safety.", "description_zh": "5,000 段模型与普通用户或临床人员之间的真实多轮、多语言健康对话,涵盖急诊转诊、追问上下文、全球健康等七类主题。每段对话附医生撰写的评分细则(来自 60 国 262 名医生的 48,562 条标准);GPT-4.1 评分器判断末轮回复满足哪些标准,得分为所得分值占满分的比例。Hard(1,000)与 Consensus(3,671)子集分别隔离未饱和与医生共识标准。医疗有用性与安全性的参考评估。", "domains": [ "knowledge", "safety", "instruction-following" ], "metric": { "name": "rubric score", "unit": "percent", "higher_is_better": true }, "task_count": 5000, "splits": [ { "name": "hard", "task_count": 1000, "note": "HealthBench Hard: examples frontier models struggled with at release." }, { "name": "consensus", "task_count": 3671, "note": "HealthBench Consensus: criteria multiply validated by physician agreement; reported as error rate in the paper." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2505.08775", "website": "https://openai.com/index/healthbench/", "leaderboard": null, "dataset": "https://github.com/openai/simple-evals", "code": "https://github.com/openai/simple-evals" }, "notes": "Announced 2025-05-12; arXiv 2505.08775 verified (Arora et al., posted 2025-05-13). Scores are 0-1 in the paper and shown here as percent. Physician responses without model help scored below the September 2024 models, and physicians could not improve on o3/GPT-4.1 responses, so no human baseline is recorded. OpenAI's 2026 launch posts report a 'HealthBench Professional (length-adjusted)' variant graded by GPT-5.4; it is not documented publicly and is not tracked here. Conversations are public with a canary string.", "sota": { "system": "o3", "developer": "OpenAI", "value": 59.9, "date": "2025-05-13", "source": { "url": "https://arxiv.org/abs/2505.08775", "kind": "paper", "accessed": "2026-09-04" }, "conditions": { "notes": "Best model at release; mean of 16 runs (Table 7). Paper states o3 also tops HealthBench Hard at 32%." } }, "result_count": 4 }, { "id": "hellaswag", "name": "HellaSwag", "full_name": "HellaSwag: Can a Machine Really Finish Your Sentence?", "layer": "model", "released": "2019-05", "maintainer": "Allen Institute for AI / University of Washington (Zellers et al.)", "description": "Commonsense sentence-completion: given a context from ActivityNet captions or WikiHow, pick the most plausible continuation among four options, where the wrong endings were adversarially filtered to fool 2019-era models. 10,042 validation and 10,003 test items, scored as accuracy (typically 10-shot). Trivial for humans and once a headline pretraining benchmark; modern models exceed 95%, so it no longer separates systems.", "description_zh": "常识句子补全:给定来自 ActivityNet 字幕或 WikiHow 的上下文,从四个选项中选出最合理的续写,其中错误结尾经过对抗式筛选以迷惑 2019 年前后的模型。包含 10,042 个验证项和 10,003 个测试项,以准确率评分(通常 10-shot)。对人类而言极其简单,曾是预训练的头条基准;现代模型已超过 95%,不再能区分各系统。", "domains": [ "commonsense" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 10042, "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 95.6, "population": "crowd workers (paper Table 1, overall)", "source": "https://arxiv.org/abs/1905.07830" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/1905.07830", "website": "https://rowanzellers.com/hellaswag/", "leaderboard": null, "dataset": "https://huggingface.co/datasets/Rowan/hellaswag", "code": "https://github.com/rowanz/hellaswag" }, "notes": "Reported scores use the validation split because test labels are hidden. Roughly a third of items contain grammatical or labeling errors, which caps meaningful accuracy.", "sota": { "system": "DeepSeek-V4-Pro-Base", "developer": "DeepSeek", "value": 88.0, "date": "2026-04-22", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "shots": 0, "notes": "DeepSeek-V4 model card, base-model table (HF repo created 2026-04-22). Base model, exact match, zero-shot; not comparable with 10-shot harness numbers." } }, "result_count": 2 }, { "id": "hle", "name": "Humanity's Last Exam", "full_name": "Humanity's Last Exam (HLE)", "layer": "model", "released": "2025-01", "maintainer": "Center for AI Safety / Scale AI", "description": "2,500 expert-written closed-ended questions (multiple choice and exact-match short answer, about 14% with images) across mathematics, natural sciences, humanities and more, filtered so frontier models failed them at collection time. Scored as accuracy by an LLM judge (o3-mini) against the reference answer; calibration error is also reported. Designed as the final broad academic benchmark, it is now the main frontier knowledge-reasoning test.", "description_zh": "2,500 道专家编写的封闭式问题(多项选择与精确匹配的简答,约 14% 含图像),覆盖数学、自然科学、人文等领域,经筛选确保前沿模型在收集时无法作答。由 LLM 评判(o3-mini)对照参考答案计算准确率;同时报告校准误差。设计为最后一个广泛的学术基准,如今是前沿知识推理的主要测试。", "domains": [ "knowledge", "reasoning", "science", "math", "multimodal" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 2500, "splits": [ { "name": "text-only", "note": "Subset without image questions, used for models that are not multimodal." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2501.14249", "website": "https://lastexam.ai/", "leaderboard": "https://lastexam.ai/", "dataset": "https://huggingface.co/datasets/cais/hle", "code": "https://github.com/centerforaisafety/hle" }, "notes": "Finalized at 2,500 questions on 2025-04-03 after a bug bounty; earlier scores used a different question set. Published in Nature (649, 1139-1146) on 2026-01-28. A private held-out set exists to detect overfitting, and an HLE-Rolling fork was released in October 2025. Tool-assisted (search/code) and no-tools runs are not comparable; the ledger marks tools explicitly.", "sota": { "system": "Claude Fable 5.1 (with tools)", "developer": "Anthropic", "value": 65.0, "date": "2026-09-01", "source": { "url": "https://www.anthropic.com/claude-fable-and-mythos-5-1", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "tools": true, "notes": "Launch post; also republished by benchlm.ai as the current top. OpenAI's GPT-6 Astra post lists Astra at 57.2 with tools." } }, "result_count": 6 }, { "id": "humaneval", "name": "HumanEval", "full_name": "HumanEval: hand-written Python programming problems (Codex paper)", "layer": "model", "released": "2021-07", "maintainer": "OpenAI", "description": "164 hand-written Python programming problems, each a function signature plus docstring, scored by running hidden unit tests and reporting pass@k (usually pass@1) for functional correctness. It introduced execution-based code evaluation and was the standard code benchmark from 2021 to 2024; frontier models now solve nearly every problem and the set is widely present in training data.", "description_zh": "164 道手工编写的 Python 编程题,每题为函数签名加文档字符串,通过运行隐藏单元测试评分,以 pass@k(通常为 pass@1)报告功能正确性。它引入了基于执行的代码评估,是 2021 至 2024 年的标准代码基准;前沿模型如今几乎能解出所有题目,且该题集已广泛存在于训练数据中。", "domains": [ "code" ], "metric": { "name": "pass@1", "unit": "percent", "higher_is_better": true }, "task_count": 164, "status": "saturated", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2107.03374", "website": "https://github.com/openai/human-eval", "leaderboard": null, "dataset": "https://huggingface.co/datasets/openai/openai_humaneval", "code": "https://github.com/openai/human-eval" }, "notes": "Many papers report EvalPlus (HumanEval+) numbers with extra tests, which are lower and not directly comparable. OpenAI's simple-evals stopped reporting new models in July 2025.", "sota": { "system": "o4-mini-high", "developer": "OpenAI", "value": 99.3, "date": "2025-04-16", "source": { "url": "https://github.com/openai/simple-evals", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "pass_k": 1, "shots": 0, "notes": "simple-evals README; effectively the ceiling (163 of 164)." } }, "result_count": 4 }, { "id": "ifeval", "name": "IFEval", "full_name": "Instruction-Following Evaluation for Large Language Models", "layer": "model", "released": "2023-11", "maintainer": "Google (Zhou et al.)", "description": "541 prompts each containing one or more of 25 types of verifiable instructions, such as 'write at least 400 words', 'respond in JSON', 'include the keyword X three times' or 'no commas'. Compliance is checked programmatically, giving prompt-level and instruction-level accuracy under strict and loose matching. Cheap, reproducible and free of LLM judges, it is the standard check that a chat model actually obeys formatting constraints.", "description_zh": "541 个提示,每个包含 25 类可验证指令中的一种或多种,例如\"至少写 400 字\"、\"以 JSON 回复\"、\"包含关键词 X 三次\"或\"不使用逗号\"。合规性通过程序检查,给出严格与宽松匹配下的提示级和指令级准确率。低成本、可复现且不依赖 LLM 评判,是检验聊天模型是否真正遵守格式约束的标准。", "domains": [ "instruction-following" ], "metric": { "name": "prompt-level strict accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 541, "splits": [ { "name": "prompt-level strict", "task_count": 541, "note": "All instructions in a prompt must be satisfied; the headline metric." }, { "name": "instruction-level loose", "note": "Per-instruction accuracy with relaxed matching; some vendors report this higher number." } ], "status": "saturating", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2311.07911", "website": "https://github.com/google-research/google-research/tree/master/instruction_following_eval", "leaderboard": null, "dataset": "https://huggingface.co/datasets/google/IFEval", "code": "https://github.com/google-research/google-research/tree/master/instruction_following_eval" }, "notes": "Vendors frequently report instruction-level or loose accuracy without saying so, which inflates numbers by 5-10 points; check the variant before comparing. Most frontier models exceed 90% prompt-level strict, and IFBench and IFEval-Hard have emerged as harder follow-ups.", "sota": { "system": "Qwen3.5-27B", "developer": "Alibaba", "value": 95.0, "date": "2026-02-24", "source": { "url": "https://huggingface.co/Qwen/Qwen3.5-27B", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "notes": "Model card benchmark table; same table lists GPT-5-mini (2025-08-07) at 93.9 as run by Qwen. Variant not stated. Also the top row on benchlm.ai's IFEval page." } }, "result_count": 4 }, { "id": "livecodebench", "name": "LiveCodeBench", "full_name": "LiveCodeBench: Holistic and Contamination Free Evaluation of Large Language Models for Code", "layer": "model", "released": "2024-03", "maintainer": "UC Berkeley / MIT / Cornell (Jain et al.)", "description": "Competitive-programming problems continuously collected from LeetCode, AtCoder and Codeforces contests, each tagged with its release date so that models can be scored only on problems published after their training cutoff. Code generation is scored as pass@1 against hidden tests (averaged over samples); self-repair, code execution and test-output prediction are separate scenarios. The rolling release-date filter makes it the standard contamination-aware coding benchmark, though the window chosen changes the score.", "description_zh": "从 LeetCode、AtCoder 和 Codeforces 竞赛中持续收集的竞赛编程题,每题标注发布日期,以便仅在模型训练截止之后发布的题目上评分。代码生成以针对隐藏测试的 pass@1(多次采样取平均)评分;自我修复、代码执行和测试输出预测为独立场景。滚动的发布日期过滤使其成为标准的防污染编程基准,但所选时间窗口会改变分数。", "domains": [ "code", "reasoning" ], "metric": { "name": "pass@1", "unit": "percent", "higher_is_better": true }, "task_count": 1055, "splits": [ { "name": "v6 (2024-08 to 2025-05)", "note": "Problems released August 2024 through May 2025; a common window in 2025 vendor reports." }, { "name": "v5 (2024-08 to 2025-02)", "note": "Problems released August 2024 through February 2025; used by DeepSeek-R1, Qwen3 and similar 2025 reports." }, { "name": "full (2023-05 to 2025-04)", "task_count": 1055, "note": "All problems in the official leaderboard data as of the last public refresh." } ], "status": "active", "contamination_risk": "low", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2403.07974", "website": "https://livecodebench.github.io/", "leaderboard": "https://livecodebench.github.io/leaderboard.html", "dataset": "https://huggingface.co/datasets/livecodebench/code_generation_lite", "code": "https://github.com/LiveCodeBench/LiveCodeBench" }, "notes": "Vendor numbers use different date windows (v5, v6, or custom) and are not comparable without the window; the public leaderboard's problem pool ends in April 2025, and the maintainers have since focused on LiveCodeBench Pro (Elo-rated). Task count is for the full public pool; each window is a subset.", "sota": { "system": "DeepSeek-V4-Pro (Think Max)", "developer": "DeepSeek", "value": 93.5, "date": "2026-04-22", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "pass_k": 1, "notes": "Model card comparison table; problem window not stated. Same table: Gemini-3.1-Pro (High) 91.7, Kimi K2.6 89.6, Claude Opus 4.6 88.8." } }, "result_count": 5 }, { "id": "lmarena-text", "name": "LMArena Text", "full_name": "LMArena (Chatbot Arena) Text leaderboard", "layer": "model", "released": "2023-05", "maintainer": "LMArena (Arena; formerly LMSYS / UC Berkeley)", "description": "Live crowdsourced human-preference ranking of chat models. Visitors type any prompt, receive answers from two anonymous models side by side and vote; millions of pairwise votes are fit with a Bradley-Terry model (originally online Elo) to produce an Arena score with bootstrap confidence intervals, plus style-controlled and category views. Because prompts are fresh and judged by real users it is hard to contaminate and captures perceived helpfulness, but it rewards persuasive formatting and depends on which models are sampled.", "description_zh": "实时众包的人类偏好排行榜。访问者输入任意提示,并排收到两个匿名模型的回答后投票;数百万成对投票以 Bradley-Terry 模型(最初为在线 Elo)拟合得到 Arena 分数及自举置信区间,另有风格控制与分类视图。提示新鲜且由真实用户评判,难以污染并反映感知有用性,但会奖励有说服力的排版并受模型采样策略影响。", "domains": [ "human-preference", "general-assistant" ], "metric": { "name": "Arena score (Bradley-Terry Elo scale)", "unit": "elo", "higher_is_better": true }, "task_count": null, "splits": [ { "name": "style-control", "note": "Default view since 2025 that regresses out response length and markdown formatting." } ], "status": "active", "contamination_risk": "low", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2403.04132", "website": "https://lmarena.ai", "leaderboard": "https://lmarena.ai/leaderboard/text", "dataset": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard", "code": "https://github.com/lmarena/arena-rank" }, "notes": "Launched 2023-05-03 as Chatbot Arena (lmsys.org blog); paper arXiv 2403.04132 verified. Ledger rows are the top text model at dated snapshots taken from LMSYS blog tables and the maintainer's leaderboard data files (elo_results_YYYYMMDD.pkl in the lmarena-ai/arena-leaderboard Space, whose last committed snapshot is 2025-08-29). Scores are only comparable within one snapshot: the scale drifts as models are added and the rating system moved from online Elo to Bradley-Terry in December 2023. The live site (lmarena.ai / arena.ai) was unreachable from this environment on 2026-09-04, so no 2026 snapshot is recorded.", "sota": { "system": "gemini-2.5-pro", "developer": "Google", "value": 1466, "date": "2025-08-29", "source": { "url": "https://huggingface.co/spaces/lmarena-ai/arena-leaderboard", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "notes": "elo_results_20250829.pkl text/full table; 35,586 battles; rounded from 1466.2. Under style control the same snapshot ranks gemini-2.5-pro 1456, gpt-5-high 1447, claude-opus-4-1 thinking 1447." } }, "result_count": 5 }, { "id": "math", "name": "MATH", "full_name": "MATH: Measuring Mathematical Problem Solving", "layer": "model", "released": "2021-03", "maintainer": "Dan Hendrycks et al. (UC Berkeley)", "description": "12,500 competition mathematics problems (7,500 train, 5,000 test) from AMC 10/12, AIME and similar contests across seven subjects and five difficulty levels, each with a full LaTeX step-by-step solution. Scored as exact match of the boxed final answer after normalization. It was the primary hard-math benchmark until reasoning models pushed it past 95%; the hardest Level 5 subset and the 500-problem MATH-500 split remain in use.", "description_zh": "12,500 道竞赛数学题(7,500 训练、5,000 测试),来自 AMC 10/12、AIME 及类似竞赛,涵盖七个学科和五个难度等级,每题附完整的 LaTeX 分步解答。归一化后以加框最终答案的精确匹配评分。在推理模型将其推过 95% 之前,它是主要的高难数学基准;最难的 Level 5 子集和 500 题的 MATH-500 划分仍在使用。", "domains": [ "math", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 5000, "splits": [ { "name": "MATH-500", "task_count": 500, "note": "Uniform 500-problem subset introduced by OpenAI's PRM800K work; the most commonly reported split since 2024." }, { "name": "Level 5", "task_count": 1324, "note": "Hardest difficulty tier of the test split, used by Epoch AI's benchmarking hub." } ], "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 90.0, "population": "three-time IMO gold medalist (single participant; a CS PhD student scored about 40%)", "source": "https://arxiv.org/abs/2103.03874" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2103.03874", "website": "https://github.com/hendrycks/math", "leaderboard": "https://epoch.ai/benchmarks/math-level-5", "dataset": "https://huggingface.co/datasets/EleutherAI/hendrycks_math", "code": "https://github.com/hendrycks/math" }, "notes": "Answer-equivalence checking differs across harnesses (string match vs sympy vs LLM grader) and can move scores by several points. The human numbers in the paper are anecdotal single-person measurements.", "sota": { "system": "o3-high", "developer": "OpenAI", "value": 98.1, "date": "2025-04-16", "source": { "url": "https://github.com/openai/simple-evals", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "split": "MATH-500", "shots": 0, "notes": "simple-evals README; o4-mini-high 98.2 on the same table. Epoch's Level-5 run of gpt-5 (high) reached 98.1 in October 2025." } }, "result_count": 4 }, { "id": "mbpp", "name": "MBPP", "full_name": "Mostly Basic Programming Problems", "layer": "model", "released": "2021-08", "maintainer": "Google Research (Austin et al.)", "description": "974 crowd-sourced short Python programming tasks aimed at entry-level programmers, each with a natural-language description, a reference solution and three test cases; a 427-problem hand-verified 'sanitized' subset is also used. Scored by executing the model's program against the tests (pass@1 or fraction solved). One of the two canonical early code-synthesis benchmarks alongside HumanEval; now saturated and largely contaminated.", "description_zh": "974 道面向入门程序员的众包短篇 Python 编程任务,每题附自然语言描述、参考解答和三个测试用例;另有 427 题经人工核验的\"sanitized\"子集。通过对测试执行模型程序评分(pass@1 或解出比例)。与 HumanEval 并列为两个经典早期代码合成基准之一;现已饱和且大量受污染。", "domains": [ "code" ], "metric": { "name": "pass@1", "unit": "percent", "higher_is_better": true }, "task_count": 974, "splits": [ { "name": "sanitized", "task_count": 427, "note": "Hand-verified subset with cleaned descriptions." }, { "name": "test", "task_count": 500, "note": "The 500-problem test split most commonly reported." } ], "status": "saturated", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2108.07732", "website": "https://github.com/google-research/google-research/tree/master/mbpp", "leaderboard": null, "dataset": "https://huggingface.co/datasets/google-research-datasets/mbpp", "code": "https://github.com/google-research/google-research/tree/master/mbpp" }, "notes": "Reported numbers mix the full set, the 500-problem test split, the sanitized subset and EvalPlus MBPP+, so compare only like with like.", "sota": { "system": "Llama 3.1 405B Instruct", "developer": "Meta", "value": 88.6, "date": "2024-07-23", "source": { "url": "https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "pass_k": 1, "shots": 0, "notes": "Llama 3.1 model card row 'MBPP ++ base version' (EvalPlus MBPP+ base tests, 378 problems): 88.6 for 405B, 86.0 for 70B. Not the original 500-problem test split. Date is the Llama 3.1 release." } }, "result_count": 2 }, { "id": "mle-bench", "name": "MLE-bench", "full_name": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "layer": "agent", "released": "2024-10", "maintainer": "OpenAI (Chan et al.)", "description": "75 Kaggle competitions (22 low, 38 medium, 15 high complexity) in which an agent must read the task, prepare data, train models and submit predictions inside a sandbox within a compute budget. The score is the fraction of competitions in which the submission would have earned at least a bronze medal on the real Kaggle leaderboard. The standard test of end-to-end ML engineering rather than isolated coding.", "description_zh": "75 个 Kaggle 竞赛(22 低、38 中、15 高复杂度),智能体需在沙盒内于算力预算之内阅读任务、准备数据、训练模型并提交预测。得分为提交结果在真实 Kaggle 排行榜上至少能获得铜牌的竞赛比例。是端到端 ML 工程(而非孤立编码)的标准测试。", "domains": [ "ml-engineering", "code", "tool-use" ], "metric": { "name": "any-medal rate", "unit": "percent", "higher_is_better": true }, "task_count": 75, "splits": [ { "name": "all", "task_count": 75, "note": "Full benchmark; the leaderboard's 'All (%)' column." }, { "name": "lite", "task_count": 22, "note": "Low-complexity subset (MLE-bench Lite)." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2410.07095", "website": "https://github.com/openai/mle-bench", "leaderboard": "https://github.com/openai/mle-bench", "dataset": "https://github.com/openai/mle-bench", "code": "https://github.com/openai/mle-bench" }, "notes": "Competitions are public Kaggle data and the paper documents contamination checks. Leaderboard rows are self-submitted with grading reports; as of April 2026 OpenAI paused new submissions while redesigning the process. Runs use 24h (sometimes 12h/36h) budgets, which affects comparability.", "sota": { "system": "Famou-Agent 2.0 + Gemini-3-Pro-Preview", "developer": "Baidu", "value": 64.44, "date": "2026-02-23", "source": { "url": "https://github.com/openai/mle-bench", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "all", "scaffold": "Famou-Agent 2.0", "notes": "Top of the main leaderboard as of access date; 24h budget." } }, "result_count": 5 }, { "id": "mmlu", "name": "MMLU", "full_name": "Measuring Massive Multitask Language Understanding", "layer": "model", "released": "2020-09", "maintainer": "Dan Hendrycks et al. (UC Berkeley)", "description": "14,042 four-option multiple-choice test questions across 57 subjects from elementary mathematics and US history to law and medicine, drawn from exams and textbooks (15,908 questions in total including dev and validation). Scored as accuracy, historically 5-shot. For three years it was the default measure of broad world knowledge; frontier models now cluster above 90%, and label noise in the test set limits further discrimination.", "description_zh": "14,042 道四选一多项选择测试题,覆盖 57 个学科,从初等数学、美国历史到法律和医学,取自考试和教科书(含开发集与验证集共 15,908 题)。以准确率评分,历史上采用 5-shot。三年来一直是广泛世界知识的默认衡量标准;前沿模型如今集中在 90% 以上,测试集的标注噪声限制了进一步区分。", "domains": [ "knowledge", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 14042, "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 89.8, "population": "estimated expert-level test takers (paper estimate from source-exam pass rates)", "source": "https://arxiv.org/abs/2009.03300" }, "supersedes": null, "superseded_by": "mmlu-pro", "links": { "paper": "https://arxiv.org/abs/2009.03300", "website": "https://github.com/hendrycks/test", "leaderboard": null, "dataset": "https://huggingface.co/datasets/cais/mmlu", "code": "https://github.com/hendrycks/test" }, "notes": "The paper's 89.8% expert figure is an estimate, not a measured panel; unspecialized crowd workers scored 34.5%. Roughly 6.5% of questions are estimated to contain errors (MMLU-Redux), so scores above ~90% are not reliably comparable.", "sota": { "system": "o1", "developer": "OpenAI", "value": 91.8, "date": "2024-12-17", "source": { "url": "https://github.com/openai/simple-evals", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "shots": 0, "notes": "simple-evals README; o3-high later scored 93.3 (April 2025)." } }, "result_count": 4 }, { "id": "mmlu-pro", "name": "MMLU-Pro", "full_name": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark", "layer": "model", "released": "2024-06", "maintainer": "TIGER-Lab (University of Waterloo)", "description": "About 12,000 questions across 14 disciplines built from MMLU, STEM sites, TheoremQA and SciBench, with the answer set expanded from four to ten options and trivial or noisy items removed. Scored as accuracy with chain-of-thought (5-shot in the official setup). Reduces guessing headroom and prompt sensitivity, making it the standard replacement for MMLU when comparing knowledge and reasoning.", "description_zh": "约 12,000 道题,覆盖 14 个学科,基于 MMLU、STEM 网站、TheoremQA 和 SciBench 构建,选项从四个扩展到十个,并移除了琐碎或有噪声的题目。以思维链下的准确率评分(官方设置为 5-shot)。减少了猜测空间和提示敏感性,成为比较知识与推理时 MMLU 的标准替代品。", "domains": [ "knowledge", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 12032, "status": "saturating", "contamination_risk": "high", "human_baseline": null, "supersedes": "mmlu", "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2406.01574", "website": "https://github.com/TIGER-AI-Lab/MMLU-Pro", "leaderboard": "https://huggingface.co/spaces/TIGER-Lab/MMLU-Pro", "dataset": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "code": "https://github.com/TIGER-AI-Lab/MMLU-Pro" }, "notes": "Developer-reported numbers vary by prompt (CoT vs direct) and shots; the official leaderboard uses 5-shot CoT. A January 2026 formatting fix to answer options can shift older STEM-subset scores slightly.", "sota": { "system": "Gemini 3.1 Pro (High)", "developer": "Google DeepMind", "value": 91.0, "date": "2026-04-22", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "kind": "independent-evaluation", "accessed": "2026-09-04" }, "conditions": { "notes": "Run by DeepSeek for its V4-Pro model card comparison table; setting not further specified." } }, "result_count": 4 }, { "id": "mmmu", "name": "MMMU", "full_name": "MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI", "layer": "model", "released": "2023-11", "maintainer": "MMMU Team (Ohio State / Waterloo / CMU; Yue et al.)", "description": "11,500 college-level questions that pair text with images (charts, diagrams, maps, tables, chemical structures, music sheets) across six disciplines and 30 subjects, collected from exams, quizzes and textbooks. Scored as accuracy on the 900-question validation set (the test set of 10,500 is graded via EvalAI). The first broad expert-level multimodal benchmark; it is now close to expert-human accuracy and has been superseded by MMMU-Pro.", "description_zh": "11,500 道大学水平的题目,将文本与图像(图表、示意图、地图、表格、化学结构、乐谱)配对,覆盖六个学科和 30 个科目,收集自考试、测验和教科书。以 900 题验证集上的准确率评分(10,500 题的测试集通过 EvalAI 评分)。首个广泛的专家级多模态基准;如今已接近专家人类准确率,并已被 MMMU-Pro 取代。", "domains": [ "multimodal", "knowledge", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 900, "splits": [ { "name": "validation", "task_count": 900, "note": "Publicly labeled split; the one almost all vendors report." }, { "name": "test", "task_count": 10500, "note": "Hidden labels, scored via the EvalAI server." } ], "status": "saturating", "contamination_risk": "high", "human_baseline": { "value": 88.6, "population": "best of three human experts on the validation set (medium expert 82.6%)", "source": "https://arxiv.org/abs/2311.16502" }, "supersedes": null, "superseded_by": "mmmu-pro", "links": { "paper": "https://arxiv.org/abs/2311.16502", "website": "https://mmmu-benchmark.github.io/", "leaderboard": "https://mmmu-benchmark.github.io/#leaderboard", "dataset": "https://huggingface.co/datasets/MMMU/MMMU", "code": "https://github.com/MMMU-Benchmark/MMMU" }, "notes": "Vendors sometimes report the average of standard and vision settings or use tools; the official leaderboard lists validation accuracy without tools. Many questions can be answered from text alone, which motivated MMMU-Pro.", "sota": { "system": "GPT-5.1", "developer": "OpenAI", "value": 85.4, "date": "2025-11-13", "source": { "url": "https://mmmu-benchmark.github.io/", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "split": "validation", "notes": "Self-reported entry on the official MMMU leaderboard (source: author). Top of the official leaderboard (last updated 2025-09-05 header, entry dated 2025-11-13); within 3 points of the best-expert 88.6." } }, "result_count": 5 }, { "id": "mmmu-pro", "name": "MMMU-Pro", "full_name": "MMMU-Pro: A More Robust Multi-discipline Multimodal Understanding Benchmark", "layer": "model", "released": "2024-09", "maintainer": "MMMU Team (Yue et al.)", "description": "1,730 MMMU questions that survived filtering out items answerable by text-only models, each with the option set expanded to ten candidates, plus a vision-only setting in which the question is rendered inside the image so the model must read and see at once. Scored as accuracy; the headline number averages the standard (10-option) and vision settings. It is the current standard for expert-level multimodal reasoning because it removes the text-only shortcuts that inflated MMMU.", "description_zh": "1,730 道 MMMU 题目,已过滤掉纯文本模型即可作答的题目,每题选项扩展为十个候选,并增加纯视觉设置:问题渲染在图像内部,模型必须同时阅读和观看。以准确率评分;主指标为标准(10 选项)设置与视觉设置的平均值。因消除了曾使 MMMU 虚高的纯文本捷径,它是当前专家级多模态推理的标准。", "domains": [ "multimodal", "knowledge", "reasoning" ], "metric": { "name": "accuracy (average of standard and vision)", "unit": "percent", "higher_is_better": true }, "task_count": 1730, "splits": [ { "name": "standard", "task_count": 1730, "note": "Text question plus image, 10 answer options." }, { "name": "vision", "task_count": 1730, "note": "Question embedded in the image; no separate text." } ], "status": "active", "contamination_risk": "medium", "human_baseline": { "value": 85.4, "population": "estimated high-expert performance derived from MMMU human data (medium 80.8%)", "source": "https://arxiv.org/abs/2409.02813" }, "supersedes": "mmmu", "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2409.02813", "website": "https://mmmu-benchmark.github.io/", "leaderboard": "https://mmmu-benchmark.github.io/#leaderboard", "dataset": "https://huggingface.co/datasets/MMMU/MMMU_Pro", "code": "https://github.com/MMMU-Benchmark/MMMU" }, "notes": "Some vendors report only the standard setting or use tools; the ledger notes the setting where the source states it. The human baseline is an approximation from MMMU annotations, not a fresh expert study on MMMU-Pro.", "sota": { "system": "Chance Vision 1.5", "developer": "Chance (chance.vision)", "value": 86.9, "date": "2026-07-01", "source": { "url": "https://mmmu-benchmark.github.io/", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "notes": "Self-reported entry on the official MMMU leaderboard (source: author). Top of the official MMMU-Pro leaderboard, above the 85.4 estimated high-expert level; leaderboard shows only month, so the 1st is used. benchlm.ai instead lists GPT-5.4 Pro at 94% from a vendor chart." } }, "result_count": 5 }, { "id": "osworld", "name": "OSWorld", "full_name": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "layer": "agent", "released": "2024-04", "maintainer": "XLANG Lab, University of Hong Kong (Xie et al.)", "description": "369 real computer-use tasks in a live Ubuntu VM spanning Chrome, LibreOffice, GIMP, VLC, VS Code, Thunderbird, OS file operations and multi-app workflows. Agents see screenshots (and optionally the accessibility tree), emit mouse/keyboard actions, and are scored by execution-based checkers on the final machine state. The standard benchmark for GUI computer-use agents; OSWorld-Verified (July 2025) fixed 300+ task issues and is the basis of the current official leaderboard.", "description_zh": "369 个在实时 Ubuntu 虚拟机中的真实计算机使用任务,涵盖 Chrome、LibreOffice、GIMP、VLC、VS Code、Thunderbird、操作系统文件操作和多应用工作流。智能体查看截图(可选无障碍树),输出鼠标/键盘动作,由基于执行的检查器对最终机器状态评分。是 GUI 计算机使用智能体的标准基准;OSWorld-Verified(2025 年 7 月)修复了 300 多个任务问题,是当前官方排行榜的基础。", "domains": [ "computer-use", "multimodal", "tool-use" ], "metric": { "name": "success rate", "unit": "percent", "higher_is_better": true }, "task_count": 369, "splits": [ { "name": "original", "task_count": 369, "note": "April 2024 task set and evaluators, pre-Verified fixes." }, { "name": "verified", "task_count": 369, "note": "OSWorld-Verified (July 2025): fixed evaluators, AWS infrastructure; 8 Google Drive tasks may be excluded (361 tasks). Official results at 15/50/100 max steps." } ], "status": "active", "contamination_risk": "medium", "human_baseline": { "value": 72.36, "population": "human computer users (paper)", "source": "https://arxiv.org/abs/2404.07972" }, "supersedes": null, "superseded_by": "osworld-2", "links": { "paper": "https://arxiv.org/abs/2404.07972", "website": "https://os-world.github.io", "leaderboard": "https://os-world.github.io", "dataset": "https://github.com/xlang-ai/OSWorld/tree/main/evaluation_examples", "code": "https://github.com/xlang-ai/OSWorld" }, "notes": "Official 'Verified' leaderboard rows are run by the XLANG team under unified settings (source: static/data/osworld_verified_results.xlsx on the site); self-reported rows are listed separately. Scores depend strongly on the max-step budget, so compare within the same budget. Task configs and evaluators are public, but live-environment execution limits memorisation. Superseded by OSWorld 2.0 (June 2026, 108 long-horizon tasks, tracked as `osworld-2`); the site now lives at osworld-v1.xlang.ai and the Verified leaderboard is still updated.", "sota": { "system": "Intelligence-Indeed Agent", "developer": "Intelligence Indeed", "value": 90.19, "date": "2026-07-25", "source": { "url": "https://os-world.github.io", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "verified", "scaffold": "Intelligence-Indeed Agent", "notes": "Top of the verified leaderboard as of access date (agentic framework, 100 max steps)." } }, "result_count": 6 }, { "id": "osworld-2", "name": "OSWorld 2.0", "full_name": "OSWorld 2.0: Benchmarking Computer-Use Agents on Long-Horizon Real-World Tasks", "layer": "agent", "released": "2026-06", "maintainer": "XLANG Lab, University of Hong Kong (Yuan, Zhou, Xiong, Xie, Yu et al.)", "description": "108 long-horizon computer-use workflows (median human time about 1.6 hours, roughly 300 agent tool calls) spanning research, creative, engineering, finance, administrative and healthcare work across desktop apps and 31 self-hosted websites. Tasks are graded on an average of 27 execution-based checkpoints, yielding a binary completion rate (primary metric, reported at a 500-step budget) and a partial score. Tasks include dynamic mid-task events, hidden state and simulated-user clarification, so it probes the failure modes OSWorld 1.0 no longer separates.", "description_zh": "108 个长程计算机使用工作流(人类完成中位时间约 1.6 小时,智能体约 300 次工具调用),覆盖科研、创意、工程、财务、行政和医疗场景,涉及桌面应用与 31 个自托管网站。每个任务平均设 27 个基于执行的检查点,给出二元完成率(主指标,500 步预算)和部分得分。任务包含任务中途的动态事件、隐藏状态和模拟用户澄清,用于区分 OSWorld 1.0 已无法分辨的失败模式。", "domains": [ "computer-use", "multimodal", "tool-use" ], "metric": { "name": "binary completion rate (500 steps)", "unit": "percent", "higher_is_better": true }, "task_count": 108, "splits": [ { "name": "full", "task_count": 108, "note": "All 108 tasks, including those that need live internet services. Task releases v2026.06.24 (paper) and v2026.08.08 (current) differ, so compare within a release." }, { "name": "offline", "task_count": null, "note": "OSWorld V2-Offline: subset that runs without internet access (test_v2_offline_no_internet.json); used by OpenAI for its GPT-6 Astra comparison." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": "osworld", "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2606.29537", "website": "https://osworld-v2.xlang.ai/", "leaderboard": "https://osworld-v2.xlang.ai/", "dataset": "https://huggingface.co/datasets/xlangai/osworld_v2_tasks", "code": "https://github.com/xlang-ai/OSWorld-V2" }, "notes": "Official rows are run by the XLANG team (static/data/leaderboard/official-results.json) at 150/300/500 step budgets with either the standard or a 'batch tool' action setting; the 500-step binary metric is primary. Vendor launch tables often quote the partial score or the offline subset, and Anthropic's Fable 5.1 system card used modified tasks and grading, so treat developer-reported numbers as non-comparable to the leaderboard unless the release version, set and metric match.", "sota": { "system": "GPT-6 Astra", "developer": "OpenAI", "value": 72.6, "date": "2026-09-03", "source": { "url": "https://openai.com/index/gpt-6-astra/", "kind": "developer-report", "accessed": "2026-09-13" }, "conditions": { "split": "offline", "notes": "Partial score (not binary completion): OpenAI launch table 2026-09-03, ~40 min/task. openai.com returned 403 from the host that added this row, so it was cross-checked against o-mega.ai, datanorth.ai and benchlm.ai." } }, "result_count": 6 }, { "id": "paperbench", "name": "PaperBench", "full_name": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "layer": "agent", "released": "2025-04", "maintainer": "OpenAI (Starace et al.)", "description": "Agents must replicate 20 ICML 2024 Spotlight and Oral papers from scratch: understand the paper, write a codebase and run the experiments, in a container with GPU access. Grading uses author-co-developed hierarchical rubrics (8,316 gradable leaf nodes) applied by an LLM judge, giving a replication score from 0 to 100. The Code-Dev variant grades code only. It measures long-horizon research engineering, with a measured human (ML PhD) baseline for comparison.", "description_zh": "智能体需从零复现 20 篇 ICML 2024 Spotlight 和 Oral 论文:理解论文、编写代码库并在可访问 GPU 的容器中运行实验。评分使用与作者共同制定的层级式评分表(8,316 个可评分叶节点),由 LLM 评判应用,给出 0 到 100 的复现分数。Code-Dev 变体仅对代码评分。它衡量长程研究工程能力,并有实测的人类(ML 博士)基线可供比较。", "domains": [ "research", "ml-engineering", "code", "tool-use" ], "metric": { "name": "replication score", "unit": "percent", "higher_is_better": true }, "task_count": 20, "splits": [ { "name": "full", "task_count": 20, "note": "Full rubric including code development, execution and result match." }, { "name": "code-dev", "task_count": 20, "note": "PaperBench Code-Dev: grades code development only, no execution." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2504.01848", "website": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "leaderboard": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "dataset": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "code": "https://github.com/openai/frontier-evals" }, "notes": "The paper reports that top ML PhDs outperformed models on a 3-paper subset over 48 hours, but the figure is on a subset and not comparable to the full 20-paper score, so no human_baseline is recorded. The repository leaderboard has not been updated since the April 2025 launch; papers and rubrics are public.", "sota": { "system": "IterativeAgent o1-high", "developer": "OpenAI", "value": 43.4, "date": "2025-04-02", "source": { "url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "code-dev", "scaffold": "IterativeAgent", "reasoning_effort": "high", "notes": "PaperBench Code-Dev leaderboard, 3 runs, +/-0.8." } }, "result_count": 4 }, { "id": "ruler", "name": "RULER", "full_name": "RULER: What's the Real Context Size of Your Long-Context Language Models?", "layer": "model", "released": "2024-04", "maintainer": "NVIDIA (Hsieh et al.)", "description": "A synthetic long-context suite of 13 tasks in four categories (needle-in-a-haystack retrieval variants, multi-hop variable tracing, aggregation of frequent words, and question answering over long inputs) generated at configurable lengths from 4K to 128K tokens or more. Scored as accuracy averaged across tasks at each length; a model's 'effective context' is the longest length at which it beats a fixed threshold (Llama-2-7B at 4K, 85.6%). It exposes the gap between claimed and usable context windows.", "description_zh": "合成长上下文套件,包含四类共 13 个任务(多种大海捞针检索变体、多跳变量追踪、高频词聚合以及长输入问答),可按 4K 到 128K 或更长的 token 长度配置生成。以各长度下跨任务平均准确率评分;模型的\"有效上下文\"为其超过固定阈值(Llama-2-7B 在 4K 下的 85.6%)的最长长度。它揭示了宣称的与可用的上下文窗口之间的差距。", "domains": [ "long-context" ], "metric": { "name": "accuracy (average over 13 tasks)", "unit": "percent", "higher_is_better": true }, "task_count": 13, "splits": [ { "name": "128K", "note": "Average across the 13 tasks at 128K tokens; the hardest standard length and the one most often quoted." }, { "name": "Avg. 4K-128K", "note": "Unweighted mean over the six standard lengths (4K, 8K, 16K, 32K, 64K, 128K)." } ], "status": "saturating", "contamination_risk": "low", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2404.06654", "website": "https://github.com/NVIDIA/RULER", "leaderboard": "https://github.com/NVIDIA/RULER", "dataset": null, "code": "https://github.com/NVIDIA/RULER" }, "notes": "Inputs are generated on the fly, so there is nothing to memorize, but the tasks are synthetic and easier than realistic long-document work; strong models now score above 90% at 128K. The maintainers' table mixes their own runs with author-reported numbers (marked with an asterisk), and closed models (GPT-4, Gemini 1.5 Pro) were last run in 2024.", "sota": { "system": "Jamba-1.5-large", "developer": "AI21 Labs", "value": 95.1, "date": "2024-08-22", "source": { "url": "https://github.com/NVIDIA/RULER", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "128K", "notes": "README main table, author-reported from the Jamba-1.5 report (arXiv 2408.12570); 4K-128K average 96.0, ranked 1st on both weighted averages. Date is the Jamba report's arXiv date." } }, "result_count": 5 }, { "id": "screenspot-pro", "name": "ScreenSpot-Pro", "full_name": "ScreenSpot-Pro: GUI Grounding for Professional High-Resolution Computer Use", "layer": "model", "released": "2025-01", "maintainer": "National University of Singapore / HKBU (Kaixin Li et al.)", "description": "1,581 GUI grounding instructions, each on a unique authentic high-resolution screenshot from 23 professional applications (CAD, IDEs, creative suites, scientific tools, office) and three operating systems. The model must output the click point or box for the element an instruction refers to; targets average 0.07% of the screen, and accuracy is the share of predictions landing inside the ground-truth box (micro-averaged over text and icon targets). The standard stress test for the visual grounding that computer-use agents depend on.", "description_zh": "1,581 条 GUI 定位指令,每条对应一张来自 23 个专业应用(CAD、IDE、创意套件、科学工具、办公)与三种操作系统的真实高分辨率截图。模型需输出指令所指元素的点击点或框;目标平均仅占屏幕 0.07%,准确率为预测落入真值框的比例(文本与图标目标微平均)。是计算机使用智能体所依赖的视觉定位能力的标准压力测试。", "domains": [ "computer-use", "multimodal" ], "metric": { "name": "grounding accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 1581, "status": "saturating", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2504.07981", "website": "https://gui-agent.github.io/grounding-leaderboard/", "leaderboard": "https://gui-agent.github.io/grounding-leaderboard/", "dataset": "https://huggingface.co/datasets/likaixin/ScreenSpot-Pro", "code": "https://github.com/likaixin2000/ScreenSpot-Pro-GUI-Grounding" }, "notes": "Paper and dataset released 2025-01-04 (GitHub changelog); arXiv 2504.07981 verified (posted 2025-04-04; ICLR 2025 workshop). Official leaderboard entries use greedy decoding and are contributed by model authors; frontier labs report their own runs, often with tools (OpenAI: Python zoom tool) or agentic zoom-in, which inflates scores versus single-shot grounding. Images and annotations are fully public, so contamination risk is high.", "sota": { "system": "GPT-6 Astra", "developer": "OpenAI", "value": 92.7, "date": "2026-09-03", "source": { "url": "https://openai.com/index/gpt-6-astra/", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "tools": false, "notes": "'ScreenSpot-Pro (no tools)' row; maximum effort. GPT-5.6 Sol: 76.9% in the same table." } }, "result_count": 5 }, { "id": "simpleqa", "name": "SimpleQA", "full_name": "SimpleQA: Measuring short-form factuality in large language models", "layer": "model", "released": "2024-11", "maintainer": "OpenAI (Wei et al.)", "description": "4,326 short fact-seeking questions with a single indisputable answer, adversarially collected against GPT-4o and verified by two independent annotators, spanning science, history, art, geography and more. A grader model marks each answer correct, incorrect or not attempted; the headline metric is percent correct, with F-score and calibration also reported. It measures whether models know what they know: hallucination-prone models score low even when otherwise capable.", "description_zh": "4,326 个简短的事实型问题,每题有唯一无可争议的答案,针对 GPT-4o 对抗式收集并经两名独立标注者验证,涵盖科学、历史、艺术、地理等。评分模型将每个回答标记为正确、错误或未尝试;主指标为正确率,同时报告 F 分数和校准。它衡量模型是否知道自己知道什么:易产生幻觉的模型即便其他能力出色,得分也会很低。", "domains": [ "factuality", "knowledge" ], "metric": { "name": "accuracy (correct)", "unit": "percent", "higher_is_better": true }, "task_count": 4326, "status": "active", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2411.04368", "website": "https://openai.com/index/introducing-simpleqa/", "leaderboard": null, "dataset": "https://github.com/openai/simple-evals", "code": "https://github.com/openai/simple-evals" }, "notes": "Answers are static facts, so retrieval or web search makes the test trivial; only no-tools runs are comparable. Google's SimpleQA Verified (1,000 relabeled questions, 2025) is a separate, stricter variant and its scores should not be mixed with the original. OpenAI stopped reporting SimpleQA in simple-evals in July 2025.", "sota": { "system": "gpt-4.5-preview-2025-02-27", "developer": "OpenAI", "value": 62.5, "date": "2025-02-27", "source": { "url": "https://github.com/openai/simple-evals", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "tools": false, "notes": "simple-evals README benchmark table (repo deprecated July 2025)." } }, "result_count": 4 }, { "id": "swe-bench", "name": "SWE-bench", "full_name": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "layer": "agent", "released": "2023-10", "maintainer": "Princeton NLP / SWE-bench team (Jimenez et al.)", "description": "2,294 task instances built from real GitHub issues and their resolving pull requests across 12 popular Python repositories. Given the issue text and a repository snapshot, an agent must produce a patch; it is scored by running the PR's FAIL_TO_PASS and PASS_TO_PASS unit tests. The original agentic-coding benchmark and the template for the Verified, Lite, Multimodal, Multilingual and Pro descendants.", "description_zh": "2,294 个任务实例,基于 12 个流行 Python 代码库中的真实 GitHub issue 及其对应的修复 pull request 构建。给定 issue 文本和代码库快照,智能体需生成补丁;通过运行该 PR 的 FAIL_TO_PASS 和 PASS_TO_PASS 单元测试评分。是最初的智能体编程基准,也是 Verified、Lite、Multimodal、Multilingual 和 Pro 等后续版本的模板。", "domains": [ "software-engineering", "code", "tool-use" ], "metric": { "name": "% resolved", "unit": "percent", "higher_is_better": true }, "task_count": 2294, "splits": [ { "name": "full", "task_count": 2294, "note": "Original test set; listed as 'Full' on swebench.com." }, { "name": "lite", "task_count": 300, "note": "Subset curated for less costly evaluation (SWE-bench Lite, March 2024)." } ], "status": "saturated", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": "swe-bench-verified", "links": { "paper": "https://arxiv.org/abs/2310.06770", "website": "https://www.swebench.com", "leaderboard": "https://www.swebench.com/#test", "dataset": "https://huggingface.co/datasets/SWE-bench/SWE-bench", "code": "https://github.com/SWE-bench/SWE-bench" }, "notes": "The full 2,294-instance set contains under-specified and untestable tasks (the reason OpenAI and the maintainers curated SWE-bench Verified). Almost all frontier evaluation moved to Verified after August 2024; the Full leaderboard still accepts submissions but receives few, so treat it as historical.", "sota": { "system": "Sonar Foundation Agent + Claude 4.5 Opus", "developer": "Anthropic", "value": 52.62, "date": "2025-12-19", "source": { "url": "https://www.swebench.com/#test", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "full", "scaffold": "Sonar Foundation Agent", "notes": "Top of the Full leaderboard; submission not run or checked by the SWE-bench team." } }, "result_count": 5 }, { "id": "swe-bench-multilingual", "name": "SWE-bench Multilingual", "full_name": "SWE-bench Multilingual: SWE-bench-style tasks across 9 programming languages", "layer": "agent", "released": "2025-03", "maintainer": "SWE-bench team (Khandpur, Lieret, Jimenez, Press, Yang)", "description": "300 curated issue-resolution tasks from 42 repositories in C, C++, Go, Java, JavaScript/TypeScript, PHP, Ruby and Rust, built with the SWE-bench collection pipeline and scored by fail-to-pass and pass-to-pass tests. It checks whether agentic coding ability transfers beyond Python, where scaffolds and models are heavily tuned to SWE-bench Verified.", "description_zh": "300 个精选的问题修复任务,来自 C、C++、Go、Java、JavaScript/TypeScript、PHP、Ruby 和 Rust 的 42 个代码库,使用 SWE-bench 收集流水线构建,并以 fail-to-pass 和 pass-to-pass 测试评分。它检验智能体编程能力能否迁移到 Python 之外,因为脚手架和模型都针对 SWE-bench Verified 做了大量调优。", "domains": [ "software-engineering", "code", "tool-use" ], "metric": { "name": "% resolved", "unit": "percent", "higher_is_better": true }, "task_count": 300, "splits": [ { "name": "bash-only", "task_count": 300, "note": "Official leaderboard view: every model in the same mini-SWE-agent environment, run by the SWE-bench team." } ], "status": "active", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": null, "website": "https://www.swebench.com/multilingual.html", "leaderboard": "https://www.swebench.com/#multilingual", "dataset": "https://huggingface.co/datasets/SWE-bench/SWE-bench_Multilingual", "code": "https://github.com/SWE-bench/SWE-bench" }, "notes": "No standalone paper; introduced as a blog post (kabirk.com/multilingual, mirrored on swebench.com). Released month taken from the March 2025 blog post. Task instances are public GitHub issues predating training cutoffs.", "sota": { "system": "Gemini 3 Flash", "developer": "Google DeepMind", "value": 72.7, "date": "2026-02-13", "source": { "url": "https://www.swebench.com/#multilingual", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "bash-only", "scaffold": "mini-SWE-agent", "cost_usd_per_task": 0.35, "notes": "Top of the Multilingual bash-only leaderboard as of access date." } }, "result_count": 6 }, { "id": "swe-bench-pro", "name": "SWE-Bench Pro", "full_name": "SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?", "layer": "agent", "released": "2025-09", "maintainer": "Scale AI", "description": "1,865 human-verified, long-horizon software engineering tasks from 41 repositories (public set: 731 tasks from copyleft-licensed OSS repos; commercial set: 276 tasks from proprietary startup codebases; held-out set: 858 tasks). Each task gives an augmented problem statement, requirements and optional interface; the patch is scored by fail-to-pass and pass-to-pass tests. Reference solutions average about 107 changed lines across 4 files, so it measures multi-file, enterprise-style work that SWE-bench Verified no longer separates.", "description_zh": "1,865 个经人工核验的长程软件工程任务,来自 41 个代码库(公开集:来自 copyleft 许可开源库的 731 个任务;商业集:来自专有初创公司代码库的 276 个任务;保留集:858 个任务)。每个任务给出增强的问题陈述、需求和可选接口;补丁以 fail-to-pass 和 pass-to-pass 测试评分。参考解答平均改动约 107 行、跨 4 个文件,因此它衡量的是 SWE-bench Verified 已无法区分的多文件、企业级工作。", "domains": [ "software-engineering", "code", "tool-use" ], "metric": { "name": "resolve rate", "unit": "percent", "higher_is_better": true }, "task_count": 1865, "splits": [ { "name": "public", "task_count": 731, "note": "Open-source copyleft repositories; tracked on the Scale public leaderboard." }, { "name": "commercial", "task_count": 276, "note": "Private startup codebases; results published on a separate Scale leaderboard." }, { "name": "held-out", "task_count": 858, "note": "Additional copyleft repositories kept private for future analysis; scores not published." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": "swe-bench-verified", "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2509.16941", "website": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "leaderboard": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "dataset": "https://github.com/scaleapi/SWE-bench_Pro-os", "code": "https://github.com/scaleapi/SWE-bench_Pro-os" }, "notes": "Scale's current leaderboard runs models with an uncapped cost and a 250-turn limit; the paper-era runs (capped cost, 50 turns) are kept on a deprecated leaderboard and are not comparable. Rows marked with an asterisk on the leaderboard use the mini-swe-agent harness, the others SWE-Agent. Contamination is mitigated by GPL-style licensing rather than by secrecy, so 'medium'.", "sota": { "system": "Muse Spark 1.1", "developer": "Meta", "value": 61.5, "date": "2026-09-04", "source": { "url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "public", "scaffold": "mini-swe-agent", "notes": "Marked NEW on the leaderboard, +/-3.10 CI. Uncapped cost, 250-turn limit. No per-row date; date is the day observed." } }, "result_count": 6 }, { "id": "swe-bench-verified", "name": "SWE-bench Verified", "full_name": "SWE-bench Verified: human-validated subset of SWE-bench", "layer": "agent", "released": "2024-08", "maintainer": "Princeton NLP / OpenAI (curation)", "description": "500 real GitHub issues from 12 Python repositories, filtered by human annotators to remove under-specified or untestable tasks. An agent receives the issue text and the repository, must produce a patch, and is scored by hidden FAIL_TO_PASS and PASS_TO_PASS tests. The de-facto standard for agentic coding; scores depend heavily on scaffold and step/cost limits.", "description_zh": "500 个来自 12 个 Python 代码库的真实 GitHub issue,经人工标注者筛选以移除描述不足或无法测试的任务。智能体获得 issue 文本和代码库,需生成补丁,并以隐藏的 FAIL_TO_PASS 和 PASS_TO_PASS 测试评分。是智能体编程的事实标准;分数高度依赖脚手架及步数/成本限制。", "domains": [ "software-engineering", "code", "tool-use" ], "metric": { "name": "% resolved", "unit": "percent", "higher_is_better": true }, "task_count": 500, "splits": [ { "name": "bash-only", "task_count": 500, "note": "Official default view: every model in the same mini-SWE-agent environment." } ], "status": "saturating", "contamination_risk": "high", "human_baseline": null, "supersedes": "swe-bench", "superseded_by": "swe-bench-pro", "links": { "paper": "https://arxiv.org/abs/2310.06770", "website": "https://www.swebench.com", "leaderboard": "https://www.swebench.com/#verified", "dataset": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified", "code": "https://github.com/SWE-bench/SWE-bench" }, "notes": "Test instances are public and predate most frontier training cutoffs; treat developer-reported gains with care and prefer the official bash-only view or independent re-runs (SWE-rebench, Epoch AI) when comparing systems.", "sota": { "system": "Claude Opus 5", "developer": "Anthropic", "value": 96.0, "date": "2026-09-03", "source": { "url": "https://benchlm.ai/benchmarks/swe-bench-verified", "kind": "aggregator", "accessed": "2026-09-04" }, "conditions": { "notes": "Developer-reported number republished by aggregator; scaffold unspecified." } }, "result_count": 4 }, { "id": "tau-bench", "name": "tau-bench", "full_name": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "layer": "agent", "released": "2024-06", "maintainer": "Sierra Research (Yao et al.)", "description": "Simulated customer-service conversations in retail (115 tasks) and airline (50 tasks) domains where the agent must talk to an LLM-simulated user, call database APIs and follow a written policy. A task passes only if the final database state matches the annotated goal; pass^k measures whether the same task succeeds in all k independent trials, exposing agent inconsistency. The reference benchmark for policy-following tool agents.", "description_zh": "零售(115 个任务)和航空(50 个任务)领域的模拟客服对话,智能体需与 LLM 模拟的用户交谈、调用数据库 API 并遵守书面政策。仅当最终数据库状态与标注目标一致时任务才算通过;pass^k 衡量同一任务在全部 k 次独立试验中是否都成功,从而暴露智能体的不一致性。是遵循政策的工具智能体的参考基准。", "domains": [ "tool-use", "instruction-following", "general-assistant" ], "metric": { "name": "pass^1", "unit": "percent", "higher_is_better": true }, "task_count": 165, "splits": [ { "name": "retail", "task_count": 115 }, { "name": "airline", "task_count": 50 } ], "status": "retired", "contamination_risk": "high", "human_baseline": null, "supersedes": null, "superseded_by": "tau2-bench", "links": { "paper": "https://arxiv.org/abs/2406.12045", "website": "https://taubench.com", "leaderboard": "https://github.com/sierra-research/tau-bench", "dataset": "https://github.com/sierra-research/tau-bench", "code": "https://github.com/sierra-research/tau-bench" }, "notes": "The repository README now warns that its airline and retail tasks are outdated and directs users to the tau2-bench repository (tau^3-bench) for fixed tasks; developer reports of 'tau-bench' after mid-2025 usually mean tau2-bench. Scores depend on the user-simulator model, which is not standardised.", "sota": { "system": "TC (claude-3-5-sonnet-20241022), retail", "developer": "Anthropic", "value": 69.2, "date": "2024-10-22", "source": { "url": "https://github.com/sierra-research/tau-bench", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "retail", "scaffold": "function calling (TC)", "notes": "Repository README leaderboard; date is the model's release date (README gives none)." } }, "result_count": 4 }, { "id": "tau2-bench", "name": "tau2-bench", "full_name": "tau^2-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "layer": "agent", "released": "2025-06", "maintainer": "Sierra Research (Barres et al.)", "description": "Successor to tau-bench that adds a telecom troubleshooting domain (114 tasks) in which both the agent and the simulated user have tools, so the agent must coordinate and guide the user rather than act alone; the verified retail (115) and airline (50) domains are retained. Tasks are generated compositionally from atomic sub-tasks and scored on final database state; pass^k measures reliability across repeated trials. Now maintained as tau^3-bench with a banking knowledge domain and a voice modality.", "description_zh": "tau-bench 的继任者,新增电信故障排查领域(114 个任务),其中智能体和模拟用户都拥有工具,因此智能体必须协调并引导用户而非独自行动;保留了经核验的零售(115)和航空(50)领域。任务由原子子任务组合生成,并按最终数据库状态评分;pass^k 衡量重复试验间的可靠性。现以 tau^3-bench 形式维护,增加了银行知识领域和语音模态。", "domains": [ "tool-use", "instruction-following", "general-assistant" ], "metric": { "name": "pass^1", "unit": "percent", "higher_is_better": true }, "task_count": 279, "splits": [ { "name": "retail", "task_count": 115 }, { "name": "airline", "task_count": 50 }, { "name": "telecom", "task_count": 114, "note": "Dual-control domain; subsampled from 2,285 generated tasks." }, { "name": "core", "task_count": 279, "note": "taubench.com 'tau2-bench' text leaderboard: retail, airline and telecom combined." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": "tau-bench", "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2506.07982", "website": "https://taubench.com", "leaderboard": "https://taubench.com/leaderboard?benchmark=core", "dataset": "https://github.com/sierra-research/tau2-bench", "code": "https://github.com/sierra-research/tau2-bench" }, "notes": "Task fixes in February 2026 (tau^3-bench v1.0) and a July 2026 grading update mean pre- and post-fix scores are not strictly comparable; taubench.com re-graded affected submissions. Domains, tools and policies are public, but tasks are compositional and the user simulator adds variance.", "sota": { "system": "Qwen3.5-397B-A17B", "developer": "Alibaba Cloud", "value": 87.9, "date": "2026-09-04", "source": { "url": "https://taubench.com", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "core", "notes": "Top of the taubench.com tau2-bench text leaderboard as of access date. No per-row date; date is the day observed." } }, "result_count": 5 }, { "id": "terminal-bench", "name": "Terminal-Bench", "full_name": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces", "layer": "agent", "released": "2026-01", "maintainer": "Stanford / Laude Institute / Harbor (Merrill, Shaw et al.)", "description": "Hard, human-authored tasks that an agent must complete inside a real terminal environment (software engineering, scientific computing, ML, security, data processing). Each task ships its own container, a human solution and tests; the score is the fraction of tasks whose tests pass, averaged over several trials with 95% confidence intervals. The de-facto benchmark for CLI coding agents, run through the Harbor framework.", "description_zh": "由人工编写的高难任务,智能体需在真实终端环境中完成(软件工程、科学计算、ML、安全、数据处理)。每个任务自带容器、人类解答和测试;得分为测试通过的任务比例,对多次试验取平均并附 95% 置信区间。是 CLI 编程智能体的事实基准,通过 Harbor 框架运行。", "domains": [ "software-engineering", "code", "tool-use", "ml-engineering" ], "metric": { "name": "resolution rate", "unit": "percent", "higher_is_better": true }, "task_count": 66, "splits": [ { "name": "2.0", "task_count": 89, "note": "Terminal-Bench 2.0 (November 2025), the version described in the paper." }, { "name": "4.0", "task_count": 66, "note": "Current official leaderboard version; 5 trials per task (330 trials)." }, { "name": "science", "task_count": 70, "note": "Terminal-Bench-Science 0.1 (August 2026): separate Stanford-led task set of 70 expert-curated scientific research workflows (life, physical, Earth, mathematical, engineering sciences); 3 trials per task; own leaderboard at terminal-bench-science.ai. Not comparable to the 4.0 numbers." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2601.11868", "website": "https://www.tbench.ai", "leaderboard": "https://www.tbench.ai/leaderboard", "dataset": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/4?tab=tasks", "code": "https://github.com/laude-institute/terminal-bench" }, "notes": "Versioned benchmark: tasks are refreshed between releases (2.0 had 89 tasks, 4.0 has 66), so scores are only comparable within a version. Public tasks carry a canary string and the site asks that benchmark data never appear in training corpora. Terminal-Bench-Science (harbor-framework/terminal-bench-science, DOI 10.5281/zenodo.22110253) is a sibling benchmark in the same family and Harbor framework, tracked here as the 'science' split.", "sota": { "system": "GPT-6 Astra", "developer": "OpenAI", "value": 64.6, "date": "2026-09-03", "source": { "url": "https://openai.com/index/gpt-6-astra/", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "split": "science", "notes": "OpenAI launch table (Academic section), maximum at any effort; run in OpenAI's research environment, scaffold not stated. Same table quotes Fable 5.1 at 52.6% and Opus 5 at 30.0%." } }, "result_count": 11 }, { "id": "tool-decathlon", "name": "Tool Decathlon (Toolathlon)", "full_name": "The Tool Decathlon: Benchmarking Language Agents for Diverse, Realistic, and Long-Horizon Task Execution", "layer": "agent", "released": "2025-10", "maintainer": "HKUST NLP (Junlong Li, Junxian He et al.) with CMU / OpenHands collaborators", "description": "108 manually sourced tasks that require an agent to drive 32 real software applications through 604 tools (mostly MCP servers) such as Google Calendar, Notion, Canvas, WooCommerce, Kubernetes and BigQuery, starting from realistic seeded environment states and taking roughly 20 turns each. Every task is graded by a dedicated execution-based evaluation script on the resulting state; the leaderboard reports Pass@1 over three runs plus Pass@3 and Pass^3. It is the broadest public test of general MCP-style tool use across heterogeneous apps.", "description_zh": "108 个人工采集或构造的任务,要求智能体通过 604 个工具(主要为 MCP server)操作 32 个真实软件应用,如 Google Calendar、Notion、Canvas、WooCommerce、Kubernetes 和 BigQuery,从真实的初始环境状态出发,平均约 20 轮交互。每个任务由专用的基于执行的评测脚本对最终状态评分;榜单报告三次运行的 Pass@1 以及 Pass@3 和 Pass^3。是目前覆盖最广的通用 MCP 风格跨应用工具使用公开测试。", "domains": [ "tool-use", "general-assistant", "software-engineering" ], "metric": { "name": "Pass@1 (mean of 3 runs)", "unit": "percent", "higher_is_better": true }, "task_count": 108, "splits": [ { "name": "original", "task_count": 108, "note": "October 2025 release described in the ICLR 2026 paper; score series retired when Toolathlon-Verified launched." }, { "name": "verified", "task_count": 108, "note": "Toolathlon-Verified (2026-06-30): same 108 tasks with repaired instructions, initial states, ground truth and evaluators across 83 task packages. Starts a new official score series, not comparable to the original." } ], "status": "active", "contamination_risk": "medium", "human_baseline": null, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2510.25726", "website": "https://toolathlon.xyz/introduction", "leaderboard": "https://toolathlon.xyz/docs/leaderboard", "dataset": "https://huggingface.co/datasets/hkust-nlp/Toolathlon-Verified_Trajectories/tree/main", "code": "https://github.com/hkust-nlp/Toolathlon" }, "notes": "Rows marked with a check on the leaderboard are run by the Toolathlon team in the Default agent configuration, three runs each, with +/- standard deviation shown. Tasks touch live services (Canvas, Notion, WooCommerce, Google APIs), so results can drift with the services themselves; the Verified release added bounded retries and per-instance isolation to reduce this. Task files and evaluators are public on GitHub.", "sota": { "system": "GLM 5.3 Flash (max)", "developer": "Z.ai", "value": 78.4, "date": "2026-08-30", "source": { "url": "https://toolathlon.xyz/docs/leaderboard", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "split": "verified", "scaffold": "Default", "reasoning_effort": "max", "notes": "Rank 1 Pass@1 as of access date; +/-1.9 over 3 runs; Pass@3 / Pass^3 not yet published." } }, "result_count": 6 }, { "id": "truthfulqa", "name": "TruthfulQA", "full_name": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "layer": "model", "released": "2021-09", "maintainer": "University of Oxford / OpenAI (Lin, Hilton, Evans)", "description": "817 questions across 38 categories (health, law, finance, politics, conspiracies) crafted so that some humans would answer falsely because of common misconceptions. Models are scored either on free-form generation (judged truthful and informative by a fine-tuned judge or human raters) or on multiple-choice variants MC1 and MC2 (probability mass on true answers). It exposed the 'inverse scaling' of imitative falsehoods and remains a reference point for factuality and sycophancy work, though the questions are widely trained on.", "description_zh": "817 个问题,覆盖 38 个类别(健康、法律、金融、政治、阴谋论),精心设计使部分人类会因常见误解而答错。模型评分方式为自由生成(由微调评判模型或人类评分者判定真实且有信息量)或多项选择变体 MC1 和 MC2(真实答案上的概率质量)。它揭示了模仿性谬误的\"逆向缩放\",至今仍是事实性与迎合性研究的参考点,尽管题目已被广泛用于训练。", "domains": [ "factuality", "safety" ], "metric": { "name": "MC2 accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 817, "splits": [ { "name": "MC1", "task_count": 817, "note": "Single-correct-answer multiple choice." }, { "name": "MC2", "task_count": 817, "note": "Normalized probability on the set of true answers; the Open LLM Leaderboard v1 metric." }, { "name": "generation", "task_count": 817, "note": "Free-form answers judged for truthfulness and informativeness." } ], "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 94.0, "population": "one human participant answering the generation task (94% true, 87% true and informative)", "source": "https://arxiv.org/abs/2109.07958" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2109.07958", "website": "https://github.com/sylinrl/TruthfulQA", "leaderboard": null, "dataset": "https://huggingface.co/datasets/truthfulqa/truthful_qa", "code": "https://github.com/sylinrl/TruthfulQA" }, "notes": "The 94% human figure is a single participant on the generation task and is not comparable with MC2. The benchmark was retired from the Open LLM Leaderboard in mid-2024 because of saturation and contamination, and frontier vendors no longer report it, so the ledger holds only the paper baseline.", "sota": { "system": "GPT-3-175B (helpful prompt)", "developer": "OpenAI (evaluated by Lin et al.)", "value": 58.0, "date": "2021-09-08", "source": { "url": "https://arxiv.org/abs/2109.07958", "kind": "paper", "accessed": "2026-09-04" }, "conditions": { "split": "generation", "notes": "Human-judged % true answers (Fig. 4); the single human participant scored 94%. MC1/MC2 numbers for later models are not comparable with this row." } }, "result_count": 1 }, { "id": "visualwebarena", "name": "VisualWebArena", "full_name": "VisualWebArena: Evaluating Multimodal Agents on Realistic Visual Web Tasks", "layer": "agent", "released": "2024-01", "maintainer": "Carnegie Mellon University (Koh et al.)", "description": "910 web tasks on Classifieds, Shopping and Reddit sites that require reading images (product photos, listings, posted pictures) to complete, extending WebArena's functional evaluation with visually grounded checks. Agents receive screenshots (optionally Set-of-Marks annotated) plus the accessibility tree and must act in the browser. It isolates the multimodal-perception gap that text-only web agents cannot close.", "description_zh": "910 个在 Classifieds、Shopping 和 Reddit 站点上的网页任务,需要读取图像(商品照片、列表、发布的图片)才能完成,以视觉落地的检查扩展了 WebArena 的功能性评估。智能体接收截图(可选 Set-of-Marks 标注)和无障碍树,并需在浏览器中行动。它隔离考察纯文本网页智能体无法弥合的多模态感知差距。", "domains": [ "web", "multimodal", "computer-use" ], "metric": { "name": "success rate", "unit": "percent", "higher_is_better": true }, "task_count": 910, "status": "active", "contamination_risk": "high", "human_baseline": { "value": 88.7, "population": "human annotators (paper)", "source": "https://jykoh.com/vwa" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2401.13649", "website": "https://jykoh.com/vwa", "leaderboard": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/edit?gid=2044883967#gid=2044883967", "dataset": "https://github.com/web-arena-x/visualwebarena/tree/main/config_files", "code": "https://github.com/web-arena-x/visualwebarena" }, "notes": "Shares the WebArena maintainer Google Sheet (separate tab) and the same self-reporting caveats. Best systems are still more than 30 points below the human baseline.", "sota": { "system": "Gemini 2.5 Flash (SGV)", "developer": "Google DeepMind", "value": 54.0, "date": "2025-07-01", "source": { "url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/edit?gid=2044883967#gid=2044883967", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "scaffold": "SGV", "notes": "Top of the sheet as of access date. Month only; 1st used." } }, "result_count": 5 }, { "id": "webarena", "name": "WebArena", "full_name": "WebArena: A Realistic Web Environment for Building Autonomous Agents", "layer": "agent", "released": "2023-07", "maintainer": "Carnegie Mellon University (Zhou, Xu et al.)", "description": "812 long-horizon natural-language tasks on self-hosted, fully functional websites (e-commerce, forum, GitLab, CMS) plus a map, calculator and wiki. Agents act through a browser and are scored by programmatic functional checks on the final site state or answer rather than by matching action sequences. The first reproducible end-to-end web-agent benchmark and still the reference for text/DOM web agents.", "description_zh": "812 个长程自然语言任务,运行于自托管、功能完整的网站(电商、论坛、GitLab、CMS)以及地图、计算器和 wiki。智能体通过浏览器行动,以对最终站点状态或答案的程序化功能检查评分,而非匹配动作序列。是首个可复现的端到端网页智能体基准,至今仍是文本/DOM 网页智能体的参考。", "domains": [ "web", "tool-use", "computer-use" ], "metric": { "name": "success rate", "unit": "percent", "higher_is_better": true }, "task_count": 812, "status": "active", "contamination_risk": "high", "human_baseline": { "value": 78.24, "population": "human annotators (paper, end-to-end task success)", "source": "https://arxiv.org/abs/2307.13854" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/2307.13854", "website": "https://webarena.dev", "leaderboard": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/edit?usp=sharing", "dataset": "https://github.com/web-arena-x/webarena/blob/main/config_files/test.raw.json", "code": "https://github.com/web-arena-x/webarena" }, "notes": "Leaderboard is a maintainer-curated Google Sheet of self-reported results (trajectories required since September 2024). Environments are public Docker images, so task configs are widely available. Top systems now approach the 78% human figure; the maintainers point to WebArena-Infinity and TheAgentCompany as successors, neither of which is tracked here yet.", "sota": { "system": "WebTactix + Deepseek v3.2", "developer": "WebTactix", "value": 74.3, "date": "2026-02-01", "source": { "url": "https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ/edit?usp=sharing", "kind": "official-leaderboard", "accessed": "2026-09-04" }, "conditions": { "scaffold": "WebTactix", "notes": "Top of the maintainer sheet as of access date. Month only; 1st used." } }, "result_count": 5 }, { "id": "winogrande", "name": "WinoGrande", "full_name": "WinoGrande: An Adversarial Winograd Schema Challenge at Scale", "layer": "model", "released": "2019-07", "maintainer": "Allen Institute for AI (Sakaguchi et al.)", "description": "44,000 binary fill-in-the-blank pronoun-resolution problems inspired by the Winograd Schema Challenge, crowd-sourced and then debiased with the AfLite adversarial filter to remove word-association shortcuts. Scored as accuracy on the 1,267-item validation set (test labels are hidden). A long-running commonsense probe in pretraining evaluations; frontier models are within a few points of the 94% human figure.", "description_zh": "44,000 个受 Winograd Schema Challenge 启发的二选一填空代词消解问题,众包构建后经 AfLite 对抗式过滤去偏,以消除词语关联捷径。以 1,267 项验证集上的准确率评分(测试集标签隐藏)。是预训练评估中长期使用的常识探针;前沿模型与 94% 的人类水平仅差几分。", "domains": [ "commonsense", "reasoning" ], "metric": { "name": "accuracy", "unit": "percent", "higher_is_better": true }, "task_count": 1267, "status": "saturated", "contamination_risk": "high", "human_baseline": { "value": 94.0, "population": "crowd workers (paper Table 3, test set)", "source": "https://arxiv.org/abs/1907.10641" }, "supersedes": null, "superseded_by": null, "links": { "paper": "https://arxiv.org/abs/1907.10641", "website": "https://winogrande.allenai.org/", "leaderboard": null, "dataset": "https://huggingface.co/datasets/allenai/winogrande", "code": "https://github.com/allenai/winogrande" }, "notes": "Most harnesses report 5-shot accuracy on the validation split of the 'xl' (40k-train) configuration.", "sota": { "system": "DeepSeek-V4-Pro-Base", "developer": "DeepSeek", "value": 81.5, "date": "2026-04-22", "source": { "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "kind": "developer-report", "accessed": "2026-09-04" }, "conditions": { "shots": 0, "notes": "DeepSeek-V4 model card, base-model table (HF repo created 2026-04-22). Base model, exact match, zero-shot." } }, "result_count": 2 } ] }