{
  "project": "llm-benchmarks-tracker",
  "url": "https://alloevil.github.io/llm-benchmarks-tracker/",
  "repository": "https://github.com/alloevil/llm-benchmarks-tracker",
  "updated": "2026-09-16",
  "claims": [
    {
      "id": "benchmarks-catalogued",
      "claim": "The catalogue describes 54 evaluation benchmarks: 31 model benchmarks and 23 agent benchmarks.",
      "value": "54",
      "metric": "benchmark metadata files that pass schema/benchmark.schema.json",
      "method": "scripts/dataset.py::load validates every data/benchmarks/*.json against the published schema and counts what loads",
      "repro": "python3 scripts/validate.py",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/benchmarks",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 scripts/validate.py",
        "expect": {
          "regex": "^54 benchmarks,"
        },
        "timeout": 60
      }
    },
    {
      "id": "benchmark-layers",
      "claim": "The 54 benchmarks split into 31 model benchmarks and 23 agent benchmarks.",
      "value": "31/23",
      "metric": "benchmark metadata files by layer, model benchmarks first",
      "method": "layer field per benchmark file; the receipt recounts it straight from data/benchmarks/*.json",
      "repro": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];L=[b['layer'] for b in B];print(L.count('model'), L.count('agent'), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/benchmarks",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];L=[b['layer'] for b in B];print(L.count('model'), L.count('agent'), sep='/')\"",
        "expect": {
          "equals": "31/23"
        },
        "timeout": 60
      }
    },
    {
      "id": "evaluators-catalogued",
      "claim": "The catalogue describes 18 evaluators: frameworks, leaderboards, independent evaluators and aggregators.",
      "value": "18",
      "metric": "evaluator metadata files that pass schema/evaluator.schema.json",
      "method": "scripts/dataset.py::load validates every data/evaluators/*.json against the published schema and counts what loads",
      "repro": "python3 scripts/validate.py",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/evaluators",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 scripts/validate.py",
        "expect": {
          "regex": ", 18 evaluators$"
        },
        "timeout": 60
      }
    },
    {
      "id": "sourced-results",
      "claim": "All 252 result rows carry a source URL, a source kind and an access date.",
      "value": "252/252",
      "metric": "result rows with source.url, source.kind and source.accessed present",
      "method": "schema/results.schema.json marks all three fields required; scripts/validate.py fails the build on a row without them",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(1 for r in R if r['source'].get('url') and r['source'].get('kind') and r['source'].get('accessed')), len(R), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/blob/main/schema/results.schema.json",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(1 for r in R if r['source'].get('url') and r['source'].get('kind') and r['source'].get('accessed')), len(R), sep='/')\"",
        "expect": {
          "equals": "252/252"
        },
        "timeout": 60
      }
    },
    {
      "id": "data-as-of",
      "claim": "The newest access date across the 252 result rows is 2026-09-13, which is the date the README publishes as `data as of`.",
      "value": "2026-09-13",
      "metric": "newest source.accessed date over all result rows",
      "method": "scripts/build.py::data_as_of takes the maximum access date, so the README stamp moves with the data and not with the build date; the receipt recomputes that maximum from data/results/*.json",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(max(r['source'].get('accessed') or r['date'] for r in R))\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(max(r['source'].get('accessed') or r['date'] for r in R))\"",
        "expect": {
          "equals": "2026-09-13"
        },
        "timeout": 60
      }
    },
    {
      "id": "top-score-provenance",
      "claim": "52% of the current top scores come from an official leaderboard, a paper or an independent evaluation rather than a vendor self-report or an aggregator.",
      "value": "52%",
      "metric": "share of per-benchmark top scores whose source.kind is official-leaderboard, paper or independent-evaluation",
      "method": "scripts/build.py::provenance_share over Dataset.sota() of every benchmark; the receipt picks the top row again from each ledger (higher_is_better, ties to the earliest date) and counts the trusted kinds",
      "repro": "python3 -c \"import json, glob;S=[(b, json.load(open('data/results/' + b['id'] + '.json'))['results']) for b in [json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')]];S=[(b, r) for b, r in S if r];print(round(100 * sum(1 for b, r in S if min(r, key=lambda x: (-(1 if b['metric']['higher_is_better'] else -1) * x['value'], x['date']))['source']['kind'] in ('official-leaderboard', 'paper', 'independent-evaluation')) / len(S)), '%', sep='')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;S=[(b, json.load(open('data/results/' + b['id'] + '.json'))['results']) for b in [json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')]];S=[(b, r) for b, r in S if r];print(round(100 * sum(1 for b, r in S if min(r, key=lambda x: (-(1 if b['metric']['higher_is_better'] else -1) * x['value'], x['date']))['source']['kind'] in ('official-leaderboard', 'paper', 'independent-evaluation')) / len(S)), '%', sep='')\"",
        "expect": {
          "equals": "52%"
        },
        "timeout": 60
      }
    },
    {
      "id": "results-official-leaderboard",
      "claim": "104 of the 252 result rows have source kind official-leaderboard.",
      "value": "104/252",
      "metric": "result rows whose source.kind is official-leaderboard",
      "method": "count over data/results/*.json after schema validation",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'official-leaderboard' for r in R), len(R), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'official-leaderboard' for r in R), len(R), sep='/')\"",
        "expect": {
          "equals": "104/252"
        },
        "timeout": 60
      }
    },
    {
      "id": "results-paper",
      "claim": "64 of the 252 result rows have source kind paper.",
      "value": "64/252",
      "metric": "result rows whose source.kind is paper",
      "method": "count over data/results/*.json after schema validation",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'paper' for r in R), len(R), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'paper' for r in R), len(R), sep='/')\"",
        "expect": {
          "equals": "64/252"
        },
        "timeout": 60
      }
    },
    {
      "id": "results-developer-report",
      "claim": "55 of the 252 result rows have source kind developer-report.",
      "value": "55/252",
      "metric": "result rows whose source.kind is developer-report",
      "method": "count over data/results/*.json after schema validation",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'developer-report' for r in R), len(R), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'developer-report' for r in R), len(R), sep='/')\"",
        "expect": {
          "equals": "55/252"
        },
        "timeout": 60
      }
    },
    {
      "id": "results-independent-evaluation",
      "claim": "16 of the 252 result rows have source kind independent-evaluation.",
      "value": "16/252",
      "metric": "result rows whose source.kind is independent-evaluation",
      "method": "count over data/results/*.json after schema validation",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'independent-evaluation' for r in R), len(R), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'independent-evaluation' for r in R), len(R), sep='/')\"",
        "expect": {
          "equals": "16/252"
        },
        "timeout": 60
      }
    },
    {
      "id": "results-aggregator",
      "claim": "13 of the 252 result rows have source kind aggregator.",
      "value": "13/252",
      "metric": "result rows whose source.kind is aggregator",
      "method": "count over data/results/*.json after schema validation",
      "repro": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'aggregator' for r in R), len(R), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;R=[r for p in glob.glob('data/results/*.json') for r in json.load(open(p))['results']];print(sum(r['source']['kind'] == 'aggregator' for r in R), len(R), sep='/')\"",
        "expect": {
          "equals": "13/252"
        },
        "timeout": 60
      }
    },
    {
      "id": "top-score-benchcad",
      "claim": "The README's Top score for BenchCAD is 0.959 (vision2code-tools), from GPT-6 Astra (aggregator), the best row in its ledger.",
      "value": "0.959 (vision2code-tools)",
      "metric": "highest value in data/results/benchcad.json under the benchmark's metric, rendered as the README cell",
      "method": "the receipt picks the top row again (value, ties to the earliest date), formats it with the benchmark's metric unit, appends conditions.split because this ledger carries rows on more than one split, and requires the README row for this benchmark to carry the same cell",
      "repro": "python3 -c \"import json, sys;B=json.load(open('data/benchmarks/benchcad.json'));R=json.load(open('data/results/benchcad.json'))['results'];T=min(R, key=lambda r: (-r['value'] if B['metric']['higher_is_better'] else r['value'], r['date']));V=('%g' % T['value']) + ('%' if B['metric']['unit'] == 'percent' else '');S={(r.get('conditions') or {}).get('split') for r in R} - {None};V+=(' (%s)' % T['conditions']['split']) if len(S) > 1 and (T.get('conditions') or {}).get('split') else '';C=[l for l in open('README.md', encoding='utf-8') if l.startswith('| [%s]' % B['name'])][0].split('|')[5].strip();print('%s: top %s · %s · %s · README cell %s' % (B['name'], V, T['system'], T['source']['kind'], C));sys.exit(0 if V == C else 1)\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/blob/main/README.md#model-benchmarks + https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results/benchcad.json",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, sys;B=json.load(open('data/benchmarks/benchcad.json'));R=json.load(open('data/results/benchcad.json'))['results'];T=min(R, key=lambda r: (-r['value'] if B['metric']['higher_is_better'] else r['value'], r['date']));V=('%g' % T['value']) + ('%' if B['metric']['unit'] == 'percent' else '');S={(r.get('conditions') or {}).get('split') for r in R} - {None};V+=(' (%s)' % T['conditions']['split']) if len(S) > 1 and (T.get('conditions') or {}).get('split') else '';C=[l for l in open('README.md', encoding='utf-8') if l.startswith('| [%s]' % B['name'])][0].split('|')[5].strip();print('%s: top %s · %s · %s · README cell %s' % (B['name'], V, T['system'], T['source']['kind'], C));sys.exit(0 if V == C else 1)\"",
        "expect": {
          "equals": "BenchCAD: top 0.959 (vision2code-tools) · GPT-6 Astra · aggregator · README cell 0.959 (vision2code-tools)"
        },
        "timeout": 60
      }
    },
    {
      "id": "top-score-osworld-2",
      "claim": "The README's Top score for OSWorld 2.0 is 72.6% (offline), from GPT-6 Astra (self-reported), the best row in its ledger.",
      "value": "72.6% (offline)",
      "metric": "highest value in data/results/osworld-2.json under the benchmark's metric, rendered as the README cell",
      "method": "the receipt picks the top row again (value, ties to the earliest date), formats it with the benchmark's metric unit, appends conditions.split because this ledger carries rows on more than one split, and requires the README row for this benchmark to carry the same cell",
      "repro": "python3 -c \"import json, sys;B=json.load(open('data/benchmarks/osworld-2.json'));R=json.load(open('data/results/osworld-2.json'))['results'];T=min(R, key=lambda r: (-r['value'] if B['metric']['higher_is_better'] else r['value'], r['date']));V=('%g' % T['value']) + ('%' if B['metric']['unit'] == 'percent' else '');S={(r.get('conditions') or {}).get('split') for r in R} - {None};V+=(' (%s)' % T['conditions']['split']) if len(S) > 1 and (T.get('conditions') or {}).get('split') else '';C=[l for l in open('README.md', encoding='utf-8') if l.startswith('| [%s]' % B['name'])][0].split('|')[5].strip();print('%s: top %s · %s · %s · README cell %s' % (B['name'], V, T['system'], T['source']['kind'], C));sys.exit(0 if V == C else 1)\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/blob/main/README.md#agent-benchmarks + https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/results/osworld-2.json",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, sys;B=json.load(open('data/benchmarks/osworld-2.json'));R=json.load(open('data/results/osworld-2.json'))['results'];T=min(R, key=lambda r: (-r['value'] if B['metric']['higher_is_better'] else r['value'], r['date']));V=('%g' % T['value']) + ('%' if B['metric']['unit'] == 'percent' else '');S={(r.get('conditions') or {}).get('split') for r in R} - {None};V+=(' (%s)' % T['conditions']['split']) if len(S) > 1 and (T.get('conditions') or {}).get('split') else '';C=[l for l in open('README.md', encoding='utf-8') if l.startswith('| [%s]' % B['name'])][0].split('|')[5].strip();print('%s: top %s · %s · %s · README cell %s' % (B['name'], V, T['system'], T['source']['kind'], C));sys.exit(0 if V == C else 1)\"",
        "expect": {
          "equals": "OSWorld 2.0: top 72.6% (offline) · GPT-6 Astra · developer-report · README cell 72.6% (offline)"
        },
        "timeout": 60
      }
    },
    {
      "id": "no-longer-discriminative",
      "claim": "17 of the 54 benchmarks are recorded as saturated or retired, so they no longer separate frontier systems.",
      "value": "17/54",
      "metric": "benchmarks whose status is saturated or retired",
      "method": "status field per benchmark file; active and saturating count as live",
      "repro": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];print(sum(b['status'] in ('saturated', 'retired') for b in B), len(B), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/benchmarks",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];print(sum(b['status'] in ('saturated', 'retired') for b in B), len(B), sep='/')\"",
        "expect": {
          "equals": "17/54"
        },
        "timeout": 60
      }
    },
    {
      "id": "contamination-high",
      "claim": "24 of the 54 benchmarks have a public, static, widely scraped test set.",
      "value": "24/54",
      "metric": "benchmarks whose contamination_risk is high",
      "method": "contamination_risk field per benchmark file",
      "repro": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];print(sum(b.get('contamination_risk') == 'high' for b in B), len(B), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/benchmarks",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];print(sum(b.get('contamination_risk') == 'high' for b in B), len(B), sep='/')\"",
        "expect": {
          "equals": "24/54"
        },
        "timeout": 60
      }
    },
    {
      "id": "human-baselines",
      "claim": "19 of the 54 benchmarks record a measured human baseline with a source.",
      "value": "19/54",
      "metric": "benchmarks with a non-null human_baseline (value, population, source)",
      "method": "human_baseline is only populated from a measured number with a citation; guesses are left null",
      "repro": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];print(sum(1 for b in B if b.get('human_baseline')), len(B), sep='/')\"",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/tree/main/data/benchmarks",
      "verified": "2026-09-16",
      "check": {
        "cmd": "python3 -c \"import json, glob;B=[json.load(open(p)) for p in glob.glob('data/benchmarks/*.json')];print(sum(1 for b in B if b.get('human_baseline')), len(B), sep='/')\"",
        "expect": {
          "equals": "19/54"
        },
        "timeout": 60
      }
    },
    {
      "id": "source-pages-not-re-read",
      "claim": "No receipt fetches a source page: all 252 rows state the number the page carried when it was read, and the gate only proves the committed value is the one the repository publishes.",
      "value": "0/252",
      "metric": "result rows whose source.url is fetched again while the gate runs",
      "method": "the receipts are offline recounts of data/; re-reading the pages is scripts/sync_ledgers.py's job for the four allow-listed machine-readable sources and a person's job for the rest, and source.accessed records when either last did it",
      "repro": "python scripts/sync_ledgers.py --dry-run  # needs the network; CI runs it twice a week",
      "evidence": "https://github.com/alloevil/llm-benchmarks-tracker/blob/main/scripts/sync_ledgers.py",
      "verified": "2026-09-16",
      "check": {
        "manual": "re-reading a source page needs the network, and not all of them answer a CI host (openai.com returned 403 for the OSWorld 2.0 row); what a receipt proves is that the published figure is the figure in the committed ledger, not that the upstream page still agrees"
      }
    }
  ]
}