[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ai-papers-keep-benchmarking-yesterdays-models":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},7825,"ai-papers-keep-benchmarking-yesterdays-models","AI Papers Keep Benchmarking Yesterday's Models","A study of 18,574 AI evaluations finds researchers routinely test models that already trail the frontier, and the gap is widening every year.","Most AI benchmarks in academic papers are testing models that were already old news by the time the paper came out.\n\nResearchers combed OpenAlex, the academic literature database, for papers mentioning large language models published between January 2022 and April 2026 - 112,303 hits, narrowed to 18,574 usable evaluations. They scored each tested model against the frontier of its era using Epoch AI's Capabilities Index, a composite benchmark score. The median paper tested a model already trailing the frontier by 10.85 points on that index, a gap growing by roughly 5.5 points a year. Only 18.4% of papers even stated when the evaluation happened, and just 3.2% of abstracts on reasoning models disclosed whether reasoning mode was switched on.\n\nThat matters because readers, reviewers, and policymakers tend to read a result attached to a named model as a verdict on current AI capability, not a snapshot from a year or two back. More than half the abstracts audited made claims at the level of \"AI\" in general rather than naming the specific model tested, which papers over exactly the gap the study is flagging.\n\nThe authors' proposed fix - a 13-item disclosure checklist called VERSIO-AI v1.2, backed by pressure from editors and funders - is a reasonable start, though good luck getting time-strapped peer reviewers to enforce a checklist that authors have every incentive to skip.","[\"ai-research\",\"benchmarks\",\"academic-publishing\",\"llm-evaluation\"]","2026-09-25T04:00:00.000Z","2026-09-26T01:01:52.354Z","2026-09-26T01:01:58.542Z","published",null,[],"ai",[26,27,28,29],"ai-research","benchmarks","academic-publishing","llm-evaluation",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.04135",0,{"sections":36},[37,41,46,51,56,61,66,71,76,81,86,91,96,101],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",4527,"2026-09-25T16:31:14.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":45},"Security","security",741,"2026-09-25T15:52:13.000Z",{"name":47,"slug":48,"count":49,"latest_published_at":50},"Policy","policy",390,"2026-09-25T16:24:59.000Z",{"name":52,"slug":53,"count":54,"latest_published_at":55},"Deals","deals",256,"2026-09-25T17:00:53.000Z",{"name":57,"slug":58,"count":59,"latest_published_at":60},"Hardware","hardware",185,"2026-09-25T15:00:22.000Z",{"name":62,"slug":63,"count":64,"latest_published_at":65},"Science","science",140,"2026-09-25T11:55:23.000Z",{"name":67,"slug":68,"count":69,"latest_published_at":70},"Consumer Tech","consumer-tech",132,"2026-09-25T15:30:00.000Z",{"name":72,"slug":73,"count":74,"latest_published_at":75},"Software","software",88,"2026-09-24T23:06:55.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Dev Tools","dev-tools",82,"2026-09-25T09:59:40.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Startups","startups",76,"2026-09-25T18:33:59.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"Gaming","gaming",48,"2026-09-25T18:35:21.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"General","general",46,"2026-09-25T02:12:57.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"Reviews","reviews",30,"2026-09-24T20:07:31.000Z",{"name":102,"slug":103,"count":104,"latest_published_at":105},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]