[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-study-finds-most-llm-benchmark-gains-are-noise":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},7545,"study-finds-most-llm-benchmark-gains-are-noise","Study Finds Most LLM Benchmark Gains Are Noise","A new study finds most claimed LLM benchmark gains fall below the scoring method's own noise floor, though pairwise judging offers a partial fix.","Most year-over-year gains on LLM benchmarks may be measurement noise, not real progress.\n\nResearchers analyzed 373,019 LLM-judge scores across benchmarks using generalizability theory, decomposing variance into system, item, judge, and interaction components. They found that pointwise rubric scoring, rating a single output on a scale, hits a hard reliability ceiling no matter how many test items you add, because judge-system interaction noise doesn't shrink with sample size. Switching to pairwise comparison, judging two outputs head-to-head in both presentation orders, cuts that noise by two orders of magnitude and pushes the ceiling to 0.986. But pairwise comparison introduces its own bias: a system shown first wins 8.6 percentage points more often than the same system shown second, a bias larger than the median improvement claimed across 53 published win-rate papers the researchers reviewed.\n\nThe measured noise floor, 0.41 to 1.24 points on a 0-5 scale, dwarfs the median reported gain of 0.28 points. On MT-Bench specifically, all 17 recovered improvements the researchers could match fell below the benchmark's own floor, and 70% of win-rate claims fell below the pairwise floor. A lot of leaderboard climbing, in other words, may not be real.\n\nThe kicker: an audit of 628 arXiv papers found fewer than one in four even state whether their evaluation was run more than once.","[\"llm-benchmarks\",\"ai-evaluation\",\"llm-judges\",\"research\"]","2026-09-24T04:00:00.000Z","2026-09-24T05:33:32.846Z","2026-09-24T05:33:38.969Z","published",null,[],"ai",[26,27,28,29],"llm-benchmarks","ai-evaluation","llm-judges","research",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.27787",0,{"sections":36},[37,41,46,51,56,61,66,71,76,81,86,91,95,100],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",4466,"2026-09-25T04:00:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":45},"Security","security",729,"2026-09-24T19:54:21.000Z",{"name":47,"slug":48,"count":49,"latest_published_at":50},"Policy","policy",386,"2026-09-24T23:50:55.000Z",{"name":52,"slug":53,"count":54,"latest_published_at":55},"Deals","deals",237,"2026-09-24T22:00:00.000Z",{"name":57,"slug":58,"count":59,"latest_published_at":60},"Hardware","hardware",182,"2026-09-25T01:25:53.000Z",{"name":62,"slug":63,"count":64,"latest_published_at":65},"Science","science",138,"2026-09-24T18:24:52.000Z",{"name":67,"slug":68,"count":69,"latest_published_at":70},"Consumer Tech","consumer-tech",128,"2026-09-24T19:24:34.000Z",{"name":72,"slug":73,"count":74,"latest_published_at":75},"Software","software",88,"2026-09-24T23:06:55.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Dev Tools","dev-tools",79,"2026-09-22T22:21:13.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Startups","startups",71,"2026-09-24T20:45:00.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"Gaming","gaming",46,"2026-09-24T17:52:29.000Z",{"name":92,"slug":93,"count":89,"latest_published_at":94},"General","general","2026-09-25T02:12:57.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",30,"2026-09-24T20:07:31.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]