[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-researchers-propose-backtesting-to-grade-ai-research-questions":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},5489,"researchers-propose-backtesting-to-grade-ai-research-questions","Researchers Propose Backtesting to Grade AI Research Questions","A new protocol scores AI-generated research questions against years of later literature, revealing the bigger problem is disagreement among human judges.","A new benchmark judges AI research-question generators the old way: by waiting years to see if reality proves them right.\n\nResearchers have built a 'historical backtesting' protocol for scoring systems that generate scientific research questions. Instead of relying on expert opinions or LLM-as-judge ratings, the method freezes a question-generating system's output at a historical cutoff date, before it has seen any later literature, then checks a temporally isolated future corpus to see whether each question was later answered, partially addressed, independently asked by someone else, or simply ignored. The team released a full astronomy pilot: frozen corpora, frozen questions, audited outcome labels, four baseline generators, and a submission interface so other systems can be tested the same way. Running the test across cutoffs from 2010 to 2024 on 798 questions, with the final window landing after the underlying models' training cutoff to rule out memorization, they found that a generator using no model weights at all correctly flagged questions whose underlying premises the future would go on to refute, while pure LLM prompting mostly recycled topics that already looked important without showing real foresight.\n\nThe more interesting result is not about the AI, it is about the humans checking its work. Two trained human annotators agreed with each other on outcome labels at a kappa of just 0.17, barely above chance, while five different judge models matched a professional annotator's calls just as well or better, in the 0.17 to 0.26 range. Meanwhile the judge models agreed with each other at 0.60, high enough that grading an AI judge only against other AI judges would have overstated its reliability by roughly three times.\n\nIt's a blunt fix to a familiar problem: most AI benchmarks get gamed or contaminated within a year of release. This one is built so the answer key doesn't exist yet, with 200 new questions frozen on August 17, 2026, that won't be scored until 2027 through 2030.","[\"ai\",\"science\",\"benchmarks\",\"astronomy\"]","2026-08-18T04:00:00.000Z","2026-08-18T22:22:08.964Z","2026-08-18T22:22:20.888Z","published",null,[],"ai",[24,26,27,28],"science","benchmarks","astronomy",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2608.16795",0,{"sections":35},[36,40,44,49,54,59,64,68,73,77,82,87,92,97],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",3293,"2026-08-20T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":39},"Security","security",435,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",210,"2026-08-19T09:32:27.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",140,"2026-08-19T18:25:42.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":63},"Consumer Tech","consumer-tech",95,"2026-08-18T16:05:00.000Z",{"name":65,"slug":26,"count":66,"latest_published_at":67},"Science",90,"2026-08-19T18:41:02.000Z",{"name":69,"slug":70,"count":71,"latest_published_at":72},"Software","software",73,"2026-08-18T07:51:50.000Z",{"name":74,"slug":75,"count":76,"latest_published_at":18},"Dev Tools","dev-tools",69,{"name":78,"slug":79,"count":80,"latest_published_at":81},"Startups","startups",47,"2026-08-19T19:13:46.000Z",{"name":83,"slug":84,"count":85,"latest_published_at":86},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":88,"slug":89,"count":90,"latest_published_at":91},"General","general",33,"2026-08-18T22:18:13.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]