[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-tackles-ai-forecastings-leakage-problem":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},10118,"new-benchmark-tackles-ai-forecastings-leakage-problem","New Benchmark Tackles AI Forecasting's Leakage Problem","LEAF, a new living benchmark, slashes future-information leakage in AI forecasting evaluations from 8.6% to 1.6% and tests whether 16 models can forecast.","A new benchmark says most AI forecasting tests have been cheating without telling anyone.\n\nResearchers built LEAF, a benchmark for testing how well large language models predict trends, events, and time series when fed outside news and data feeds. The core problem: automated web searches during evaluation often surface articles published after the event being predicted, letting a model \"forecast\" something it already read about. LEAF's fix is a two-agent system - one agent retrieves context, a second cross-checks it for future leaks - paired with a recursive retrieval setup meant to keep auxiliary material temporally honest. The team says a manual audit of 500 tasks by 47 domain specialists found this pipeline cut leakage from 8.6% of cases down to 1.6%.\n\nContaminated benchmarks are a quiet problem in AI evaluation - a model that scores well because it peeked at the answer looks identical, on a leaderboard, to one that actually reasoned its way there. By testing 16 proprietary and open-weight models and finding they genuinely extract useful signal from verified, leak-checked events, LEAF offers a cleaner read on whether LLMs can do real forecasting rather than retrieval with extra steps.\n\n\"Living\" is the operative word here: if LEAF's event pool doesn't keep refreshing, it risks becoming exactly the kind of stale, memorizable test it was built to replace.","[\"ai-benchmarks\",\"forecasting\",\"llm-evaluation\",\"data-contamination\"]","2026-10-05T04:00:00.000Z","2026-10-06T00:00:44.032Z","2026-10-06T00:00:50.599Z","published",null,[],"ai",[26,27,28,29],"ai-benchmarks","forecasting","llm-evaluation","data-contamination",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.16358",0,{"sections":36},[37,41,45,50,55,60,64,69,74,79,84,89,94,99],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",6317,"2026-10-05T09:51:57.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":18},"Security","security",871,{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",446,"2026-10-05T10:25:00.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",340,"2026-10-05T09:18:03.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",205,"2026-10-05T10:58:22.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":18},"Science","science",179,{"name":65,"slug":66,"count":67,"latest_published_at":68},"Consumer Tech","consumer-tech",160,"2026-10-05T10:23:15.000Z",{"name":70,"slug":71,"count":72,"latest_published_at":73},"Dev Tools","dev-tools",99,"2026-10-05T10:47:06.000Z",{"name":75,"slug":76,"count":77,"latest_published_at":78},"Software","software",97,"2026-10-04T10:00:00.000Z",{"name":80,"slug":81,"count":82,"latest_published_at":83},"Startups","startups",93,"2026-10-05T11:13:51.000Z",{"name":85,"slug":86,"count":87,"latest_published_at":88},"Gaming","gaming",53,"2026-10-02T02:50:39.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",51,"2026-10-05T02:35:01.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",32,"2026-10-02T18:00:00.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]