[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-tests-whether-ai-science-agents-actually-learn":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},10620,"new-benchmark-tests-whether-ai-science-agents-actually-learn","New Benchmark Tests Whether AI Science Agents Actually Learn","ScienceClaw benchmarks 23 disciplines to see if AI agents' one-off fixes turn into lasting skills, not just lucky one-time wins.","A new benchmark asks a blunt question: when an AI agent fixes a broken science workflow, does it actually get better, or does it just get lucky once?\n\nResearchers behind ScienceClaw built a framework and a companion benchmark, ScienceClaw-Eval, spanning 23 disciplines across the natural and social sciences. The system treats improvement as fixed-parameter program self-evolution, meaning the agent cannot simply retrain its weights; it has to repair and update its own executable workflows through multi-turn interaction. A fix only counts as a real update if replaying the original failing task reproduces the repair, and the change also helps on separate, independent tasks. ScienceClaw-Eval then tracks correctness, evolutionary gain, retention, cross-dataset transfer, and evolution cost across sequential task streams.\n\nMost claims that AI science agents 'get better over time' rest on anecdote, a single success story rather than a measured trajectory. By forcing every update to survive replay and independent testing, ScienceClaw sets a higher bar than most coding-agent benchmarks, which usually just check whether a task passed once. That distinction matters because science agents are being pitched as lab collaborators that accumulate expertise, not disposable one-shot tools.\n\nThe code is open-source on GitHub, so other labs can test the retention claims themselves instead of taking the paper's word for it.","[\"ai agents\",\"benchmarking\",\"ai for science\",\"research\"]","2026-10-07T04:00:00.000Z","2026-10-08T23:19:14.551Z","2026-10-08T23:19:19.797Z","published",null,[],"ai",[26,27,28,29],"ai agents","benchmarking","ai for science","research",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.08691",0,{"sections":36},[37,41,46,51,56,61,66,71,76,80,85,90,95,100],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",6448,"2026-10-07T18:45:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":45},"Security","security",904,"2026-10-07T19:53:42.000Z",{"name":47,"slug":48,"count":49,"latest_published_at":50},"Policy","policy",474,"2026-10-07T18:23:21.000Z",{"name":52,"slug":53,"count":54,"latest_published_at":55},"Deals","deals",453,"2026-10-07T23:58:31.000Z",{"name":57,"slug":58,"count":59,"latest_published_at":60},"Hardware","hardware",222,"2026-10-07T21:19:54.000Z",{"name":62,"slug":63,"count":64,"latest_published_at":65},"Science","science",186,"2026-10-06T21:20:39.000Z",{"name":67,"slug":68,"count":69,"latest_published_at":70},"Consumer Tech","consumer-tech",174,"2026-10-07T17:41:41.000Z",{"name":72,"slug":73,"count":74,"latest_published_at":75},"Software","software",113,"2026-10-07T18:10:00.000Z",{"name":77,"slug":78,"count":74,"latest_published_at":79},"Startups","startups","2026-10-07T23:36:57.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Dev Tools","dev-tools",105,"2026-10-07T16:59:11.000Z",{"name":86,"slug":87,"count":88,"latest_published_at":89},"General","general",61,"2026-10-07T22:00:24.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"Gaming","gaming",56,"2026-10-07T12:00:00.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",33,"2026-10-05T11:57:17.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]