[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-shows-lab-robots-fail-half-their-experiments":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},8200,"new-benchmark-shows-lab-robots-fail-half-their-experiments","New Benchmark Shows Lab Robots Fail Half Their Experiments","A new tool auto-generates verifiable lab tasks from written protocols, and the best AI system tested still succeeds only about half the time.","Researchers built a system that turns written lab instructions into robot test tasks - and even the best AI today fails roughly half of them.\n\nThe tool, called SciHorizon-eLab, takes a natural-language description of a scientific experiment and automatically compiles it into a runnable, checkable robot task. It does this in stages: grounding the protocol's language in a simulated lab environment, generating an executable manipulation program, and certifying the result through multi-step simulation checks. The team used the pipeline to build a 300-task benchmark spanning common lab operations, complete with expert demonstrations and step-by-step scoring. In tests, the strongest AI policy averaged a 49.7% success rate, with particular trouble when tasks required coordinating with a human or another robot.\n\nThe real contribution here isn't the robots - it's the compiler. Building test scenarios for lab automation has always meant engineers hand-crafting each task, which caps how fast benchmarks can grow. Turning protocols directly into verifiable tasks removes that bottleneck, and could do for lab robotics what synthetic test generation has done for software.\n\nStill, a coin-flip success rate on tasks the system itself curated is a useful reality check. Coordination failures, not raw manipulation, are the sticking point - which is exactly the skill an unsupervised lab assistant would need most.","[\"ai agents\",\"robotics\",\"benchmarks\",\"embodied ai\"]","2026-09-28T04:00:00.000Z","2026-09-28T16:35:48.931Z","2026-09-28T16:35:54.021Z","published",null,[],"ai",[26,27,28,29],"ai agents","robotics","benchmarks","embodied ai",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.30971",0,{"sections":36},[37,40,44,49,54,59,63,68,73,78,83,88,92,97],{"name":38,"slug":24,"count":39,"latest_published_at":18},"AI",4844,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",762,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",399,"2026-09-27T18:39:02.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",265,"2026-09-28T14:00:00.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",189,"2026-09-28T10:52:40.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":18},"Science","science",151,{"name":64,"slug":65,"count":66,"latest_published_at":67},"Consumer Tech","consumer-tech",135,"2026-09-26T14:30:00.000Z",{"name":69,"slug":70,"count":71,"latest_published_at":72},"Software","software",91,"2026-09-25T20:55:00.000Z",{"name":74,"slug":75,"count":76,"latest_published_at":77},"Dev Tools","dev-tools",84,"2026-09-26T04:20:58.000Z",{"name":79,"slug":80,"count":81,"latest_published_at":82},"Startups","startups",76,"2026-09-25T18:33:59.000Z",{"name":84,"slug":85,"count":86,"latest_published_at":87},"Gaming","gaming",48,"2026-09-25T18:35:21.000Z",{"name":89,"slug":90,"count":86,"latest_published_at":91},"General","general","2026-09-26T17:02:42.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Reviews","reviews",30,"2026-09-24T20:07:31.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]