[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-study-contracts-aim-to-catch-what-ai-research-reviews-miss":10,"sections":36},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":24,"persona_id":24,"persona_name":24,"section":25,"tags":26,"sources":31,"feedback":35,"feedback_at":24,"cost_usd":35,"total_tokens":35},10897,"study-contracts-aim-to-catch-what-ai-research-reviews-miss","Study Contracts Aim to Catch What AI Research Reviews Miss","A new paper tests whether AI judges can catch planted flaws in AI-written research without seeing the original plan, and most still miss some.","A new study proposes giving AI research agents something like a signed contract - and shows today's AI judges often can't tell when it's been broken.\n\nResearchers built eight paired cases: a clean version of a self-authored study and a version with one planted defect. A deterministic checker, given a registered rule for each specific fault plus the approved and executed records of a study, caught every one of the eight planted defects. The harder test came next: eighteen separate AI judge instances were handed only the finished study packages, stripped of pairing information and the specific fault to look for. Those judges flagged defects in 104 of 144 mutated cases, abstained 32 times, and failed outright eight times, and never once cleared a mutated case as clean.\n\nThe gap matters because most AI-generated research today gets reviewed the second way: someone reads the output and decides if it looks right. This paper argues that's the wrong test entirely - some defects are only visible if you know what was approved before the work started, not just what came out the other end.\n\nThe authors are upfront that this is a narrow, deliberately lopsided test, not a verdict on AI judges generally or proof that agents are gaming their own grading - but it's a clean demonstration that output-only review has a blind spot worth taking seriously.","[\"ai agents\",\"research verification\",\"llm evaluation\",\"ai safety\"]","2026-10-09T04:00:00.000Z","2026-10-09T20:35:25.280Z","2026-10-17T21:37:11.342Z","published","Duplicate of \"New Benchmark Tests Whether AI Can Catch Errors in Research Papers\"",[],null,"ai",[27,28,29,30],"ai agents","research verification","llm evaluation","ai safety",[32],{"name":33,"url":34},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.11754",0,{"sections":37},[38,42,47,52,57,62,67,72,77,82,87,92,97,102],{"name":39,"slug":25,"count":40,"latest_published_at":41},"AI",7038,"2026-10-17T19:58:31.000Z",{"name":43,"slug":44,"count":45,"latest_published_at":46},"Security","security",1001,"2026-10-17T22:20:00.000Z",{"name":48,"slug":49,"count":50,"latest_published_at":51},"Policy","policy",538,"2026-10-17T19:38:12.000Z",{"name":53,"slug":54,"count":55,"latest_published_at":56},"Deals","deals",521,"2026-10-17T07:00:00.000Z",{"name":58,"slug":59,"count":60,"latest_published_at":61},"Hardware","hardware",260,"2026-10-17T19:55:36.000Z",{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",235,"2026-10-16T23:36:26.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":71},"Science","science",202,"2026-10-16T11:00:53.000Z",{"name":73,"slug":74,"count":75,"latest_published_at":76},"Startups","startups",133,"2026-10-14T19:51:34.000Z",{"name":78,"slug":79,"count":80,"latest_published_at":81},"Software","software",128,"2026-10-14T22:23:48.000Z",{"name":83,"slug":84,"count":85,"latest_published_at":86},"Dev Tools","dev-tools",116,"2026-10-13T16:20:18.000Z",{"name":88,"slug":89,"count":90,"latest_published_at":91},"General","general",81,"2026-10-16T16:06:38.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Gaming","gaming",77,"2026-10-17T16:59:39.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"Reviews","reviews",37,"2026-10-14T10:00:00.000Z",{"name":103,"slug":104,"count":105,"latest_published_at":106},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]