[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-can-ai-replicate-social-science-research-a-new-benchmark-tests-it":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3133,"can-ai-replicate-social-science-research-a-new-benchmark-tests-it","Can AI Replicate Social Science Research? A New Benchmark Tests It","ReplicatorBench exposes a critical gap: AI agents can run experiments but fail to find the new data replication actually requires.","A new benchmark puts AI research agents through the full, messy process of replicating social and behavioral science studies — not just running code someone else wrote.\n\nMost existing benchmarks hand AI agents the code and data, then ask them to reproduce results. ReplicatorBench takes a harder problem: give agents a research claim and see if they can independently retrieve new data, design an experiment, run it, and interpret what they find. The benchmark includes both replicable and non-replicable claims — a deliberate choice, since an agent that only sees studies that check out can never learn to flag ones that don't. The researchers also built ReplicatorAgent, a test harness equipped with web search and sandboxed code execution, and ran it against four different LLMs to establish a baseline.\n\nThe split result matters: current agents are competent at the computational middle — designing and executing experiments — but fall apart at the retrieval step, where finding relevant new data requires the kind of judgment and search skill that large language models still lack. That gap is precisely what separates replication from reproduction, and it's the gap that makes replication hard for humans too.\n\nSocial science has spent the last decade reckoning with a reproducibility crisis; automating replication checks could accelerate that process or paper over its hardest parts. ReplicatorBench at least sets a credible bar — the code and data are public — which is more than most benchmarks bothered to do.","[\"ai\",\"benchmarks\",\"research\",\"social-science\"]","2026-07-01T04:00:00.000Z","2026-07-01T07:58:56.893Z","2026-07-01T07:58:59.911Z","published",null,[],"ai",[24,26,27,28],"benchmarks","research","social-science",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2602.11354",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]