[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-finds-llms-bad-at-asking-clarifying-questions":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},9962,"new-benchmark-finds-llms-bad-at-asking-clarifying-questions","New Benchmark Finds LLMs Bad at Asking Clarifying Questions","ClarifyCodeBench shows top coding models still struggle to ask the right clarifying questions before writing code.","AI coding assistants are great at writing code and apparently terrible at asking what you actually meant.\n\nResearchers built ClarifyCodeBench, an interactive benchmark drawn from real-world programming tasks, to test whether large language models can spot ambiguous or incomplete requirements and ask the right clarifying questions before generating code. The benchmark includes manually annotated ambiguity types, sample clarification questions, and ground-truth answers, plus two new metrics: one that penalizes models for asking too many questions, and one that measures how close they get to the ideal number of clarifying rounds. The team tested six state-of-the-art LLMs. Across the board, the models that wrote the most correct code were not necessarily the ones that asked the best questions.\n\nThat's a meaningful split from how the industry usually measures coding assistants, which is almost entirely on whether the final code works. The study found that giving a model more reasoning time improved code correctness but barely helped it notice unclear requirements, and performance fell off fast once a prompt contained multiple ambiguities at once, exactly the messy, multi-part requirements a reader gets from an actual product manager, not a tidy ticket.\n\nMost coding benchmarks still hand the model a perfectly worded spec, which is a bit like judging a translator by sentences nobody actually speaks.","[\"ai\",\"code-generation\",\"benchmarks\",\"requirements\"]","2026-10-05T04:00:00.000Z","2026-10-05T15:32:34.849Z","2026-10-05T15:32:40.854Z","published",null,[],"ai",[24,26,27,28],"code-generation","benchmarks","requirements",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.00711",0,{"sections":35},[36,39,43,48,53,58,62,67,71,76,81,86,91,96],{"name":37,"slug":24,"count":38,"latest_published_at":18},"AI",6233,{"name":40,"slug":41,"count":42,"latest_published_at":18},"Security","security",868,{"name":44,"slug":45,"count":46,"latest_published_at":47},"Policy","policy",444,"2026-10-03T15:02:01.000Z",{"name":49,"slug":50,"count":51,"latest_published_at":52},"Deals","deals",323,"2026-10-04T13:00:00.000Z",{"name":54,"slug":55,"count":56,"latest_published_at":57},"Hardware","hardware",204,"2026-10-03T14:50:50.000Z",{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",177,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",158,"2026-10-03T03:21:12.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":18},"Dev Tools","dev-tools",98,{"name":72,"slug":73,"count":74,"latest_published_at":75},"Software","software",97,"2026-10-04T10:00:00.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Startups","startups",92,"2026-10-04T14:36:25.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",53,"2026-10-02T02:50:39.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"General","general",51,"2026-10-05T02:35:01.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"Reviews","reviews",32,"2026-10-02T18:00:00.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"How-To","how-to",7,"2026-10-01T09:00:00.000Z"]