[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ai-safety-tests-are-missing-context-study-finds":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},2853,"ai-safety-tests-are-missing-context-study-finds","AI Safety Tests Are Missing Context, Study Finds","A new benchmark shows LLMs over-refuse safe requests, exposing a blind spot in how labs measure model safety.","Most AI safety benchmarks ask the wrong question.\n\nResearchers have released CASE-Bench, a safety evaluation framework that factors in context when judging whether a large language model's response is appropriate. Current benchmarks typically flag queries in isolation — a question about medication dosages looks dangerous without knowing the asker is a nurse. CASE-Bench uses Contextual Integrity theory to assign formally described settings to each query, then measures whether the model's response fits that setting. The team also ran statistical power analysis to determine how many human annotators were needed, a step most prior benchmarks skip.\n\nThe results expose a real gap. Context significantly shifted human judgments about what counts as a safe response — with p\u003C0.0001, that is not noise. More pointedly, commercial models showed the largest mismatches with human expectations specifically in contexts that were already safe, meaning they refused or hedged when they didn't need to.\n\nOver-refusal is the safety problem labs talk about least, because it is easier to defend a refusal than explain a harmful output. But a model that won't answer a nurse's drug question, or a lawyer's query about criminal statutes, is not safe — it is just unhelpful. CASE-Bench won't fix alignment on its own, but it at least asks the industry to grade on context, not just content.","[\"ai\",\"safety\",\"benchmarks\",\"llm\"]","2026-06-30T04:00:00.000Z","2026-06-30T14:00:48.176Z","2026-06-30T14:00:51.013Z","published",null,[],"ai",[24,26,27,28],"safety","benchmarks","llm",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2501.14940",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]