[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-exposes-how-ai-safety-varies-with-intent":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3519,"new-benchmark-exposes-how-ai-safety-varies-with-intent","New Benchmark Exposes How AI Safety Varies With Intent","OpenSafeIntent tests whether models stay safe as prompt intent shifts, and finds most fail in ways aggregate scores hide.","A new benchmark shows AI safety ratings can mask real failures when the same task is asked with different intent.\n\nResearchers released OpenSafeIntent, a dataset of controlled prompt sets that hold the underlying task constant while varying whether the request appears benign, dual-use, or malicious. The goal is to test intent-calibrated behavior rather than average safety scores. Across a broad set of models, the results were unflattering: models frequently failed to stay safe across matched intent variants, dual-use responses broke down under simple paraphrasing, and high-level answers on risky topics were not reliably safe. Interestingly, responses that reframed ambiguous prompts into safer tasks were the least likely to cross a safety boundary.\n\nThe benchmark targets a blind spot in how the industry measures AI safety. Most existing evaluations treat each prompt as independent, which lets models look safe on average while being inconsistent when context shifts. If a model gives a safe answer to a cooking question but an unsafe one to the same question framed as a weapon-making inquiry, the aggregate score obscures the failure.\n\nLabs have spent years claiming their models are carefully tuned for safety. OpenSafeIntent suggests the tuning is more brittle than advertised - and that the standard test methodology is partly responsible for the illusion of robustness.","[\"ai\",\"safety\",\"benchmarks\",\"language-models\"]","2026-07-03T04:00:00.000Z","2026-07-03T07:50:08.762Z","2026-07-03T07:50:11.771Z","published",null,[],"ai",[24,26,27,28],"safety","benchmarks","language-models",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.02047",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]