[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-tests-ai-chatbots-on-crisis-support-safety":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":24,"persona_id":22,"persona_name":22,"section":25,"tags":26,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},2511,"new-benchmark-tests-ai-chatbots-on-crisis-support-safety","New Benchmark Tests AI Chatbots on Crisis Support Safety","EMPATH uses multilingual, multi-turn conversations to expose safety gaps that fixed-prompt benchmarks routinely miss in emotional-support AI.","A new benchmark called EMPATH puts emotional-support chatbots through crisis conversations instead of static prompts — and finds the cracks.\n\nResearchers built EMPATH to evaluate how AI chatbots handle emotionally sensitive, multi-turn interactions. An auditor model simulates help-seeking users across 140 seed instructions and 34 personas, generating full conversations that a separate judge model then scores across 19 metrics in five dimensions: crisis handling, therapeutic quality, conversational integrity, emotional safety, and cultural adaptation. The benchmark currently targets Mexican Spanish and US English. Using auditor and judge models from different families is intentional — it reduces the risk of one model grading its own homework.\n\nThe findings cut against the assumption that aggregate scores tell the safety story. Three frontier models tested within 0.74 points of each other overall, but per-metric profiles diverged by up to six points in model-specific spots. More unsettling: even the most consistent model swung between 2 and 10 on a single crisis metric across identical re-runs, and one model returned a different conversation on every run even at temperature 0. The researchers argue that run-to-run reliability is itself a safety property — not noise to be averaged away.\n\nMost safety benchmarks optimize for scale by locking prompt, language, and turn structure. That works well for factual tasks; it quietly fails for crisis conversations where context shifts by the sentence and a single wrong turn can cause harm. The pipeline, seeds, personas, and rubrics are all being released publicly, so other teams can apply EMPATH to their own systems — which is either encouraging openness or an invitation to benchmark-game, depending on how charitable you're feeling.","[\"ai\",\"safety\",\"benchmarks\",\"mental-health-tech\"]","2026-06-30T04:00:00.000Z","2026-06-30T07:05:33.045Z","2026-06-30T07:05:42.844Z","published",null,[],"https:\u002F\u002Fcdn.xyz.onl\u002Farticle-images\u002Fnew-benchmark-tests-ai-chatbots-on-crisis-support-safety.webp","ai",[25,27,28,29],"safety","benchmarks","mental-health-tech",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.30256",0,{"sections":36},[37,41,46,51,56,61,66,71,76,81,86,90,95,100],{"name":38,"slug":25,"count":39,"latest_published_at":40},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":45},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":47,"slug":48,"count":49,"latest_published_at":50},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":52,"slug":53,"count":54,"latest_published_at":55},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":57,"slug":58,"count":59,"latest_published_at":60},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":62,"slug":63,"count":64,"latest_published_at":65},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":67,"slug":68,"count":69,"latest_published_at":70},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":72,"slug":73,"count":74,"latest_published_at":75},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":87,"slug":88,"count":84,"latest_published_at":89},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]