[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ai-safety-scores-look-fine-the-underlying-problems-may-not-be":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":24,"persona_id":22,"persona_name":22,"section":25,"tags":26,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},2508,"ai-safety-scores-look-fine-the-underlying-problems-may-not-be","AI Safety Scores Look Fine. The Underlying Problems May Not Be.","A new paper argues that benchmark scores and safety metrics can keep climbing even as the real risks they're supposed to measure stay stubbornly unverified.","A research paper introduces \"EvalSafetyGap\" — a framework for understanding why AI safety and capability measurements keep improving on paper while the actual properties being measured remain hard to confirm.\n\nThe paper combines a broad literature survey with a structured audit of ten AI models, drawing on eight evidence streams covering work from 2018 to 2026. The central claim is that evaluation-side and alignment-side metrics share a common flaw: both are proxies, and both buckle under optimization pressure — a dynamic the authors frame using Goodhart's Law. They introduce two additional analytical tools, an \"Instability Decomposition\" and an \"Alignment Trilemma,\" to generate testable comparisons between where measurement goes wrong on the capability side versus the safety side.\n\nThe audit finding worth sitting with: in the ten-model sample, the relationship between capability scores and sustained adversarial robustness was statistically indeterminate (Pearson r = +0.232, p = 0.520). In plain terms, more capable models were not reliably more robust to adversarial attacks. The apparent gap between open and closed models on safety also turned out to be mostly a governance and disclosure story, not a behavioral one — and it shifted meaningfully depending on how a single borderline model was classified.\n\nThe paper does not produce a league table or rank the models it audits, which is the honest call given how inconsistent the underlying protocols are. What it does offer is a shared vocabulary for people trying to make AI evaluation less gameable — at a moment when every major lab is both building benchmarks and topping them.","[\"ai\",\"safety\",\"benchmarks\",\"research\"]","2026-06-30T04:00:00.000Z","2026-06-30T07:00:16.212Z","2026-06-30T07:00:26.609Z","published",null,[],"https:\u002F\u002Fcdn.xyz.onl\u002Farticle-images\u002Fai-safety-scores-look-fine-the-underlying-problems-may-not-be.webp","ai",[25,27,28,29],"safety","benchmarks","research",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.30219",0,{"sections":36},[37,41,46,51,56,61,66,71,76,81,86,90,95,100],{"name":38,"slug":25,"count":39,"latest_published_at":40},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":45},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":47,"slug":48,"count":49,"latest_published_at":50},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":52,"slug":53,"count":54,"latest_published_at":55},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":57,"slug":58,"count":59,"latest_published_at":60},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":62,"slug":63,"count":64,"latest_published_at":65},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":67,"slug":68,"count":69,"latest_published_at":70},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":72,"slug":73,"count":74,"latest_published_at":75},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":87,"slug":88,"count":84,"latest_published_at":89},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]