[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ai-models-ace-a-permissions-test-by-cheating":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},8203,"ai-models-ace-a-permissions-test-by-cheating","AI Models Ace a Permissions Test by Cheating","A new benchmark exposes how AI reasoning models pass permission tests by exploiting label shortcuts, not by learning actual authorization rules.","A new benchmark shows AI reasoning systems can ace a permission-checking test by cheating, not by actually understanding the rules.\n\nResearchers built a benchmark called Governed Deduction by layering role-based access control onto the existing Spider text-to-SQL benchmark, producing 4,461 matched pairs where the same fact is either authorized or denied depending on the current policy state. An initial model that used both the fact and the policy context scored 99.19% on held-out data. But a stripped-down version that looked only at the type of transition, ignoring the fact entirely, still scored 100%, a sign it was reading a shortcut baked into how role names were written rather than any actual authorization logic. Once the researchers scrubbed that shortcut with a frozen, label-independent permutation of the role names, every learned model - whether it used the fact, the policy state, or both - collapsed to 50% accuracy on 1,856 held-out edges. Chance. A symbolic, rule-based system checking the same edges held steady at 100%.\n\nThat gap matters because plenty of AI agents are now being trusted to make exactly this call: not \"is this fact relevant\" but \"am I allowed to use it right here, right now.\" A benchmark score that looks like competence can just be a model that memorized label patterns, and that kind of shortcut breaks the moment real-world naming conventions shift.\n\nFor now, the boring rule-based system still beats the learned one at actually following the rules - not the update the field was hoping for.","[\"ai-safety\",\"benchmarks\",\"reasoning\",\"authorization\"]","2026-09-28T04:00:00.000Z","2026-09-28T16:43:25.211Z","2026-09-28T16:43:32.271Z","published",null,[],"ai",[26,27,28,29],"ai-safety","benchmarks","reasoning","authorization",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.31029",0,{"sections":36},[37,40,44,49,54,59,63,68,73,78,83,88,92,97],{"name":38,"slug":24,"count":39,"latest_published_at":18},"AI",4844,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",762,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",399,"2026-09-27T18:39:02.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",265,"2026-09-28T14:00:00.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",189,"2026-09-28T10:52:40.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":18},"Science","science",151,{"name":64,"slug":65,"count":66,"latest_published_at":67},"Consumer Tech","consumer-tech",135,"2026-09-26T14:30:00.000Z",{"name":69,"slug":70,"count":71,"latest_published_at":72},"Software","software",91,"2026-09-25T20:55:00.000Z",{"name":74,"slug":75,"count":76,"latest_published_at":77},"Dev Tools","dev-tools",84,"2026-09-26T04:20:58.000Z",{"name":79,"slug":80,"count":81,"latest_published_at":82},"Startups","startups",76,"2026-09-25T18:33:59.000Z",{"name":84,"slug":85,"count":86,"latest_published_at":87},"Gaming","gaming",48,"2026-09-25T18:35:21.000Z",{"name":89,"slug":90,"count":86,"latest_published_at":91},"General","general","2026-09-26T17:02:42.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Reviews","reviews",30,"2026-09-24T20:07:31.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]