[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-study-finds-llm-lie-detectors-get-fooled-by-role-play":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},9174,"new-study-finds-llm-lie-detectors-get-fooled-by-role-play","New Study Finds LLM Lie Detectors Get Fooled by Role Play","Researchers show that AI truthfulness probes often track whether a model is following instructions rather than whether it is telling the truth.","AI companies want a way to peek inside a model and tell when it is lying. A new paper says the leading methods for doing that mostly do not work.\n\nResearchers tested eight existing lie detection probes, tools that scan a language model's internal activations to flag dishonest outputs. They built a dataset of 8,916 human-reviewed responses from three LLMs instructed to role-play anti-factual personas, like a conspiracy theorist, where the \"correct\" persona answer is itself false. Most probes failed to separate true from false statements once a persona was involved, especially when comparing right and wrong answers generated under the same character prompt. Digging into why, the team built confounder datasets where truthfulness was deliberately set against other traits, and found several probes were actually tracking things like instruction compliance or how statistically likely a response was, not truth itself.\n\nThis matters because lie detection probes are pitched as a safety backstop for exactly the moment they fail: when a model is playing a character or following a jailbreak-adjacent prompt, which is also when you most want to know if it is being honest. A probe that actually measures \"did the model do what it was told\" is not a lie detector, it is an obedience detector wearing a lie detector's coat.\n\nThe researchers' own fix, a simpler linear probe trained to avoid these confounds, held up better on both tests, which suggests the fix is in the training data, not a more exotic architecture.","[\"ai-safety\",\"llm-evaluation\",\"interpretability\",\"research\"]","2026-10-01T04:00:00.000Z","2026-10-01T23:59:38.421Z","2026-10-01T23:59:43.716Z","published",null,[],"ai",[26,27,28,29],"ai-safety","llm-evaluation","interpretability","research",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.39807",0,{"sections":36},[37,40,44,48,53,58,62,67,72,76,81,86,91,96],{"name":38,"slug":24,"count":39,"latest_published_at":18},"AI",5598,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",815,{"name":45,"slug":46,"count":47,"latest_published_at":18},"Policy","policy",430,{"name":49,"slug":50,"count":51,"latest_published_at":52},"Deals","deals",298,"2026-09-30T21:00:26.000Z",{"name":54,"slug":55,"count":56,"latest_published_at":57},"Hardware","hardware",196,"2026-09-30T13:00:00.000Z",{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",163,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",149,"2026-09-30T22:57:11.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":71},"Dev Tools","dev-tools",93,"2026-10-01T02:30:48.000Z",{"name":73,"slug":74,"count":70,"latest_published_at":75},"Software","software","2026-09-30T21:41:11.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Startups","startups",84,"2026-09-30T20:39:09.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",51,"2026-09-30T16:24:30.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"General","general",50,"2026-09-30T21:37:54.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"Reviews","reviews",31,"2026-09-28T14:31:34.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]