[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-llms-fail-logic-tests-they-should-pass":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3576,"llms-fail-logic-tests-they-should-pass","LLMs Fail Logic Tests They Should Pass","A new testing framework shows that AI models ace standard reasoning benchmarks but break down when the same problems are rephrased in logically equivalent ways.","AI reasoning benchmarks are easier to game than they look.\n\nResearchers introduced LGMT, a testing framework that evaluates large language models not on whether they get answers right, but on whether they get them right *consistently*. The approach draws on first-order logic to generate semantically equivalent variations of the same problem — different surface forms, same underlying truth. Running six state-of-the-art LLMs through this gauntlet revealed defects that standard reference-based benchmarks missed entirely. Models proved especially brittle when symbols or conclusions were reworded, even when the logical content stayed identical.\n\nThe finding matters because benchmark scores drive purchasing decisions, research funding, and deployment confidence. If a model aces a test only because it pattern-matches the exact phrasing, that score is measuring memorization as much as reasoning — a gap that matters most in high-stakes applications like legal analysis, medical triage, or code verification. LGMT's oracle-free design also means it doesn't require human-labeled ground-truth answers, making it cheaper to scale than conventional evaluation pipelines.\n\nAdvanced prompting techniques like few-shot chain-of-thought helped somewhat but didn't close the gap — which is the kind of result that should make any vendor's benchmark-touting press release worth reading twice.","[\"ai\",\"llms\",\"benchmarks\",\"reasoning\"]","2026-07-03T04:00:00.000Z","2026-07-03T08:55:03.477Z","2026-07-03T08:55:06.439Z","published",null,[],"ai",[24,26,27,28],"llms","benchmarks","reasoning",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.23965",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]