[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-study-finds-no-single-signal-predicts-when-llm-updates-regress":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},5008,"study-finds-no-single-signal-predicts-when-llm-updates-regress","Study Finds No Single Signal Predicts When LLM Updates Regress","New research finds no single signal reliably predicts when an LLM update silently breaks a previously correct answer.","A new study finds there's no reliable warning sign that an AI model update is about to break something that used to work.\n\nResearchers tested whether signals available at inference time could predict sample-level regression - cases where a response correct under an old model version turns incorrect under a new one, even though the new model scores higher overall. They compared single-model signals such as confidence, logit margin, and attention entropy against cross-version signals such as output KL divergence, likelihood drift, and representation drift, using a test designed to isolate each signal's real gain over a basic confidence baseline. The comparison spanned six benchmarks across three task types - multiple-choice questions, math reasoning, and code generation - and six different pairs of model versions.\n\nWhich signal works turns out to depend on the task, not the model: confidence is enough for multiple-choice and easy math, but harder math and code need likelihood and KL-based signals to catch the regressions confidence misses. Some cross-version signals kept working even when confidence failed outright, including without labeled answers, which let the researchers sketch a fallback that routes shaky responses back to the older model version. That's a concrete step toward regression-testing AI updates the way engineers already test software releases, rather than trusting a benchmark average.\n\nVersion upgrades get marketed as pure improvement, but this paper is a reminder that \"better on average\" and \"better for you\" are different claims, and the industry doesn't yet have a single tool for telling them apart.","[\"ai\",\"llm-evaluation\",\"machine-learning\",\"model-updates\"]","2026-08-17T04:00:00.000Z","2026-08-17T05:11:06.690Z","2026-08-17T05:11:18.531Z","published",null,[],"ai",[24,26,27,28],"llm-evaluation","machine-learning","model-updates",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2608.13607",0,{"sections":35},[36,40,44,49,54,59,64,69,74,79,84,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",3293,"2026-08-20T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":39},"Security","security",435,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",210,"2026-08-19T09:32:27.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",140,"2026-08-19T18:25:42.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":63},"Consumer Tech","consumer-tech",95,"2026-08-18T16:05:00.000Z",{"name":65,"slug":66,"count":67,"latest_published_at":68},"Science","science",90,"2026-08-19T18:41:02.000Z",{"name":70,"slug":71,"count":72,"latest_published_at":73},"Software","software",73,"2026-08-18T07:51:50.000Z",{"name":75,"slug":76,"count":77,"latest_published_at":78},"Dev Tools","dev-tools",69,"2026-08-18T04:00:00.000Z",{"name":80,"slug":81,"count":82,"latest_published_at":83},"Startups","startups",47,"2026-08-19T19:13:46.000Z",{"name":85,"slug":86,"count":87,"latest_published_at":88},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",33,"2026-08-18T22:18:13.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]