[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-exposes-flaws-in-ai-confidence-scoring":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},10870,"new-benchmark-exposes-flaws-in-ai-confidence-scoring","New Benchmark Exposes Flaws in AI Confidence Scoring","TypedBench finds AI confidence scores are wording-sensitive and underconfident, which can backfire once real costs enter the decision.","A new benchmark says the confidence scores behind automated AI decisions often cannot be trusted.\n\nResearchers built TypedBench, a benchmark for so-called System One decision models - AI systems that output calibrated probabilities over typed answers like categories, ordinal levels, or yes\u002Fno outcomes, rather than generating text. The benchmark draws on seven policy-labelled generators and nine evaluation suites, testing accuracy across paraphrased wording, calibration error against a theoretical perfectly-calibrated predictor, and performance under asymmetric cost matrices. The team ran it on a hosted model, an open encoder, and a family of open decoders ranging from 0.8 billion to 9 billion parameters, all on identical test items.\n\nThe hosted model stuck to its stated policy, but its confidence scores shifted depending on how a question was phrased, and it was systematically underconfident. Under cost-sensitive conditions, that underconfidence was bad enough that just taking the model's top answer beat actually using its probabilities. The open decoders were exact but got slower as more options were added, and their accuracy dropped on policy questions as the choice set grew.\n\nThat is a real problem for anyone wiring these scores into automated triage, routing, or escalation systems: a probability that moves when you rephrase the question is not really a probability.","[\"ai-benchmarks\",\"model-calibration\",\"llm-evaluation\"]","2026-10-09T04:00:00.000Z","2026-10-09T19:18:11.705Z","2026-10-09T19:18:16.610Z","published",null,[],"ai",[26,27,28],"ai-benchmarks","model-calibration","llm-evaluation",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.11392",0,{"sections":35},[36,39,43,48,53,58,62,67,72,77,82,87,92,97],{"name":37,"slug":24,"count":38,"latest_published_at":18},"AI",6619,{"name":40,"slug":41,"count":42,"latest_published_at":18},"Security","security",927,{"name":44,"slug":45,"count":46,"latest_published_at":47},"Policy","policy",486,"2026-10-08T22:40:11.000Z",{"name":49,"slug":50,"count":51,"latest_published_at":52},"Deals","deals",474,"2026-10-08T22:00:00.000Z",{"name":54,"slug":55,"count":56,"latest_published_at":57},"Hardware","hardware",229,"2026-10-08T20:47:10.000Z",{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",192,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",181,"2026-10-08T23:26:35.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":71},"Startups","startups",117,"2026-10-08T16:45:00.000Z",{"name":73,"slug":74,"count":75,"latest_published_at":76},"Software","software",114,"2026-10-08T17:57:01.000Z",{"name":78,"slug":79,"count":80,"latest_published_at":81},"Dev Tools","dev-tools",105,"2026-10-07T16:59:11.000Z",{"name":83,"slug":84,"count":85,"latest_published_at":86},"General","general",66,"2026-10-09T04:46:11.000Z",{"name":88,"slug":89,"count":90,"latest_published_at":91},"Gaming","gaming",58,"2026-10-08T20:08:45.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Reviews","reviews",34,"2026-10-08T14:00:22.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]