[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ai-system-wins-benchmark-for-judging-sentence-certainty":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},7800,"ai-system-wins-benchmark-for-judging-sentence-certainty","AI System Wins Benchmark for Judging Sentence Certainty","A new fine-tuned system topped a competition that grades AI on how well it judges whether a sentence implies something is actually true.","A parameter-efficient AI system called UO-FIE just won a competition that tests whether machines can tell how strongly a sentence commits to something being fact.\n\nThe competition, FIE2026, sorts Chinese context-hypothesis pairs into nine ordered \"factivity\" intervals, and its scoring rewards both exact answers and near-misses. The training data is lopsided: 64.1% of the 566 examples fall into a single class. That skew broke earlier attempts - mDeBERTa classifiers mostly just guessed the dominant class, while a simpler Huber-regression baseline landed closer to correct answers more often but nailed exact matches less often. UO-FIE, built on Qwen3.5-9B with LoRA fine-tuning, split the difference by combining hard-label training with a graded utility score, scheduled class weighting, and an ordinal loss, then applying expected-utility decoding and calibration. It won the fine-tuning track with a macro utility of 0.8316.\n\nFactivity inference underpins anything that has to treat text as an assertion rather than a hedge - search, summarization, fact-checking tools. But the results also expose a wrinkle in how these systems get judged: a separate prompt-based ensemble with no fine-tuning at all scored a higher macro utility, 0.8450, yet placed third because rival non-fine-tuned entries scored even higher.\n\nA benchmark that rewards proximity as well as precision is a reasonable fix for skewed data, but it also means the leaderboard order and the raw utility scores can tell different stories - worth remembering before crowning any single approach the winner.","[\"ai\",\"nlp\",\"benchmarks\",\"language-models\"]","2026-09-25T04:00:00.000Z","2026-09-25T23:56:02.879Z","2026-09-25T23:56:07.456Z","published",null,[],"ai",[24,26,27,28],"nlp","benchmarks","language-models",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.28605",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,90,95,100],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",4534,"2026-09-25T17:16:30.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",741,"2026-09-25T15:52:13.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",390,"2026-09-25T16:24:59.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",256,"2026-09-25T17:00:53.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",185,"2026-09-25T15:00:22.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Science","science",140,"2026-09-25T11:55:23.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Consumer Tech","consumer-tech",132,"2026-09-25T15:30:00.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Software","software",88,"2026-09-24T23:06:55.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",82,"2026-09-25T09:59:40.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Startups","startups",76,"2026-09-25T18:33:59.000Z",{"name":86,"slug":87,"count":88,"latest_published_at":89},"Gaming","gaming",48,"2026-09-25T18:35:21.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"General","general",46,"2026-09-25T02:12:57.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",30,"2026-09-24T20:07:31.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]