[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-exposes-llm-gaps-in-office-file-comprehension":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3460,"new-benchmark-exposes-llm-gaps-in-office-file-comprehension","New Benchmark Exposes LLM Gaps in Office File Comprehension","A new public benchmark tests AI systems on Word, Excel, and PowerPoint files - and even the best models top out at 59% on expert reasoning tasks.","AI systems still struggle to read the files that run most offices.\n\nResearchers have released the Office Comprehension Bench (OCB), the first public benchmark designed to test large language models on native Microsoft Office formats - .docx, .xlsx, and .pptx files. OCB runs two tracks: File Fidelity Q&A, which checks whether models can parse structural and visual elements like tables, charts, formulas, speaker notes, and named ranges; and Domain Q&A, which throws expert-level reasoning tasks at models using real-world industry documents across 12 professional fields. Answers are graded by breaking each reference response into atomic, binary-gradable claims and running an ensemble of LLM judges against them.\n\nThe results are a quiet indictment of how vendors talk about AI document handling. The strongest frontier model in standard mode scores about 59.3% on Domain Q&A - meaning it gets roughly four in ten expert reasoning questions wrong. Notably, cranking up \"thinking depth\" within the same product tier does almost nothing; only moving to a higher product tier produces modest improvement, and even then the gains are marginal.\n\nThis matters because office document comprehension is the unglamorous core of most enterprise AI pitches. Every major lab and productivity suite vendor - Microsoft, Google, and a long tail of startups - is selling AI that can \"understand your documents.\" OCB gives buyers a way to pressure-test that claim against a public leaderboard. The dataset, evaluation tooling, and judge prompts are all released openly, which makes it harder to quietly game.\n\nFor context, 59% on a domain reasoning task is roughly the floor where a tool becomes a liability in high-stakes professional settings - which is exactly where these vendors are trying to sell.","[\"ai\",\"benchmarks\",\"large-language-models\",\"enterprise\"]","2026-07-03T04:00:00.000Z","2026-07-03T06:33:37.100Z","2026-07-03T06:33:39.994Z","published",null,[],"ai",[24,26,27,28],"benchmarks","large-language-models","enterprise",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.01245",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]