[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-the-benchmark-harness-is-changing-what-ai-agents-believe":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3697,"the-benchmark-harness-is-changing-what-ai-agents-believe","The Benchmark Harness Is Changing What AI Agents Believe","New research finds that the scaffolding around an AI agent - not just the model or task - shapes the beliefs that drive its decisions.","The software wrapping your AI benchmark may be rewriting what the agent thinks is happening.\n\nResearchers introduced a diagnostic called a belief rollout, which traces an agent's internal reasoning across multiple steps - covering its sense of progress, risk, recoverability, and likely next actions. When they held the task, environment, and underlying model constant but swapped the surrounding harness, the agent's beliefs shifted. Blocked actions, compressed repair logs, selective state verification, and evidence pruning all changed how the agent reasoned mid-task, even when the final success rate stayed the same. The divergence was decomposed into two components: an arrival term capturing immediate shifts from interface changes, and a growth term tracking how those shifts compound over longer decision horizons.\n\nThis matters because agent evaluations are almost always reported as pass-fail rates on a fixed benchmark - a number that conceals how the agent actually got there. If two harnesses produce the same terminal success but different belief trajectories, comparing agents across those harnesses is comparing apples to something that just happens to look like an apple. The researchers also propose BIWM, a protocol that standardizes observations, logs censored decision branches, and runs risky actions in shadow to align belief trajectories across harness configurations - no model retraining required.\n\nAgent benchmarking is already a contested space: critics have long noted that leaderboard scores often reflect prompt engineering and evaluation tricks as much as genuine capability. This work adds a subtler concern - that the harness itself is an uncontrolled experimental variable, not neutral infrastructure, and published scores should probably say so.","[\"ai\",\"benchmarks\",\"llm-agents\",\"research\"]","2026-07-07T04:00:00.000Z","2026-07-07T06:03:29.489Z","2026-07-07T06:03:32.413Z","published",null,[],"ai",[24,26,27,28],"benchmarks","llm-agents","research",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.04528",0,{"sections":35},[36,40,45,50,55,60,65,70,75,79,84,88,93,98],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":18},"Dev Tools","dev-tools",59,{"name":80,"slug":81,"count":82,"latest_published_at":83},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":85,"slug":86,"count":82,"latest_published_at":87},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]