[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-shows-ai-agents-invent-tools-that-do-not-exist":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},6757,"new-benchmark-shows-ai-agents-invent-tools-that-do-not-exist","New Benchmark Shows AI Agents Invent Tools That Do Not Exist","A new arXiv benchmark finds AI agents fabricate tools and arguments no schema declares, and bigger models are no better at avoiding it.","A new benchmark shows AI agents routinely invent tools that do not exist, then hand them arguments no schema ever defined.\n\nThe paper, posted to arXiv, introduces a five-class taxonomy of tool hallucination and a training-free 'Resolution Rung' resolver that checks whether a called tool is registered and whether its arguments match a declared signature. Testing ten hosted models across two ways of invoking tools, the authors logged 322 genuine hallucinations. Fabricated-tool calls clustered heavily on unconstrained raw-JSON interfaces, 34 cases versus just 3 on a more structured surface, and a 675-billion-parameter model hallucinated about as often as one with 7 to 8 billion parameters. The team then tested the Model Context Protocol, where merging multiple tool servers into one namespace produced a second, distinct taxonomy of failures and 154 more hallucinations, some from frontier models that had been clean on the single-registry test.\n\nThis matters because most tool-security work assumes an agent's call refers to something real, then argues about whether it should be allowed to run. The paper's point is structural: a hallucinated call was never a decision any permission system made, so no amount of gating catches it before the fact. That is a real gap in how agentic systems are secured today, not just a benchmark curiosity.\n\nThe team released the underlying Hallucinated-Tools Benchmark so other resolvers can be tested against the same numbers, which at minimum ends the era of vague claims that an agent 'doesn't really' call fake tools.","[\"ai agents\",\"llm hallucination\",\"benchmarks\",\"mcp\"]","2026-09-18T04:00:00.000Z","2026-09-18T15:20:58.285Z","2026-09-18T15:21:10.208Z","published",null,[],"ai",[26,27,28,29],"ai agents","llm hallucination","benchmarks","mcp",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.19425",0,{"sections":36},[37,40,44,49,54,58,62,67,72,77,82,87,92,97],{"name":38,"slug":24,"count":39,"latest_published_at":18},"AI",3959,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",652,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",338,"2026-09-11T04:00:00.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":18},"Hardware","hardware",155,{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",116,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",99,"2026-09-09T17:27:33.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":71},"Dev Tools","dev-tools",76,"2026-09-18T01:04:54.000Z",{"name":73,"slug":74,"count":75,"latest_published_at":76},"Software","software",75,"2026-09-10T20:41:21.000Z",{"name":78,"slug":79,"count":80,"latest_published_at":81},"Startups","startups",55,"2026-09-09T23:14:29.000Z",{"name":83,"slug":84,"count":85,"latest_published_at":86},"Gaming","gaming",43,"2026-09-10T12:18:06.000Z",{"name":88,"slug":89,"count":90,"latest_published_at":91},"General","general",41,"2026-09-08T01:57:23.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]