[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-benchmark-catches-legal-ai-search-tools-making-things-up":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},5085,"new-benchmark-catches-legal-ai-search-tools-making-things-up","New Benchmark Catches Legal AI Search Tools Making Things Up","A new bilingual dataset grades legal RAG systems claim by claim, and even top performers stumble on both retrieval and generation.","A new benchmark says the retrieval-augmented search tools law firms are starting to trust still invent things, and it can now show exactly where.\n\nResearchers introduced ClaimRAG-LAW, a dataset built to stress-test retrieval-augmented generation (RAG) systems in legal search. Unlike prior legal RAG benchmarks, it covers both French and English, includes questions written for non-experts as well as lawyers, and spans a range of realistic query types. The team then ran current legal RAG systems through a fine-grained evaluation framework that scores retrieval accuracy, generation quality, and individual claims separately, rather than judging a system's output as one pass-or-fail block. The results showed measurable weaknesses in all three areas.\n\nThis matters because RAG is the industry's default answer to hallucination risk in high-stakes fields, and law is about as high-stakes as text generation gets. Most existing legal benchmarks are English-only and written for people who already know the vocabulary, which quietly ignores how non-lawyers actually use these tools. A benchmark that separates \"the system fetched the wrong case\" from \"the system fetched the right case but described it wrong\" gives builders a real diagnosis instead of a vague failure grade.\n\nCall it an audit, not an indictment: the paper does not name which vendors failed where. But it confirms what plenty of legal-tech skeptics already suspected -- \"RAG-powered\" is not the same as \"fact-checked.\"","[\"legal-tech\",\"rag\",\"llm-evaluation\",\"hallucination\"]","2026-08-17T04:00:00.000Z","2026-08-17T09:43:21.775Z","2026-08-17T09:43:33.654Z","published",null,[],"ai",[26,27,28,29],"legal-tech","rag","llm-evaluation","hallucination",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2605.21071",0,{"sections":36},[37,41,45,50,55,60,65,70,75,80,85,90,95,100],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",3293,"2026-08-20T04:00:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":40},"Security","security",435,{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",210,"2026-08-19T09:32:27.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",140,"2026-08-19T18:25:42.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",95,"2026-08-18T16:05:00.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Science","science",90,"2026-08-19T18:41:02.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Software","software",73,"2026-08-18T07:51:50.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",69,"2026-08-18T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Startups","startups",47,"2026-08-19T19:13:46.000Z",{"name":86,"slug":87,"count":88,"latest_published_at":89},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"General","general",33,"2026-08-18T22:18:13.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]