[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-researchers-build-a-tool-to-stress-test-ai-judges":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},7281,"researchers-build-a-tool-to-stress-test-ai-judges","Researchers Build a Tool to Stress-Test AI Judges","A new open-source workbench called MAWILE tests whether AI judges grading other AI systems change their verdicts for the wrong reasons.","A new tool checks whether the AI models being used to grade other AI models are actually grading consistently, or just reacting to noise.\n\nResearchers built MAWILE, an open-source workbench that audits LLM-based judges by deliberately perturbing four things: the judge's prompt, its scoring rubric, the input given to the system under evaluation, and that system's output. For each perturbation, MAWILE declares up front whether the judge's verdict should hold steady or shift in a specific direction, then checks whether it actually does. It works on binary, ordinal, and pairwise judges without needing gold-standard labels, and it's aimed at developers auditing a judge they've already configured, not building a new one. The code is available on GitHub.\n\nLLM judges have quietly become the default way to grade chatbot replies, rank model outputs, and score agent behavior, mostly because paying humans to do it at scale is slow and expensive. The problem is that a judge swayed by a reworded prompt or a reshuffled answer order isn't measuring quality, it's measuring phrasing. Every benchmark or product decision built on top of that judge inherits the same blind spot without anyone noticing.\n\nIt's a diagnostic, not a cure: flagging that your judge is inconsistent doesn't make it consistent, and someone still has to decide what the \"correct\" verdict should be for each test case, which puts human judgment right back in the loop this approach was meant to reduce.","[\"ai\",\"llm-evaluation\",\"dev-tools\",\"benchmarking\"]","2026-09-23T04:00:00.000Z","2026-09-23T04:49:23.637Z","2026-09-23T04:49:27.993Z","published",null,[],"ai",[24,26,27,28],"llm-evaluation","dev-tools","benchmarking",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.22599",0,{"sections":35},[36,39,43,48,53,58,62,67,72,76,81,86,91,96],{"name":37,"slug":24,"count":38,"latest_published_at":18},"AI",4247,{"name":40,"slug":41,"count":42,"latest_published_at":18},"Security","security",706,{"name":44,"slug":45,"count":46,"latest_published_at":47},"Policy","policy",369,"2026-09-23T02:13:52.000Z",{"name":49,"slug":50,"count":51,"latest_published_at":52},"Deals","deals",202,"2026-09-22T23:00:04.000Z",{"name":54,"slug":55,"count":56,"latest_published_at":57},"Hardware","hardware",168,"2026-09-22T23:56:03.000Z",{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",133,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",110,"2026-09-22T20:00:00.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":71},"Software","software",80,"2026-09-22T23:32:52.000Z",{"name":73,"slug":27,"count":74,"latest_published_at":75},"Dev Tools",79,"2026-09-22T22:21:13.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Startups","startups",65,"2026-09-22T22:06:48.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",45,"2026-09-22T15:35:06.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"General","general",43,"2026-09-21T23:48:56.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"Reviews","reviews",27,"2026-09-22T13:00:00.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]