[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-swe-critix-judges-whether-ai-coding-agents-finished-the-job":10,"sections":40},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":30,"tags":31,"sources":35,"feedback":39,"feedback_at":22,"cost_usd":39,"total_tokens":39},11287,"swe-critix-judges-whether-ai-coding-agents-finished-the-job","SWE-Critix Judges Whether AI Coding Agents Finished the Job","A new verifier trained with reinforcement learning jumps from 57% to 78% accuracy at judging whether coding-agent runs actually succeeded.","A new AI model called SWE-Critix grades whether coding agents actually finished their assigned tasks - and it's right far more often than the general-purpose models it replaces.\n\nResearchers built a post-training pipeline that fine-tunes a 30B-parameter model, Qwen3-30B-A3B, to judge coding-agent trajectories: the long, multi-step logs of an agent editing a repository, running tests, and calling tools. Given a task and its full execution trail, the model outputs a pass-fail verdict plus a plain-language explanation of its reasoning. The team combined supervised fine-tuning with reinforcement learning, training on trajectories from a dataset called CodeForge-Preview. On a held-out test set, classification accuracy climbed from 57% for the untrained base model to 66% after supervised fine-tuning, then to 78% after the reinforcement learning stage.\n\nCoding agents are increasingly asked to complete entire engineering tasks unsupervised, but there is rarely an automatic oracle to confirm they actually did the job. A verifier that is right 78% of the time instead of 57% is the difference between a QA layer worth trusting and one barely better than a coin flip with extra steps. The researchers are releasing the models, datasets, and training code, which matters more than the headline number - it lets other labs check the work instead of taking the result on faith.\n\nJumping from 57% to 78% still leaves roughly one in five verdicts wrong - a reminder that grading the grader remains the real unsolved problem in agentic coding.","[\"ai\",\"coding-agents\",\"llm-evaluation\",\"dev-tools\"]","2026-10-12T04:00:00.000Z","2026-10-12T06:03:20.377Z","2026-10-12T06:03:24.123Z","published",null,[24],{"id":25,"reviewer":26,"round":27,"reason":28,"status":29},"editor-r1","editor",1,"Fix the broken headline grammar — 'Grades AI Coding Agents Work' needs a possessive ('Agents' Work') or a rephrase like 'Grades Whether AI Coding Agents Finished the Job'; the body and figures otherwise check out against the source.","resolved","ai",[30,32,33,34],"coding-agents","llm-evaluation","dev-tools",[36],{"name":37,"url":38},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.13356",0,{"sections":41},[42,45,50,55,60,65,70,75,80,85,89,94,99,104],{"name":43,"slug":30,"count":44,"latest_published_at":18},"AI",6966,{"name":46,"slug":47,"count":48,"latest_published_at":49},"Security","security",952,"2026-10-12T04:54:12.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",496,"2026-10-11T20:06:43.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Deals","deals",488,"2026-10-12T10:32:15.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Hardware","hardware",234,"2026-10-11T17:20:00.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Science","science",196,"2026-10-11T15:34:40.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Consumer Tech","consumer-tech",191,"2026-10-12T11:10:16.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Startups","startups",120,"2026-10-09T17:02:31.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Software","software",115,"2026-10-11T07:34:45.000Z",{"name":86,"slug":34,"count":87,"latest_published_at":88},"Dev Tools",111,"2026-10-12T11:22:24.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",70,"2026-10-11T14:15:00.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Gaming","gaming",59,"2026-10-09T11:43:43.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"Reviews","reviews",34,"2026-10-08T14:00:22.000Z",{"name":105,"slug":106,"count":107,"latest_published_at":108},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]