[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-tool-cuts-the-cost-of-testing-coding-agent-changes":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},6860,"new-tool-cuts-the-cost-of-testing-coding-agent-changes","New Tool Cuts the Cost of Testing Coding Agent Changes","DeltaSelect picks a small, budget-capped set of benchmark tasks that reliably predicts full-suite results, cutting the cost of repeated coding-agent tests.","A new open-source tool called DeltaSelect trims coding agent benchmarks down to a small set of tasks that reliably predicts the full suite's score, aiming to make routine A\u002FB testing cheaper.\n\nThe researchers first checked whether individual benchmark tasks are even useful signals. Resampling published trials from DeepSWE, they found only 22 of 113 tasks, about 19.5%, had a fifth-percentile Pearson correlation of at least 0.50 with full benchmark performance. DeltaSelect uses that same correlation approach to pick a fixed, budget-capped task set, then maps partial credit verifier results onto a common scale with linear regression. In a case study tuning custom skills and instructions for a low-reasoning version of gpt-5.6-luna, the team ran 13 evaluations for a total of $27.86. The version they ultimately adopted cost 58.1% less to test than their initial version, $1.75 versus $4.18, a statistically significant drop, and it scored higher on the calibrated benchmark, 42.36% versus 36.46%.\n\nMost benchmark tooling is built for one-time model rankings, not the daily grind of comparing a baseline agent to a candidate change. DeltaSelect is explicitly not meant for the former. That distinction matters because teams iterating on prompts, skills, or fine-tunes need fast, cheap feedback loops, and running a full suite on every tweak is the kind of cost that quietly discourages experimentation.\n\nWorth noting: the cost savings held up statistically, but the score improvement did not, with a p value of 0.326, a reminder that cheaper testing and better testing are not automatically the same thing.","[\"coding-agents\",\"benchmarking\",\"ai-evaluation\",\"open-source\"]","2026-09-18T04:00:00.000Z","2026-09-18T19:53:12.159Z","2026-09-18T19:53:24.076Z","published",null,[],"ai",[26,27,28,29],"coding-agents","benchmarking","ai-evaluation","open-source",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.19607",0,{"sections":36},[37,40,44,49,54,58,62,67,71,76,81,86,91,96],{"name":38,"slug":24,"count":39,"latest_published_at":18},"AI",4031,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",654,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",338,"2026-09-11T04:00:00.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":18},"Hardware","hardware",155,{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",121,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",99,"2026-09-09T17:27:33.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":18},"Dev Tools","dev-tools",78,{"name":72,"slug":73,"count":74,"latest_published_at":75},"Software","software",75,"2026-09-10T20:41:21.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Startups","startups",55,"2026-09-09T23:14:29.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",43,"2026-09-10T12:18:06.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"General","general",41,"2026-09-08T01:57:23.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]