[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-jarvisbench-measures-when-ai-agents-should-interrupt-you":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},5139,"jarvisbench-measures-when-ai-agents-should-interrupt-you","JarvisBench Measures When AI Agents Should Interrupt You","A new arXiv benchmark grades AI systems on knowing when to interrupt busy humans and when to quietly keep working, using 45 tasks across 19 domains.","Researchers have proposed a benchmark for a problem most agent evaluations skip: knowing when to interrupt a human.\n\nA new research paper introduces JarvisBench, a test suite for an \"attention-coordination layer\" the authors call Jarvis, named after the fictional Iron Man assistant. The system sits between a person and one or more autonomous agents, deciding when an agent should keep working alone and when it needs a human's judgment. JarvisBench includes 45 task instances - 20 single-agent tasks and 25 workstreams grouped into 10 multi-agent projects - spanning 19 domains, selected and adapted from more than 2,000 public candidates. The tasks are built so the need for a human check-in surfaces naturally mid-task, not because someone left something out of the original prompt.\n\nMost agent benchmarks score whether a task got finished, not whether the agent bothered the right person at the right moment - a gap that matters more as coding agents and browser agents run unattended for hours at a stretch. JarvisBench also grades the reverse direction: whether the coordination layer can answer a user's question about background work accurately and promptly without derailing the agent. That's a genuinely underexplored axis, closer to human-computer-interaction research than to a typical task-completion leaderboard.\n\nThe reference implementation ships with a full-duplex speech interface, a nice demo but also a reminder that a 45-task benchmark for a system named after a movie AI is a proof of concept, not an industry standard yet.","[\"ai agents\",\"benchmarks\",\"human-computer interaction\",\"research\"]","2026-08-18T04:00:00.000Z","2026-08-18T06:40:18.273Z","2026-08-18T06:40:30.087Z","published",null,[],"ai",[26,27,28,29],"ai agents","benchmarks","human-computer interaction","research",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2608.14870",0,{"sections":36},[37,41,45,50,55,60,65,70,75,79,84,89,94,99],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",3293,"2026-08-20T04:00:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":40},"Security","security",435,{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",210,"2026-08-19T09:32:27.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",140,"2026-08-19T18:25:42.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",95,"2026-08-18T16:05:00.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Science","science",90,"2026-08-19T18:41:02.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Software","software",73,"2026-08-18T07:51:50.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":18},"Dev Tools","dev-tools",69,{"name":80,"slug":81,"count":82,"latest_published_at":83},"Startups","startups",47,"2026-08-19T19:13:46.000Z",{"name":85,"slug":86,"count":87,"latest_published_at":88},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",33,"2026-08-18T22:18:13.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]