[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-researchers-build-a-benchmark-for-ai-that-sees-and-hears-you":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":24,"persona_id":22,"persona_name":22,"section":25,"tags":26,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},7093,"researchers-build-a-benchmark-for-ai-that-sees-and-hears-you","Researchers Build a Benchmark for AI That Sees and Hears You","A new benchmark and training method push AI models to answer questions straight from raw audio and video, skipping the transcription step entirely.","A team of researchers has defined a task for AI assistants that skips typing altogether: you talk, the camera watches, and the model answers, with no text box and no speech-to-text step in between.\n\nThey call the setup OmniVChat, for omni video chat. In it, an AI model gets a user's question embedded directly in a video and audio clip, rather than a separate typed prompt or a captioned transcript. To train and grade this, the researchers built OmniVChat-Studio, a multi-agent system that generates synthetic audio-visual conversations, and OmniVChat-Bench, a test that scores replies across five ability categories. They also designed OmniVChat-RL, a reinforcement-learning reward that grades answers on correctness, efficiency, and style together. Applying it to Qwen3-Omni-Instruct improved its scores on both the synthetic benchmark and a smaller benchmark built from real human recordings.\n\nThe real obstacle here isn't model design, it's data and grading. Recordings of people actually talking to their devices, with their faces, rooms, and gestures in frame, are rare and awkward to collect at scale. And a good answer to something like \"what's wrong with my plant\" while holding up a wilting leaf can't be checked against a fixed set of keywords. Building synthetic dialogues to train and test that problem, and showing the gains hold up on real recordings too, is the useful part.\n\nStill, this is one lab's benchmark trained on one open model, not a shootout against Gemini's or GPT's live video modes, so treat the improvement numbers as a proof of concept rather than a verdict on who builds the best video-chatting assistant.","[\"ai\",\"multimodal-ai\",\"benchmarks\",\"voice-assistants\"]","2026-09-21T04:00:00.000Z","2026-09-21T06:00:09.872Z","2026-09-21T06:00:22.154Z","published",null,[],"https:\u002F\u002Fcdn.xyz.onl\u002Farticle-images\u002Fresearchers-build-a-benchmark-for-ai-that-sees-and-hears-you.webp","ai",[25,27,28,29],"multimodal-ai","benchmarks","voice-assistants",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.21465",0,{"sections":36},[37,40,44,49,54,59,64,69,74,79,84,89,94,99],{"name":38,"slug":25,"count":39,"latest_published_at":18},"AI",4158,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",679,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",350,"2026-09-20T20:32:43.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",156,"2026-09-19T11:00:00.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":63},"Science","science",130,"2026-09-20T13:48:11.000Z",{"name":65,"slug":66,"count":67,"latest_published_at":68},"Consumer Tech","consumer-tech",99,"2026-09-09T17:27:33.000Z",{"name":70,"slug":71,"count":72,"latest_published_at":73},"Dev Tools","dev-tools",78,"2026-09-18T04:00:00.000Z",{"name":75,"slug":76,"count":77,"latest_published_at":78},"Software","software",75,"2026-09-10T20:41:21.000Z",{"name":80,"slug":81,"count":82,"latest_published_at":83},"Startups","startups",55,"2026-09-09T23:14:29.000Z",{"name":85,"slug":86,"count":87,"latest_published_at":88},"Gaming","gaming",43,"2026-09-10T12:18:06.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",42,"2026-09-18T22:35:10.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]