[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-vision-language-models-learn-to-point-at-what-matters":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3854,"vision-language-models-learn-to-point-at-what-matters","Vision-Language Models Learn to Point at What Matters","A new zero-shot framework called TokAG uses attention signals inside large vision-language models to pinpoint where an action can happen in an image.","A research team has built a system that extracts spatial awareness from vision-language models without any task-specific training data.\n\nThe paper introduces TokAG, a framework that solves affordance grounding — identifying which parts of an image support a given action, like \"where can you grasp this object?\" Earlier methods leaned on weakly supervised learning, feeding models labeled images from an outside observer's perspective. Those approaches stumbled when images contained multiple overlapping actions or when two actions looked nearly identical and the labels weren't descriptive enough to distinguish them. TokAG sidesteps the supervision problem entirely by mining the attention maps that large vision-language models already produce internally, then selecting the specific output tokens whose attention lands on the relevant object rather than the background.\n\nThe practical upside is significant: a zero-shot system that needs no labeled affordance data yet outperforms methods that do have it. On the AGD20K benchmark's unseen split, TokAG improved a standard spatial-accuracy metric by 10.7%; on HICO-IIF, the gain was 29.7%. That matters for robotics and embodied AI, where collecting action-annotated training data for every new object or environment is expensive and slow.\n\nThe broader trend here is researchers treating large vision-language models less as end products and more as feature banks — squeezing out spatial signals that the models learned implicitly but were never asked to expose. Whether that approach scales to the noisy, cluttered scenes a real robot encounters is the open question the benchmarks don't yet answer.","[\"ai\",\"computer-vision\",\"robotics\",\"research\"]","2026-07-07T04:00:00.000Z","2026-07-07T10:07:23.577Z","2026-07-07T10:07:26.515Z","published",null,[],"ai",[24,26,27,28],"computer-vision","robotics","research",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.03595",0,{"sections":35},[36,40,45,50,55,60,65,70,75,79,84,88,93,98],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":18},"Dev Tools","dev-tools",59,{"name":80,"slug":81,"count":82,"latest_published_at":83},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":85,"slug":86,"count":82,"latest_published_at":87},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]