[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-lumos-gives-ai-agents-a-smarter-way-to-read-your-screen":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3063,"lumos-gives-ai-agents-a-smarter-way-to-read-your-screen","LUMOS Gives AI Agents a Smarter Way to Read Your Screen","A new research layer converts OS accessibility metadata into machine-readable blueprints, cutting AI agents' reliance on costly screenshot interpretation.","AI agents that control computers have been squinting at screenshots — and a new research system wants to fix that.\n\nLUMOS, short for Language Model Unified Machine-Readable Operating-System Semantics, sits between an AI agent and the operating system, converting existing accessibility metadata and browser UI structures into structured semantic blueprints. Each element gets a stable identifier, a role, a name, bounds, and a list of what actions it supports. Instead of parsing a pixel grid, the agent reads a clean map. The system also supports real-time cursor grounding by querying which UI element sits under or near the pointer via OS automation APIs.\n\nThe practical upside is meaningful: screenshot-based agents burn tokens interpreting images, introduce coordinate uncertainty, and slow down on visually ambiguous layouts. LUMOS sidesteps those costs wherever the OS already exposes semantic structure through its accessibility layer — which most modern desktop and browser environments do, quietly and largely unused by current agent frameworks. That is a lot of free infrastructure that agents have been ignoring.\n\nThe honest caveat is that accessibility APIs were designed for assistive technology, not agent control, and real-world coverage is uneven — plenty of apps expose incomplete or stale accessibility trees. LUMOS does not claim to replace visual agents entirely; it claims to reduce how often they are necessary. That is a narrower promise than the paper's title implies, but also a more achievable one.","[\"ai\",\"accessibility\",\"agents\",\"research\"]","2026-07-01T04:00:00.000Z","2026-07-01T06:21:01.454Z","2026-07-01T06:21:04.446Z","published",null,[],"ai",[24,26,27,28],"accessibility","agents","research",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.30697",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]