[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-a-smarter-sparse-attention-that-handles-far-longer-contexts":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3788,"a-smarter-sparse-attention-that-handles-far-longer-contexts","A Smarter Sparse Attention That Handles Far Longer Contexts","HiLS Attention matches full attention quality at training length while extrapolating to 64 times that range, without the usual efficiency penalty.","Researchers have a new sparse attention design that may finally make the efficiency-versus-performance trade-off a false choice.\n\nThe paper introduces Hierarchical Landmark Sparse (HiLS) Attention, a chunk-based method that learns which text chunks to retrieve as part of normal language-model training rather than as a bolted-on heuristic. The key architectural move: each query attends to retrieved chunks independently, then the outputs are fused using retrieval scores that feed directly back into the loss function. That loop lets the model optimize its own retrieval end-to-end. In benchmarks, HiLS matches or beats standard full attention at the context lengths it was trained on, and extrapolates to more than 64 times that length while maintaining 90 percent retrieval accuracy.\n\nThat extrapolation number matters because length generalization has been the persistent weak spot of sparse approaches. Most prior methods pick which chunks to attend to using fixed rules or separately trained components — which means errors compound at lengths the model never saw. Tying retrieval directly to the language-modeling objective removes that seam. Equally useful: the paper shows existing full-attention models can be converted to HiLS through lightweight continued pretraining, so this is not a train-from-scratch proposition.\n\nThe usual caveat with arxiv papers applies: results here are the authors' own benchmarks, and independent replication at scale is the real test. Still, if the extrapolation claims hold up, HiLS addresses the exact problem that makes long-context inference expensive enough to price most developers out.","[\"ai\",\"large language models\",\"attention mechanisms\",\"research\"]","2026-07-07T04:00:00.000Z","2026-07-07T08:32:13.120Z","2026-07-07T08:32:16.138Z","published",null,[],"ai",[24,26,27,28],"large language models","attention mechanisms","research",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.02980",0,{"sections":35},[36,40,45,50,55,60,65,70,75,79,84,88,93,98],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":18},"Dev Tools","dev-tools",59,{"name":80,"slug":81,"count":82,"latest_published_at":83},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":85,"slug":86,"count":82,"latest_published_at":87},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]