[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-atma-pushes-context-windows-32x-past-training-length":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},2978,"atma-pushes-context-windows-32x-past-training-length","ATMA Pushes Context Windows 32x Past Training Length","A new hybrid architecture combines polar attention and recurrent memory compression to keep language models coherent far beyond their training context.","A research team has published ATMA, an architecture designed to stop language models from falling apart when asked to process long documents.\n\nThe core problem ATMA targets is well-known: standard softmax attention spreads probability mass thinner as context grows, causing models to lose the plot past their training length. The researchers also identify a structural conflict between sliding-window attention — which stays sharp locally but misses distant information — and full-context attention, which preserves global recall but degrades badly on out-of-distribution lengths. ATMA is a hybrid convolutional-attention architecture that addresses both failure modes by splitting attention into three channels: one that tracks direction independently of token count, one that measures how many tokens actually contributed to a match, and a recurrent compression memory updated via a gated-delta rule.\n\nThe practical result is notable: in ablation tests across 120 runs, ATMA maintained needle-in-a-haystack retrieval accuracy above 90% at 64K tokens — 32 times its 2K training length — while document perplexity continued to improve rather than explode. That combination has been elusive; most architectures trade one for the other.\n\nThe long-context arms race is crowded. Transformers with extended positional encodings, state-space models like Mamba, and linear-attention hybrids have all promised to crack this problem at various points. ATMA's published code invites scrutiny, but independent benchmarking at production scale — with real-world document distributions rather than needle tests — is where these claims either hold or quietly disappear.","[\"ai\",\"research\",\"transformers\",\"long-context\"]","2026-06-30T04:00:00.000Z","2026-06-30T16:18:41.928Z","2026-06-30T16:18:44.847Z","published",null,[],"ai",[24,26,27,28],"research","transformers","long-context",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.25156",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]