[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-routing-method-lets-ai-models-read-their-own-attention":10,"sections":36},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":24,"persona_id":22,"persona_name":22,"section":25,"tags":26,"sources":31,"feedback":35,"feedback_at":22,"cost_usd":35,"total_tokens":35},7064,"new-routing-method-lets-ai-models-read-their-own-attention","New Routing Method Lets AI Models Read Their Own Attention","A new technique feeds attention patterns back into MoE routers, boosting math reasoning on GSM8K by 3.37 points without touching the base model.","A new routing method lets Mixture-of-Experts models make smarter decisions by borrowing information from their own attention patterns, not just a single snapshot of the hidden state.\n\nIn MoE language models, a router decides which expert sub-network handles each token, typically based only on that token's current hidden state. A team of researchers built Attention-Aware Routing (AAR), which adds temporal and spectral signals pulled from a sliding window of attention weights, essentially a compressed history of what the model has been focusing on. They froze the rest of the transformer and trained only the routing parameters on OLMoE, isolating routing as the single variable that changed. AAR lifted GSM8K math benchmark scores by 3.37 percentage points over a routing-only baseline, and wrong answers got shorter while correct ones stayed the same length.\n\nThe more interesting finding is mechanistic: routing decisions at one layer measurably reshape attention behavior, specifically strengthening so-called attention sinks, at the next layer, even though nothing about attention itself was directly changed. That means routing and attention aren't separate systems bolted together, they're a coupled circuit influencing each other through the residual stream. It's a small but genuine clue about how these models actually process information, distinct from another leaderboard-chasing tweak.\n\nThe catch: applying AAR indiscriminately across every layer hurts factual retrieval, even as it helps math reasoning when placed deeper in the network. This isn't a drop-in upgrade yet. It's a scalpel, tested so far on exactly one model.","[\"mixture-of-experts\",\"llm-routing\",\"ai-research\",\"reasoning\"]","2026-09-21T04:00:00.000Z","2026-09-21T04:30:42.519Z","2026-09-21T04:30:54.429Z","published",null,[],"https:\u002F\u002Fcdn.xyz.onl\u002Farticle-images\u002Fnew-routing-method-lets-ai-models-read-their-own-attention.webp","ai",[27,28,29,30],"mixture-of-experts","llm-routing","ai-research","reasoning",[32],{"name":33,"url":34},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.20974",0,{"sections":37},[38,41,45,50,55,60,65,70,75,80,85,90,95,100],{"name":39,"slug":25,"count":40,"latest_published_at":18},"AI",4158,{"name":42,"slug":43,"count":44,"latest_published_at":18},"Security","security",679,{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",350,"2026-09-20T20:32:43.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",156,"2026-09-19T11:00:00.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Science","science",130,"2026-09-20T13:48:11.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Consumer Tech","consumer-tech",99,"2026-09-09T17:27:33.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Dev Tools","dev-tools",78,"2026-09-18T04:00:00.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Software","software",75,"2026-09-10T20:41:21.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Startups","startups",55,"2026-09-09T23:14:29.000Z",{"name":86,"slug":87,"count":88,"latest_published_at":89},"Gaming","gaming",43,"2026-09-10T12:18:06.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"General","general",42,"2026-09-18T22:35:10.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]