[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-tiny-lora-patch-unlocks-depth-transformers-already-had":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},8739,"tiny-lora-patch-unlocks-depth-transformers-already-had","Tiny LoRA Patch Unlocks Depth Transformers Already Had","A rank-8 LoRA patch, trained on one task, lets frozen transformers follow far more reference chains than their pretrained defaults allow.","A small, frozen-weight patch shows transformer language models can track far longer chains of context than they normally bother to.\n\nResearchers tested thirteen pretrained transformer base models on a synthetic task that requires following chains of variable references. Each model reliably tracked only 1.4 to 3.6 steps before losing the thread, and giving the models extra pretrained processing loops barely helped. So the team froze every original weight and added one small rank-8 LoRA adapter at a single early layer, trained only on the chain-following task. That tiny patch pushed Qwen3-8B's accuracy on 24-step chains from 15.5% to 99%, and a longer-trained version handled chains up to 50 steps. A different model, Ouro-1.4B, reached 60-step chains after four processing loops and more than 160 after eight. The same patch also boosted scores on MuSiQue, a multi-hop question-answering benchmark.\n\nThat's a bigger deal than it sounds. It suggests the reasoning limits models show by default aren't hard architectural ceilings baked in during pretraining. They're more like an unused setting left off. A few million new parameters, applied to a single layer, recovered computation the base model apparently had all along.\n\nStill, this is one synthetic task and one QA benchmark, not messy real-world reasoning, and a benchmark built to test exactly the skill you just patched in is the easiest kind of benchmark to ace.","[\"ai\",\"llms\",\"fine-tuning\",\"reasoning\"]","2026-09-30T04:00:00.000Z","2026-09-30T22:59:00.966Z","2026-09-30T22:59:07.585Z","published",null,[],"ai",[24,26,27,28],"llms","fine-tuning","reasoning",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.36585",0,{"sections":35},[36,40,45,50,55,59,63,67,72,76,81,86,91,96],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",5214,"2026-09-30T13:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",793,"2026-09-30T12:55:00.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",419,"2026-09-30T12:24:32.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",291,"2026-09-30T10:38:22.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":39},"Hardware","hardware",196,{"name":60,"slug":61,"count":62,"latest_published_at":18},"Science","science",155,{"name":64,"slug":65,"count":66,"latest_published_at":39},"Consumer Tech","consumer-tech",144,{"name":68,"slug":69,"count":70,"latest_published_at":71},"Dev Tools","dev-tools",91,"2026-09-30T12:58:00.000Z",{"name":73,"slug":74,"count":70,"latest_published_at":75},"Software","software","2026-09-25T20:55:00.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Startups","startups",83,"2026-09-29T21:51:36.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"General","general",49,"2026-09-28T16:44:57.000Z",{"name":87,"slug":88,"count":89,"latest_published_at":90},"Gaming","gaming",48,"2026-09-25T18:35:21.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"Reviews","reviews",31,"2026-09-28T14:31:34.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]