[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-method-fixes-mixture-of-experts-upcycling-collapse":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},10860,"new-method-fixes-mixture-of-experts-upcycling-collapse","New Method Fixes Mixture-of-Experts Upcycling Collapse","A new training trick lets smaller mixture-of-experts models match bigger rivals by forcing tokens to pull from different domain specialists.","A new technique called DivMoE fixes a routing problem that has made fine-grained mixture-of-experts upcycling nearly useless.\n\nResearchers describe DivMoE, a method for converting dense language models into sparse mixture-of-experts models without training from scratch. The team found that when fine-grained experts are split from a single source model, the router collapses and accuracy drops close to random guessing - on Qwen3-1.7B, one existing method scored just 23.2% average accuracy across 15 benchmarks, barely above training from scratch at 22.2%. DivMoE instead builds experts from models that have already gone through domain-specific continual pre-training, then forces each token to draw from experts in different domain groups rather than letting the router pick favorites. Across two base models and 15 benchmarks, DivMoE hit 55.6% average accuracy versus 51.6% for the best existing upcycling baseline, and beat the dense base model on every single benchmark.\n\nThat gap matters because upcycling exists to make scaling cheaper, not to produce a model that performs worse than the one you started with. A fine-grained DivMoE model with 12 billion parameters matched the 16 billion parameter Moonlight-MoE at 64.5% average accuracy after fine-tuning, suggesting the routing fix - not just extra experts - was the missing piece for getting fine-grained MoE to pay off.\n\nMixture-of-experts has been the industry's favorite way to grow models without growing compute bills; this paper is a reminder that the cheap route to get there has had a hidden tax most labs were paying without noticing.","[\"mixture-of-experts\",\"llm-training\",\"ai-research\",\"model-architecture\"]","2026-10-09T04:00:00.000Z","2026-10-09T18:49:03.059Z","2026-10-09T18:49:08.260Z","published",null,[],"ai",[26,27,28,29],"mixture-of-experts","llm-training","ai-research","model-architecture",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.11317",0,{"sections":36},[37,40,44,49,54,59,63,68,73,78,83,88,93,98],{"name":38,"slug":24,"count":39,"latest_published_at":18},"AI",6605,{"name":41,"slug":42,"count":43,"latest_published_at":18},"Security","security",926,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",486,"2026-10-08T22:40:11.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",474,"2026-10-08T22:00:00.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",229,"2026-10-08T20:47:10.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":18},"Science","science",192,{"name":64,"slug":65,"count":66,"latest_published_at":67},"Consumer Tech","consumer-tech",181,"2026-10-08T23:26:35.000Z",{"name":69,"slug":70,"count":71,"latest_published_at":72},"Startups","startups",117,"2026-10-08T16:45:00.000Z",{"name":74,"slug":75,"count":76,"latest_published_at":77},"Software","software",114,"2026-10-08T17:57:01.000Z",{"name":79,"slug":80,"count":81,"latest_published_at":82},"Dev Tools","dev-tools",105,"2026-10-07T16:59:11.000Z",{"name":84,"slug":85,"count":86,"latest_published_at":87},"General","general",66,"2026-10-09T04:46:11.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"Gaming","gaming",58,"2026-10-08T20:08:45.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",34,"2026-10-08T14:00:22.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]