[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-new-method-helps-small-ai-models-learn-without-forgetting":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},11316,"new-method-helps-small-ai-models-learn-without-forgetting","New method helps small AI models learn without forgetting","A new training technique called SynthSelf lets small AI models learn new skills while retaining old ones, addressing a gap in on-device AI.","A new paper describes a way to keep small AI models learning new tricks without erasing the old ones.\n\nResearchers introduce SynthSelf, a continual post-training method built for small models used in low-latency, on-device settings. It builds on an earlier technique called Self-Distillation Fine-Tuning, which generates training data from a model's own in-context demonstrations, but that approach falls apart on small models that lack strong in-context learning and instruction-following skills. The team traced the failure to distribution shift: the more a model's self-generated training data strays from its original distribution, the more it forgets old skills while learning new ones. SynthSelf fixes this by handing the rewriting step to a slightly larger model from the same family, then filtering the output with off-policy supervision and rejection sampling, so the small model gets high-quality training data that stays close to its own distribution.\n\nAcross four acquisition tasks and fourteen retention benchmarks, SynthSelf beat six rival methods, including both on-policy and off-policy distillation, at three different small model sizes. That's notable because most continual-learning research assumes a model big enough to lean on its own in-context tricks - a mismatch for the compact models actually running on phones and laptops.\n\nIt's a narrow fix, but a useful one: the same-family, slightly-bigger-teacher trick swaps a small model's self-confidence for someone else's judgment, without asking it to outgrow its own distribution.","[\"ai\",\"continual-learning\",\"small-language-models\",\"on-device-ai\"]","2026-10-12T04:00:00.000Z","2026-10-12T07:40:41.805Z","2026-10-12T07:40:47.191Z","published",null,[],"ai",[24,26,27,28],"continual-learning","small-language-models","on-device-ai",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.13113",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,84,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",6984,"2026-10-13T16:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",967,"2026-10-13T15:10:00.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",501,"2026-10-13T12:15:53.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",498,"2026-10-13T14:59:12.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",246,"2026-10-13T14:21:55.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",205,"2026-10-13T15:24:17.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Science","science",199,"2026-10-13T12:01:37.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Startups","startups",125,"2026-10-13T15:41:00.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",116,"2026-10-13T16:20:18.000Z",{"name":81,"slug":82,"count":78,"latest_published_at":83},"Software","software","2026-10-12T19:00:00.000Z",{"name":85,"slug":86,"count":87,"latest_published_at":88},"General","general",73,"2026-10-13T13:11:33.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"Gaming","gaming",63,"2026-10-12T20:59:50.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",35,"2026-10-12T14:00:00.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]