[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-measuring-text-randomness-could-fix-ais-collapse-problem":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},9733,"measuring-text-randomness-could-fix-ais-collapse-problem","Measuring Text Randomness Could Fix AI's Collapse Problem","A text-only entropy measurement beat model-based filtering at stopping AI models from losing diversity after repeated fine-tuning on their own output.","A new filtering method can catch degrading AI training data without needing a model to check it.\n\nResearchers studying \"model collapse\" - the way AI models lose vocabulary and start repeating phrases after repeated rounds of fine-tuning on their own generated text - tested a statistic called the Kontoyiannis entropy rate estimator. It is computed directly from raw text using match-length statistics, with no model involved at all. In a six-generation fine-tuning test on Llama-3.1-8B, filtering training data with this entropy measure increased unique trigrams by 42%, vocabulary by 30%, and cut repetition by 19%, all statistically significant results. A standard baseline that filters using a model's own log-probabilities showed no measurable improvement on any of those diversity metrics in the same test.\n\nModel collapse is a growing concern as more of the text available for training is itself AI-generated, and most existing fixes assume you have a working model or clean human data on hand to check against. This method needs neither. Because it is just text statistics, it is cheap enough to run at scale, and it could help keep multi-agent systems from converging into the same repetitive outputs when no single trusted model is available to referee.\n\nThe researchers tested the approach across four domains, two temperatures, and two generator-scorer model pairs. That is still a narrow slice of the real world, but it beat a well-established baseline that could not clear statistical significance at all - a reminder that fancier does not automatically mean better.","[\"ai\",\"model collapse\",\"synthetic data\",\"llm training\"]","2026-10-02T04:00:00.000Z","2026-10-03T08:18:23.979Z","2026-10-03T08:18:28.163Z","published",null,[],"ai",[24,26,27,28],"model collapse","synthetic data","llm training",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2610.01493",0,{"sections":35},[36,39,43,47,52,56,60,65,70,75,80,85,90,95],{"name":37,"slug":24,"count":38,"latest_published_at":18},"AI",6041,{"name":40,"slug":41,"count":42,"latest_published_at":18},"Security","security",848,{"name":44,"slug":45,"count":46,"latest_published_at":18},"Policy","policy",439,{"name":48,"slug":49,"count":50,"latest_published_at":51},"Deals","deals",317,"2026-10-01T22:00:00.000Z",{"name":53,"slug":54,"count":55,"latest_published_at":18},"Hardware","hardware",199,{"name":57,"slug":58,"count":59,"latest_published_at":18},"Science","science",176,{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",155,"2026-10-01T19:54:10.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Dev Tools","dev-tools",96,"2026-10-01T16:57:03.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Software","software",93,"2026-09-30T21:41:11.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Startups","startups",90,"2026-10-01T21:55:22.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",53,"2026-10-02T02:50:39.000Z",{"name":86,"slug":87,"count":88,"latest_published_at":89},"General","general",50,"2026-09-30T21:37:54.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"Reviews","reviews",31,"2026-09-28T14:31:34.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"How-To","how-to",7,"2026-10-01T09:00:00.000Z"]