[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-how-you-search-a-tokenizer-matters-more-than-what-you-optimize":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},6663,"how-you-search-a-tokenizer-matters-more-than-what-you-optimize","How You Search a Tokenizer Matters More Than What You Optimize","A new preprint untangles two tokenizer-building algorithms and finds that how you search for tokens matters more than what you optimize for.","A new preprint pulls apart the two ingredients of a language model's vocabulary-builder and finds one matters far more than the other.\n\nByte-pair encoding (BPE) and UnigramLM are the two tokenization methods behind most modern LLMs, but they differ along two separate dimensions that prior research had lumped together: what they optimize for (squeezing text into fewer tokens vs. maximizing statistical likelihood) and how they search for a vocabulary (building it up merge-by-merge vs. pruning it down from a huge starting set). Researchers built two new algorithms, BottomUpLL and TopDownComp, that swap the objective and search method between BPE and UnigramLM, filling out all four combinations. They trained language models on each of the four tokenizers, testing different model sizes, vocabulary sizes, and English-only versus multilingual data. Performance was measured on bits-per-byte, a compression-efficiency metric, and on BLiMP, a benchmark for grammatical judgment.\n\nThe search procedure, not the optimization objective, turned out to be the deciding factor: tokenizers built bottom-up beat top-down ones on bits-per-byte in most setups, regardless of whether they optimized for compression or likelihood. But that advantage vanished on BLiMP, where no design choice reliably predicted better grammar performance. In other words, the metric you pick to judge a tokenizer changes which design choice looks smart.\n\nTokenization rarely gets attention next to flashier model-architecture news, but it is one of the few places left where a free, unglamorous engineering choice can still move the needle.","[\"tokenization\",\"llms\",\"nlp\",\"ai-research\"]","2026-09-17T04:00:00.000Z","2026-09-18T04:58:54.249Z","2026-09-18T04:59:06.201Z","published",null,[],"ai",[26,27,28,29],"tokenization","llms","nlp","ai-research",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.19145",0,{"sections":36},[37,41,45,50,55,59,63,68,73,77,82,87,92,97],{"name":38,"slug":24,"count":39,"latest_published_at":40},"AI",3853,"2026-09-17T08:27:09.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":18},"Security","security",648,{"name":46,"slug":47,"count":48,"latest_published_at":49},"Policy","policy",338,"2026-09-11T04:00:00.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":18},"Hardware","hardware",154,{"name":60,"slug":61,"count":62,"latest_published_at":18},"Science","science",114,{"name":64,"slug":65,"count":66,"latest_published_at":67},"Consumer Tech","consumer-tech",99,"2026-09-09T17:27:33.000Z",{"name":69,"slug":70,"count":71,"latest_published_at":72},"Software","software",75,"2026-09-10T20:41:21.000Z",{"name":74,"slug":75,"count":76,"latest_published_at":18},"Dev Tools","dev-tools",73,{"name":78,"slug":79,"count":80,"latest_published_at":81},"Startups","startups",55,"2026-09-09T23:14:29.000Z",{"name":83,"slug":84,"count":85,"latest_published_at":86},"Gaming","gaming",43,"2026-09-10T12:18:06.000Z",{"name":88,"slug":89,"count":90,"latest_published_at":91},"General","general",41,"2026-09-08T01:57:23.000Z",{"name":93,"slug":94,"count":95,"latest_published_at":96},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":98,"slug":99,"count":100,"latest_published_at":101},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]