[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-a-smarter-way-to-deduplicate-text-at-scale":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3391,"a-smarter-way-to-deduplicate-text-at-scale","A Smarter Way to Deduplicate Text at Scale","SemHash-LLM combines multiple detection layers to find near-duplicate documents without routing most of them through a full neural model.","Researchers have published a framework that makes large-scale document deduplication cheaper by doing most of the work before ever touching an LLM.\n\nThe system, called SemHash-LLM, stacks four techniques in sequence: semantic projection hashing, attention-weighted MinHash, contrastive boundary learning, and LLM-based adjudication as a last resort. Each layer filters candidates down so the expensive neural step handles less than one percent of the total corpus. The method operates at character, token, and document levels simultaneously, fusing those signals through a gating mechanism. It also includes uncertainty estimation to handle edge cases like boilerplate-heavy templates, short text variations, and viral content fragments that copy partially rather than verbatim.\n\nDeduplication sounds like housekeeping, but it has a direct effect on model quality. Training sets riddled with near-duplicates skew weights toward repeated content, and post-training retrieval systems that index redundant documents degrade both speed and relevance. The push to process corpora at trillion-token scale makes brute-force pairwise comparison impractical, which is why hybrid pipelines like this one matter.\n\nThe cascaded approach is not new — MinHash-based deduplication has been standard infrastructure since at least the C4 dataset era — but pairing it with learned binary codes in a distilled embedding space and reserving the full model only for uncertain cases is a cleaner architecture than most published alternatives. Whether the benchmark gains hold on truly heterogeneous web crawls, rather than cleaner research corpora, is the question the paper leaves open.","[\"ai\",\"nlp\",\"data-infrastructure\",\"research\"]","2026-07-03T04:00:00.000Z","2026-07-03T04:53:49.615Z","2026-07-03T04:53:52.604Z","published",null,[],"ai",[24,26,27,28],"nlp","data-infrastructure","research",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.01601",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]