[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-full-stack-fp4-pretraining-closes-the-gap-on-bf16-llms":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3938,"full-stack-fp4-pretraining-closes-the-gap-on-bf16-llms","Full-Stack FP4 Pretraining Closes the Gap on BF16 LLMs","A new framework tackles all three numerical stability problems blocking 4-bit LLM pretraining, leaving only a 1.47% loss gap versus standard precision.","Researchers have built the first complete 4-bit pretraining pipeline for large language models that doesn't fall apart at the optimizer or attention layers.\n\nPrior NVFP4 pretraining work focused almost entirely on transformer linear layers, leaving optimizer states and attention computations in higher precision — a workaround, not a solution. The new Full-Stack FP4 framework tackles all three problem areas at once. For linear projections, a technique called LoRA-SVD decomposition cuts the quantization loss gap from 1.40% to 0.61%. For the AdamW optimizer's second moments — which are heavy-tailed and notoriously fragile at low precision — the researchers redesigned how values are stored and computed in NVFP4. Attention gets a mixed-precision treatment: Q, K, and V matrices are quantized while the numerically sensitive paths stay in BF16, with a tensor-reuse scheme to keep forward and backward passes consistent. A 3-billion-parameter model trained on 64 billion tokens came within 1.47% of standard BF16 performance.\n\nFP4 arithmetic can roughly double memory efficiency and throughput compared to BF16, so a stable end-to-end pipeline matters well beyond a single benchmark. Until now, every attempt to push 4-bit math into optimizers or attention introduced error accumulation that made full training runs unreliable — a wall this work claims to clear.\n\nThe caveat is that the validation run is 3B parameters and 64B tokens — respectable for an academic paper, but smaller than the frontier models this technique would need to survive in production. Whether the 1.47% loss gap stays that narrow at 70B parameters and a trillion tokens is the question the next paper will have to answer.","[\"ai\",\"llm\",\"quantization\",\"training\"]","2026-07-07T04:00:00.000Z","2026-07-07T12:41:20.986Z","2026-07-07T12:41:23.945Z","published",null,[],"ai",[24,26,27,28],"llm","quantization","training",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.04422",0,{"sections":35},[36,40,45,50,55,60,65,70,75,79,84,88,93,98],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":18},"Dev Tools","dev-tools",59,{"name":80,"slug":81,"count":82,"latest_published_at":83},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":85,"slug":86,"count":82,"latest_published_at":87},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]