[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ps-ppo-cuts-rlhf-training-cost-without-a-critic":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},2723,"ps-ppo-cuts-rlhf-training-cost-without-a-critic","PS-PPO Cuts RLHF Training Cost Without a Critic","A new algorithm trims compute and GPU memory in reinforcement learning from human feedback by skipping the parts of a reasoning trace that don't add new signal.","A research paper proposes a leaner way to fine-tune large language models with human feedback — by not processing every token in a response.\n\nCurrent critic-free RLHF methods apply a single reward signal uniformly across an entire generated sequence, token by token, even when the early part of that sequence already determines where things are headed. Prefix-Sampling PPO, or PS-PPO, sidesteps this by randomly sampling a cutoff point in each trajectory during training and backpropagating only through that shorter prefix. An importance-weighting correction keeps the gradient estimate mathematically consistent with what you'd get from processing the full sequence. The method is built on top of Proximal Policy Optimization, a well-established training algorithm, and tested on mathematical reasoning tasks and standard RLHF benchmarks.\n\nThe practical upshot is meaningful: less compute per training step and lower peak GPU memory, without a measurable drop in accuracy compared to existing critic-free baselines. That matters because long reasoning traces — the kind increasingly produced by chain-of-thought and reasoning models — make RLHF expensive to run, and most efficiency work in this space has focused on inference rather than training. Cutting training cost without degrading the reward signal is a harder problem than it sounds.\n\nCritic-free RLHF methods have gained traction partly because actor-critic setups require a separate value network that doubles memory overhead; PS-PPO chips away at the remaining cost on the actor side. Whether the gains hold at the scale labs actually train on is the obvious question the paper's benchmark numbers can't fully answer.","[\"ai\",\"machine-learning\",\"rlhf\",\"training-efficiency\"]","2026-06-30T04:00:00.000Z","2026-06-30T11:35:38.684Z","2026-06-30T11:35:41.588Z","published",null,[],"ai",[24,26,27,28],"machine-learning","rlhf","training-efficiency",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.29758",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]