[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-kara-trims-kv-cache-to-speed-up-reasoning-ai-inference":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3455,"kara-trims-kv-cache-to-speed-up-reasoning-ai-inference","Kara Trims KV Cache to Speed Up Reasoning AI Inference","A new sliding-window compression method cuts the memory overhead of long chain-of-thought reasoning without gutting output quality.","Researchers have a new approach to one of reasoning AI's quieter bottlenecks: the ballooning memory cost of thinking out loud.\n\nWhen a reasoning model works through a problem step by step, it builds a key-value cache that grows with every token generated. That cache sits in GPU memory, and at scale it slows throughput and drives up latency. Kara, a sliding-window KV cache compression method described in a new paper, tackles this by compressing the cache at decoding time — operating only on recently generated context rather than the full sequence. It uses bidirectional attention to score which KV pairs are worth keeping, then a Token2Chunk module expands selected pairs into flexible-sized chunks rather than forcing rigid fixed-size boundaries. The team also built KvLLM, an inference framework layered on top of vLLM, to make Kara compatible with PagedAttention — the memory management technique most production inference systems already rely on.\n\nThe practical upside is throughput: the paper reports consistent performance improvements across experiments, with reduced KV cache memory usage. Prior compression methods either triggered too infrequently to matter or wiped out entire token blocks, risking worse information loss than no compression at all — Kara's flexible chunk selection is the fix for both.\n\nThis sits inside a broader engineering arms race to make long chain-of-thought models cheaper to run; rivals like quantization and speculative decoding attack the same latency problem from different angles. The vLLM integration is the detail worth watching — it's the inference stack most teams are already running, which means Kara could be adopted without a full infrastructure swap.","[\"ai\",\"inference\",\"llm\",\"research\"]","2026-07-03T04:00:00.000Z","2026-07-03T06:28:25.845Z","2026-07-03T06:28:28.652Z","published",null,[],"ai",[24,26,27,28],"inference","llm","research",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.01237",0,{"sections":35},[36,40,45,50,55,60,65,70,75,80,85,89,94,99],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":86,"slug":87,"count":83,"latest_published_at":88},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":90,"slug":91,"count":92,"latest_published_at":93},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":95,"slug":96,"count":97,"latest_published_at":98},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":100,"slug":101,"count":102,"latest_published_at":103},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]