From edf54621d20f1195befad8020eba7bbac1ff1dda Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Sun, 15 Mar 2026 17:18:27 +0100 Subject: [PATCH] Fix long max. context bottleneck (#1430) --- src/llama.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 148abcdc..63363281 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -1182,6 +1182,18 @@ static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) { return 0; } +static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache, uint32_t pad) { + for (uint32_t i = cache.size; i > 0; i -= pad) { + const llama_kv_cell & cell = cache.cells[i - 1]; + + if (cell.pos >= 0 && !cell.is_empty()) { + return i; + } + } + + return 0; +} + static void llama_kv_cache_clear(struct llama_kv_cache & cache) { for (int32_t i = 0; i < (int32_t) cache.size; ++i) { cache.cells[i].pos = -1; @@ -3398,8 +3410,8 @@ static int llama_decode_internal( // after enough generations, the benefit from this heuristic disappears // if we start defragmenting the cache, the benefit from this will be more important const uint32_t pad = llama_kv_cache_get_padding(cparams); - kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad))); - //kv_self.n = llama_kv_cache_cell_max(kv_self); + auto max_cell = llama_kv_cache_cell_max(kv_self, pad); + kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(max_cell, pad))); } } if (stop_internal_decode) {