diff --git a/src/llama.cpp b/src/llama.cpp index 148abcdc..63363281 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -1182,6 +1182,18 @@ static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) { return 0; } +static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache, uint32_t pad) { + for (uint32_t i = cache.size; i > 0; i -= pad) { + const llama_kv_cell & cell = cache.cells[i - 1]; + + if (cell.pos >= 0 && !cell.is_empty()) { + return i; + } + } + + return 0; +} + static void llama_kv_cache_clear(struct llama_kv_cache & cache) { for (int32_t i = 0; i < (int32_t) cache.size; ++i) { cache.cells[i].pos = -1; @@ -3398,8 +3410,8 @@ static int llama_decode_internal( // after enough generations, the benefit from this heuristic disappears // if we start defragmenting the cache, the benefit from this will be more important const uint32_t pad = llama_kv_cache_get_padding(cparams); - kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad))); - //kv_self.n = llama_kv_cache_cell_max(kv_self); + auto max_cell = llama_kv_cache_cell_max(kv_self, pad); + kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(max_cell, pad))); } } if (stop_internal_decode) {