nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix long max. context bottleneck (#1430)

This commit is contained in:
Kawrakow 2026-03-15 17:18:27 +01:00 committed by GitHub
parent 49778a6ce0
commit edf54621d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/llama.cpp
View File

@ -1182,6 +1182,18 @@ static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache) {
return 0;
}
static uint32_t llama_kv_cache_cell_max(const struct llama_kv_cache & cache, uint32_t pad) {
for (uint32_t i = cache.size; i > 0; i -= pad) {
const llama_kv_cell & cell = cache.cells[i - 1];
if (cell.pos >= 0 && !cell.is_empty()) {
return i;
}
}
return 0;
}
static void llama_kv_cache_clear(struct llama_kv_cache & cache) {
for (int32_t i = 0; i < (int32_t) cache.size; ++i) {
cache.cells[i].pos = -1;
@ -3398,8 +3410,8 @@ static int llama_decode_internal(
// after enough generations, the benefit from this heuristic disappears
// if we start defragmenting the cache, the benefit from this will be more important
const uint32_t pad = llama_kv_cache_get_padding(cparams);
kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(llama_kv_cache_cell_max(kv_self), pad)));
//kv_self.n = llama_kv_cache_cell_max(kv_self);
auto max_cell = llama_kv_cache_cell_max(kv_self, pad);
kv_self.n = std::min(kv_self.size, std::max(pad, GGML_PAD(max_cell, pad)));
}
}
if (stop_internal_decode) {