diff --git a/examples/server/server-common.cpp b/examples/server/server-common.cpp index 4b35f488..c8ec90d6 100644 --- a/examples/server/server-common.cpp +++ b/examples/server/server-common.cpp @@ -1199,7 +1199,9 @@ size_t server_tokens::size_up_to_pos(llama_pos max_pos) const { size_t idx = 0; llama_pos pos = 0; - + if (pos >= max_pos) { + return idx; + } while (idx < tokens.size()) { const auto media_it = map_idx_to_media.find(idx); if (media_it != map_idx_to_media.end()) { diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 66695af0..fcdae78b 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -449,8 +449,8 @@ void server_slot::prompt_save(server_prompt_cache& prompt_cache) const { llama_state_seq_get_data(ctx, cur->data.data(), cur_size, id, 0); } -void server_slot::prompt_load(server_prompt_cache& prompt_cache, const server_tokens& tokens) { - bool res = prompt_cache.load(server_cached_prompt, tokens, ctx, id); +void server_slot::prompt_load(server_prompt_cache& prompt_cache, const server_tokens& tokens, float min_reusable_fraction) { + bool res = prompt_cache.load(server_cached_prompt, tokens, ctx, id, min_reusable_fraction); if (!res) { LLAMA_LOG_INFO("failed to load prompt from cache\n"); } @@ -1010,7 +1010,7 @@ server_slot* server_context::get_available_slot(const server_task& task) { const int64_t t_start = ggml_time_us(); copy_data_to_cached_prompt(tokens, *ret); - ret->prompt_load(*prompt_cache, task.tokens); + ret->prompt_load(*prompt_cache, task.tokens, cache_ram_similarity); prompt_cache->update(); ret->cache_tokens = ret->server_cached_prompt.tokens.clone(); // recover cache tokens @@ -3816,25 +3816,14 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t GGML_ASSERT(slot.ga_n == 1); // reuse any previously computed tokens that are common with the new prompt - common_prefix prefix = slot.cache_tokens.get_common_prefix(ctx, prompt_tokens, true); // string level match - common_prefix prefix_nonexact = slot.cache_tokens.get_common_prefix(ctx, prompt_tokens, false); - auto n_past0 = slot.cache_tokens.get_common_prefix_exact(prompt_tokens); // token level match - LLAMA_LOG_INFO("======== Cache: cache_size = %d, n_past0 = %d, n_past1 = %d, n_past_prompt1 = %d, n_past2 = %d, n_past_prompt2 = %d\n", (int32_t)slot.cache_tokens.size(), (int32_t)n_past0, (int32_t)prefix.first, (int32_t)prefix.second, (int32_t)prefix_nonexact.first, (int32_t)prefix_nonexact.second); + common_prefix prefix = slot.cache_tokens.get_common_prefix(ctx, prompt_tokens); + LLAMA_LOG_INFO("======== Cache: cache_size = %d, n_past = %d, n_past_prompt = %d\n", (int32_t)slot.cache_tokens.size(), (int32_t)prefix.first, (int32_t)prefix.second); int32_t size_threshold = 20; - if (prefix.first + size_threshold < prefix_nonexact.first) { - // LLAMA_LOG_WARN("Common part contains missing or extra space and new line\n"); - prefix = prefix_nonexact; - } slot.n_past = prefix.first; slot.n_past_prompt = prefix.second; slot.n_past_offset = slot.n_past_prompt - slot.n_past; - - //if (slot.n_past != slot.n_past_prompt) { - // LLAMA_LOG_INFO("Mistokenization found and handled successfully.\n"); - //} if ((slot.n_past + size_threshold < slot.cache_tokens.size())) { - LLAMA_LOG_WARN("Common part does not match fully\n"); int32_t back = 4; if (prefix.second >= back && prefix.first >= back) { print_tokens(slot.prompt_tokens, slot.cache_tokens, prefix.second - back, prefix.first - back, 30); @@ -3847,7 +3836,6 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t } } } - apply_checkpoint(slot); if (slot.n_past_prompt == slot.n_prompt_tokens && slot.n_past_prompt > 0) { // we have to evaluate at least 1 token to generate logits. LOG_INFO("we have to evaluate at least 1 token to generate logits", { @@ -3861,6 +3849,7 @@ void server_context::batch_pending_prompt(const int32_t n_ubatch, const int32_t slot.n_past_se--; } } + apply_checkpoint(slot); slot.n_prompt_tokens_cache = slot.n_past_prompt; slot.n_prompt_tokens_processed = 0; } diff --git a/examples/server/server-context.h b/examples/server/server-context.h index 68e0d215..015311a5 100644 --- a/examples/server/server-context.h +++ b/examples/server/server-context.h @@ -116,7 +116,7 @@ struct server_slot { void prompt_save(server_prompt_cache& prompt_cache) const; - void prompt_load(server_prompt_cache& prompt_cache, const server_tokens& tokens); + void prompt_load(server_prompt_cache& prompt_cache, const server_tokens& tokens, float min_reusable_fraction); size_t checkpoint_pos = 0; bool do_checkpoint = false; diff --git a/examples/server/server-task.cpp b/examples/server/server-task.cpp index 95287820..82be8b45 100644 --- a/examples/server/server-task.cpp +++ b/examples/server/server-task.cpp @@ -1073,7 +1073,7 @@ size_t server_prompt_cache::n_tokens() const { } -bool server_prompt_cache::load(server_prompt& prompt, const server_tokens& tokens_new, llama_context* ctx, int32_t id_slot) { +bool server_prompt_cache::load(server_prompt& prompt, const server_tokens& tokens_new, llama_context* ctx, int32_t id_slot, float min_reusable_fraction) { thinking_tokens think_tokens; for (auto it = states.begin(); it != states.end(); ++it) { think_tokens = it->think_tokens; @@ -1107,6 +1107,9 @@ bool server_prompt_cache::load(server_prompt& prompt, const server_tokens& token } const auto lcp_cur = tokens.get_common_prefix(ctx, tokens_new_ex); const float f_keep_cur = float(lcp_cur.first) / tokens.size(); + if (f_keep_cur < min_reusable_fraction) { + continue; + } const float sim_cur = tokens.get_tokens_similarity(ctx, tokens_new_ex, it->n_kept_prompt, it->n_discarded_prompt); if (sim_best < sim_cur) { f_keep_best = f_keep_cur; diff --git a/examples/server/server-task.h b/examples/server/server-task.h index 76a6bad3..417d0a3f 100644 --- a/examples/server/server-task.h +++ b/examples/server/server-task.h @@ -446,7 +446,7 @@ struct server_prompt_cache { server_prompt* alloc(const server_prompt& prompt, size_t state_size); - bool load(server_prompt& prompt, const server_tokens& tokens_new, llama_context* ctx, int32_t id_slot); + bool load(server_prompt& prompt, const server_tokens& tokens_new, llama_context* ctx, int32_t id_slot, float min_reusable_fraction); void update(); };