nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compute the sequence fingerprint once per decode (#2265) 

Compute it before the reuse check, pass it to can_reuse_graph(), and reuse the same
value when the graph is rebuilt, so it is computed exactly once whether the existing
graph is kept or rebuilt.

Verified at np=6, same binary, only IK_LEGACY_GRAPH_REUSE differing: legacy 6/12
replies kept their own codeword, this branch 12/12. The value at the call site matched
the value at the store site on 50 of 50 rebuilds sampled at np=2 and np=6.
This commit is contained in:
ShubhamPriyadarshi 2026-08-06 20:04:42 +05:30 committed by GitHub
parent 4a4a6d3c14
commit 21819724b3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llama-context.h
View File

@ -651,7 +651,7 @@ struct llama_context {
int32_t mtp_n_heads = 0;
void reset_scheduler();
bool can_reuse_graph(const llama_batch & u_batch);
bool can_reuse_graph(const llama_batch & u_batch, uint64_t seq_fingerprint);
struct CacheCopy {
ggml_tensor * cpy = nullptr;

9
src/llama.cpp
View File

@ -617,13 +617,13 @@ static void why_not_reuse_previous(const llama_batch & u_batch, const llama_cont
printf(" update_cache_copies() must have failed\n");
}
bool llama_context::can_reuse_graph(const llama_batch & u_batch) {
bool llama_context::can_reuse_graph(const llama_batch & u_batch, uint64_t seq_fingerprint) {
if (!cparams.graph_reuse) return false;
if (model.arch == LLM_ARCH_DEEPSEEK4) return false;
auto the_prev = cparams.mtp_op_type == MTP_OP_NONE ? prev.get() : prev_mtp.get();
if (!the_prev || !the_prev->graph) return false;
if (u_batch.embd) return false;
if (llama_ubatch_seq_fingerprint(u_batch, model.arch) != the_prev->seq_fingerprint) {
if (seq_fingerprint != the_prev->seq_fingerprint) {
return false;
}
auto & kv_self_used = (model.arch == LLM_ARCH_GEMMA4_MTP || model.arch == LLM_ARCH_GEMMA4_ASSISTANT) &&
@ -6314,7 +6314,8 @@ static int llama_decode_internal(
#endif
auto & prev = cparams.mtp_op_type == MTP_OP_NONE ? lctx.prev : lctx.prev_mtp;
ggml_cgraph * gf = nullptr;
if (!lctx.can_reuse_graph(u_batch)) {
const uint64_t seq_fingerprint = llama_ubatch_seq_fingerprint(u_batch, lctx.model.arch);
if (!lctx.can_reuse_graph(u_batch, seq_fingerprint)) {
lctx.reset_scheduler();
ggml_backend_sched_set_eval_callback(lctx.sched, lctx.cparams.cb_eval, lctx.cparams.cb_eval_user_data);
#if IK_PRINT_TIMING
@ -6353,7 +6354,7 @@ static int llama_decode_internal(
cparams.mtp_op_type, lctx.mtp_step_idx, lctx.mtp_n_heads,
lctx.swa_window_view.w_view,
lctx.swa_window_view.win_off,
llama_ubatch_seq_fingerprint(u_batch, model.arch), gf});
seq_fingerprint, gf});
}
} else {
//printf("Reusing graph with type = %d, n_kv = %d, n_tokens = %d\n", cparams.mtp_op_type, (int)prev->n_kv, (int)prev->n_tokens);