diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 63711e83..9a943d21 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -3179,11 +3179,14 @@ void server_context::create_checkpoint_at_interval(server_slot & slot, const gp void server_context::apply_checkpoint(server_slot & slot) { llama_pos pos_next = slot.cache_tokens.pos_next(slot.n_past); + const bool has_recurrent = llama_model_has_recurrent(llama_get_model(slot.ctx)); + // For hybrid/recurrent models, pos_min semantics don't apply: the recurrent state is a single + // snapshot, not a per-token window. Use pos_max against n_past to match whole-prefix checkpoints. const auto pos_min_thold = std::max(0, pos_next - 1); if (slot.n_past > 0 && slot.n_past < slot.cache_tokens.n_tokens()) { int32_t pos_min = llama_kv_cache_seq_pos_min(slot.ctx, slot.id); - if (pos_min > pos_min_thold) { + if (has_recurrent || pos_min > pos_min_thold) { SLT_WRN(slot, "n_past = %d, slot.prompt.tokens.size() = %d, seq_id = %d, pos_min = %d\n", slot.n_past, (int)slot.cache_tokens.size(), slot.id, pos_min); // search for a context checkpoint @@ -3192,6 +3195,11 @@ void server_context::apply_checkpoint(server_slot & slot) { slot.server_cached_prompt.checkpoints.rend(), [&](const auto & cur) { // guarantee that a checkpoint will result in at least one token being processed [TAG_PROMPT_LOGITS] + if (has_recurrent) { + // recurrent/hybrid: only whole-prefix checkpoints are valid; pick the latest one + // that covers no more than the current n_past and still leaves tokens to decode. + return cur.pos_max <= slot.n_past && cur.pos_max < pos_next; + } return cur.pos_min < pos_min_thold; } ); @@ -3218,19 +3226,30 @@ void server_context::apply_checkpoint(server_slot & slot) { } if (do_reset) { - SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see %s)\n", - "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055"); - slot.n_past = 0; - slot.n_past_prompt = 0; + if (has_recurrent) { + // Hybrid/recurrent: do NOT zero n_past. The prompt prefix is already in cache_tokens + // and update_slots() reprocesses from slot.n_past_prompt; dropping to 0 forces a full + // recompute on every turn and — combined with cached state — trips llama_decode ret=-3. + SLT_WRN(slot, "no usable hybrid/recurrent checkpoint; preserving slot state (n_past = %d, n_past_prompt = %d)\n", + (int)slot.n_past, (int)slot.n_past_prompt); + } else { + SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA, see %s)\n", + "https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055"); + slot.n_past = 0; + slot.n_past_prompt = 0; + } } } } { - // erase any checkpoints with pos_min > pos_min_thold + // erase checkpoints that are no longer consistent with the current decode position. + // Transformer: anything with pos_min beyond the threshold is stale. + // Recurrent/hybrid: anything with pos_max past pos_next refers to future tokens we've rewound past. for (auto it = slot.server_cached_prompt.checkpoints.begin(); it != slot.server_cached_prompt.checkpoints.end();) { const auto & cur = *it; - if (cur.pos_min > pos_min_thold) { + const bool stale = has_recurrent ? (cur.pos_max > pos_next) : (cur.pos_min > pos_min_thold); + if (stale) { SLT_WRN(slot, "erased invalidated context checkpoint (pos_min = %d, pos_max = %d, size = %.3f MiB)\n", cur.pos_min, cur.pos_max, (float)cur.data.size() / 1024 / 1024); it = slot.server_cached_prompt.checkpoints.erase(it); } else { diff --git a/src/llama.cpp b/src/llama.cpp index b9e62e0d..3c3f1ef4 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -7918,6 +7918,11 @@ int32_t llama_encode( int32_t llama_decode( struct llama_context * ctx, struct llama_batch batch) { + // Clear any leftover stop signal from a previous (already-returned) decode. llama_decode_stop() + // is intended to interrupt the decode that is currently in flight; without this reset, a stop + // that arrived after the interrupted call returned would bleed into the next decode and cause + // an immediate ret=-3, which servers interpret as a fatal decode failure. + stop_internal_decode = false; const int ret = llama_decode_internal(*ctx, batch); if (ret < 0) { LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret);