From 1c13288164660dae5644e7495739ba39834f7df9 Mon Sep 17 00:00:00 2001
From: Yap Sok Ann <sokann@gmail.com>
Date: Fri, 24 Apr 2026 15:48:21 +0800
Subject: [PATCH] Fix infinite tool calls loop with `--parallel-tool-calls`
 (#1679)

Previously, the end-of-turn token would be added to prompt cache by
`slot.cache_tokens.push_back(slot.sampled)`, without going through
`llama_decode()` first. As such, the token doesn't exist yet in the
actual KV cache. Then, in the next turn, processing of this EOT token
will be skipped since it is already in the prompt cache.

The expected sequence (with Qwen3.5 chat template):

    </tool_call><|im_end|>
    <|im_start|>user
    <tool_response>

The actual sequence that the model sees:

    </tool_call>
    <|im_start|>user
    <tool_response>

As the conversation goes on, the model starts losing the ability to
generate an EOT token after `</tool_call>`, since it hasn't seen this
pattern in the context. With `--parallel-tool-calls`, the grammar
allows `</tool_call>` to be followed by either another tool call or an
EOT token. So, this eventually leads to the model making infinite tool
calls.

We fix this by deferring all the processing of the EOT token to the
next turn.

Fixes #1661
---
 examples/server/server-context.cpp | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp
index 9a943d21..71a818b3 100644
--- a/examples/server/server-context.cpp
+++ b/examples/server/server-context.cpp
@@ -3755,10 +3755,6 @@ void server_context::speculative_decoding_accept() {
             if (slot.n_buffer == 0 || !params_base.can_ban_phrases) {
                 if (!process_token(result, slot)) {
                     // release slot because of stop condition
-                    slot.i_batch_dft.push_back(batch.n_tokens);
-                    common_batch_add(batch, slot.sampled, slot.cache_tokens.pos_next(), { slot.id }, true);
-                    slot.cache_tokens.push_back(slot.sampled);
-                    slot.n_past++;
                     send_final_response(slot);
                     release_slot_after_final_response(slot);
                     break;
@@ -3816,10 +3812,6 @@ void server_context::send_token_results(completion_token_outputs& results, serve
             if (slot.stopped_limit && !slot.stopped_eos && !slot.stopped_word) {
                 continue;
             }
-            slot.i_batch = batch.n_tokens;
-            common_batch_add(batch, slot.sampled, slot.cache_tokens.pos_next(), { slot.id }, true);
-            slot.cache_tokens.push_back(slot.sampled);
-            slot.n_past++;
             send_final_response(slot);
             release_slot_after_final_response(slot);
             released = true;
@@ -3831,10 +3823,6 @@ void server_context::send_token_results(completion_token_outputs& results, serve
     }
 
     if (!released && slot.stopped_limit && !slot.stopped_eos && !slot.stopped_word) {
-        slot.i_batch = batch.n_tokens;
-        common_batch_add(batch, slot.sampled, slot.cache_tokens.pos_next(), { slot.id }, true);
-        slot.cache_tokens.push_back(slot.sampled);
-        slot.n_past++;
         send_final_response(slot);
         release_slot_after_final_response(slot);
     }