From b166e2696e81f31aa2c7bc75a72a3f2cefc15b4e Mon Sep 17 00:00:00 2001 From: Joel Farthing Date: Wed, 26 Aug 2026 01:00:47 -0500 Subject: [PATCH] server: charge the generation budget per accepted token (#2358) --- examples/server/server-context.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index b4fcbee7..58855240 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -4286,7 +4286,6 @@ void server_context::speculative_decoding_accept() { slot.drafted.clear(); slot.n_past += ids.size(); - slot.n_decoded += ids.size(); const int64_t t_current = ggml_time_us(); slot.t_token_generation = std::max(1, t_current - slot.t_start_generation) / 1e3; @@ -4337,6 +4336,8 @@ void server_context::speculative_decoding_accept() { for (size_t i = 0; i < ids.size(); ++i) { completion_token_output result; + slot.n_decoded += 1; + result.tok = ids[i]; result.text_to_send = common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok)); result.prob = 1.0f; // set later