nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server: charge the generation budget per accepted token (#2358)

This commit is contained in:
Joel Farthing 2026-08-26 01:00:47 -05:00 committed by GitHub
parent 08b500b958
commit b166e2696e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/server/server-context.cpp
View File

@ -4286,7 +4286,6 @@ void server_context::speculative_decoding_accept() {
slot.drafted.clear();
slot.n_past += ids.size();
slot.n_decoded += ids.size();
const int64_t t_current = ggml_time_us();
slot.t_token_generation = std::max<int64_t>(1, t_current - slot.t_start_generation) / 1e3;
@ -4337,6 +4336,8 @@ void server_context::speculative_decoding_accept() {
for (size_t i = 0; i < ids.size(); ++i) {
completion_token_output result;
slot.n_decoded += 1;
result.tok = ids[i];
result.text_to_send = common_token_to_piece(ctx, result.tok, accept_special_token(slot, result.tok));
result.prob = 1.0f; // set later