diff --git a/common/sampling.cpp b/common/sampling.cpp index a842d2b8..ca398400 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -109,8 +109,6 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co } } - result->n_rewind = -1; - return result; } @@ -147,12 +145,10 @@ void common_sampler_reset(common_sampler * ctx) { llama_sampler_dry_reset(ctx->smpl); } -void common_sampler_review(common_sampler * ctx) { - const int32_t n_rewind = ctx->n_rewind; - +void common_sampler_review(common_sampler * ctx, const size_t n_unsent, const bool rewind_status) { // add stateful samplers here if (ctx->adapt_p_ctx != nullptr) { - llama_review_adaptive_p(ctx->adapt_p_ctx, n_rewind); + llama_review_adaptive_p(ctx->adapt_p_ctx, n_unsent, rewind_status); } } @@ -425,13 +421,14 @@ static llama_token llama_sampling_sample_impl( id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu); } else if (adaptive_target >= 0.0f && ctx_sampling->adapt_p_ctx!=nullptr) { // adaptive p sampling + llama_prep_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx); sampler_queue(ctx_main, params, ctx_sampling, cur_p, std::max(1, params.min_keep)); id = llama_sample_token_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx); } else { // temperature sampling size_t min_keep = std::max(1, params.min_keep); - sampler_queue(ctx_main, params,ctx_sampling, cur_p, min_keep); + sampler_queue(ctx_main, params,ctx_sampling, cur_p, min_keep); id = llama_sample_token_with_rng(ctx_main, &cur_p, ctx_sampling->rng); } @@ -496,11 +493,6 @@ static llama_token_data_array llama_sampling_prepare_impl( *original_logits = {logits, logits + n_vocab}; } - if ((params.temp > 0) && (params.mirostat == 0) && (params.adaptive_target >= 0) && (ctx_sampling->adapt_p_ctx != nullptr)) { - // collect original probability before logit bias is applied - llama_prep_adaptive_p(ctx_main, logits, ctx_sampling->adapt_p_ctx); - } - // apply params.logit_bias map for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) { logits[it->first] += it->second; diff --git a/common/sampling.h b/common/sampling.h index 66d9e613..7f6a3df7 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -134,8 +134,6 @@ struct common_sampler { llama_token_data_array cur_p; // current candidates std::mt19937 rng; - - int32_t n_rewind; // number of tokens to rewind }; @@ -152,7 +150,7 @@ void common_sampler_reset(common_sampler * ctx); // Review stateful samplers // - rewind internal states (maybe) -void common_sampler_review(common_sampler * ctx); +void common_sampler_review(common_sampler * ctx, const size_t n_unsent, const bool rewind_status); // Set the sampler seed void llama_sampling_set_rng_seed(struct common_sampler * ctx, uint32_t seed); diff --git a/examples/server/server-context.cpp b/examples/server/server-context.cpp index 28c0cbc3..627a3f99 100644 --- a/examples/server/server-context.cpp +++ b/examples/server/server-context.cpp @@ -882,6 +882,7 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task) slot.sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta); slot.sparams.adaptive_target = json_value(data, "adaptive_target", default_sparams.adaptive_target); slot.sparams.adaptive_decay = json_value(data, "adaptive_decay", default_sparams.adaptive_decay); + slot.sparams.adaptive_updt_w_cur = json_value(data, "adaptive_updt_w_cur", default_sparams.adaptive_updt_w_cur); slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl); slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep); slot.params.n_discard = json_value(data, "n_discard", defaults.n_discard); @@ -1667,6 +1668,7 @@ json server_context::get_formated_generation(const server_slot& slot) const { {"mirostat_eta", slot.sparams.mirostat_eta}, {"adaptive_target", slot.sparams.adaptive_target}, {"adaptive_decay", slot.sparams.adaptive_decay}, + {"adaptive_updt_w_cur", slot.sparams.adaptive_updt_w_cur}, {"penalize_nl", slot.sparams.penalize_nl}, {"stop", slot.params.antiprompt}, {"max_tokens", slot.params.n_predict}, // User configured n_predict @@ -3332,7 +3334,7 @@ void server_context::speculative_decoding_accept() { buffer_and_check_string_ban(slot, result); } - common_sampler_review(slot.ctx_sampling); + common_sampler_review(slot.ctx_sampling, slot.token_buffer.size(), slot.rewind_status); } SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int)ids.size() - 1, (int)slot.drafted.size(), slot.n_past); LOG_VERBOSE("speculative decoding result", { @@ -3545,7 +3547,6 @@ void server_context::buffer_and_check_string_ban(server_slot & slot, completion_ bool buffer_full = slot.token_buffer.size() >= slot.n_buffer; int32_t ban_pos = -1; - int32_t n_rewind = 0; bool sent_results = false; // Always reset logit bias to base before checking bans @@ -3553,12 +3554,6 @@ void server_context::buffer_and_check_string_ban(server_slot & slot, completion_ if (slot.ban_phrases.size() > 0 || slot.ban_regex.size() > 0 || slot.ban_regex_ci.size() > 0) { ban_pos = check_ban_phrase(slot); - if (ban_pos >= 0 && slot.sparams.adaptive_target >= 0.0f) { - int32_t buffer_start_pos = slot.n_past - (int32_t)slot.token_buffer.size() + 1; - int32_t n_keep_buffer = ban_pos - buffer_start_pos; - if (n_keep_buffer < 0) n_keep_buffer = 0; - n_rewind = (int32_t)slot.token_buffer.size() - n_keep_buffer; - } } bool allow_rewind = true; @@ -3600,17 +3595,11 @@ void server_context::buffer_and_check_string_ban(server_slot & slot, completion_ // send 1 token from the front (FIFO) send_token_results(slot.token_buffer, slot, 1); } - if (slot.sparams.adaptive_target >= 0.0f) { - sent_results = true; - } } else { // buffer the result, wait for more tokens to validate string slot.sampled = result.tok; } - if (slot.sparams.adaptive_target >= 0.0f) { - slot.ctx_sampling->n_rewind = sent_results ? -1 : n_rewind; -} } void server_context::process_batch_tokens(int32_t & n_batch) { @@ -3761,7 +3750,7 @@ void server_context::process_batch_tokens(int32_t & n_batch) { buffer_and_check_string_ban(slot, result); } - common_sampler_review(slot.ctx_sampling); + common_sampler_review(slot.ctx_sampling, slot.token_buffer.size(), slot.rewind_status); slot.i_batch = -1; } diff --git a/include/llama.h b/include/llama.h index 34e25402..f1729662 100644 --- a/include/llama.h +++ b/include/llama.h @@ -1406,7 +1406,7 @@ LLAMA_API struct llama_grammar* llama_sampler_init_grammar_lazy_patterns( const uint32_t seed); void llama_prep_adaptive_p(struct llama_context * ctx, - float * logits, + llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); /// @details Adaptive p sampler described in https://github.com/MrJackSpade/adaptive-p-docs/blob/main/README.md @@ -1414,7 +1414,7 @@ LLAMA_API struct llama_grammar* llama_sampler_init_grammar_lazy_patterns( llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); - void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind); + void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status); /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. diff --git a/src/llama-sampling.cpp b/src/llama-sampling.cpp index 1663312d..a503fcc6 100644 --- a/src/llama-sampling.cpp +++ b/src/llama-sampling.cpp @@ -1046,49 +1046,30 @@ struct llama_sampler_dry* llama_sampler_init_dry_impl(const struct llama_vocab& // adaptive p -void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind) { - if ((n_rewind == 0) || (adapt_p_ctx->target < 0.0f)) { - return; - } - // auto & weighted_sum = adapt_p_ctx->weighted_sum; - // auto & total_weight = adapt_p_ctx->total_weight; - - const int32_t sz = adapt_p_ctx->history.size(); - if ((sz <= 0) || (sz <= n_rewind)) { - // critically short history. reset to initial state - LLAMA_LOG_WARN("%s: sz=%d, n_rewind=%d should not be possible\n", __func__, sz, n_rewind); - adapt_p_ctx->history.clear(); - adapt_p_ctx->history.push_back({ - adapt_p_ctx->target / adapt_p_ctx->decay, // weighted_sum - 1.0f / adapt_p_ctx->decay }); // total_weight +void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status) { + // LLAMA_LOG_DEBUG("%s: n_unsent = %zu, rewind_status = %s\n", __func__, n_unsent, rewind_status ? "true" : "false"); + if (adapt_p_ctx->target < 0.0f) { + // LLAMA_LOG_DEBUG("%s: sampler disabled, target = %f\n", __func__, adapt_p_ctx->target); return; } - if (n_rewind < 0) { - // clear history except most recent - adapt_p_ctx->history.front() = adapt_p_ctx->history.back(); - adapt_p_ctx->history.resize(1); - } else { - // rewind - adapt_p_ctx->history.resize(sz - n_rewind); + auto & history = adapt_p_ctx->history; + const size_t hsz = history.size(); + const size_t hsz_next = 1 + n_unsent; + // LLAMA_LOG_DEBUG("%s: hsz = %zu, hsz_next = %zu\n", __func__, hsz, hsz_next); + if (hsz_next >= hsz >> 1) { return; } // skip small update - // int32_t sz = weighted_sum.size() - n_rewind; - // if (sz > 0) { - // weighted_sum.resize(sz); - // } else { - // LLAMA_LOG_WARN("%s: n_rewind=%d, sz=%d should not be possible\n", __func__, n_rewind, sz); - // weighted_sum.clear(); - // weighted_sum.push_back(adapt_p_ctx->target / adapt_p_ctx->decay); // set to default value - // } - // sz = total_weight.size() - n_rewind; - // if (sz > 0) { - // total_weight.resize(sz); - // } else { - // LLAMA_LOG_WARN("%s: n_rewind=%d, sz=%d should not be possible\n", __func__, n_rewind, sz); - // total_weight.clear(); - // total_weight.push_back(1.0f / adapt_p_ctx->decay); // set to default value - // } + if (!rewind_status) { + // sent results, overwrite old history + // LLAMA_LOG_DEBUG("%s: hsz = %zu, hsz_next = %zu\n", __func__, hsz, hsz_next); + // LLAMA_LOG_DEBUG("%s: history[hsz-1].first = %f\n", __func__, history[hsz-1].first); + const size_t hsz_diff = hsz - hsz_next; + for (int j = 0; j < hsz_next; ++j) { + history[j] = history[j + hsz_diff]; + } } + history.resize(hsz_next); + // LLAMA_LOG_DEBUG("%s: history[hsz_next-1].first = %f\n", __func__, history[hsz_next-1].first); } llama_token llama_sample_token_adaptive_p_impl( @@ -1126,8 +1107,6 @@ llama_token llama_sample_token_adaptive_p_impl( ctx->history.push_back({ ctx->decay * ctx->history.back().first + update_prob, // weighted_sum ctx->decay * ctx->history.back().second + 1.0f }); // total_weight - // ctx->weighted_sum.push_back(ctx->decay * ctx->weighted_sum.back() + update_prob); - // ctx->total_weight.push_back(ctx->decay * ctx->total_weight.back() + 1.0f); } smpl->t_sample_us += ggml_time_us() - t_start_sample_us; @@ -1139,7 +1118,7 @@ llama_token llama_sample_token_adaptive_p_impl( void llama_sample_adaptive_p_impl(struct llama_sampling * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { if (adapt_p_ctx->target < 0.0f) { - // sampler is disabled + // LLAMA_LOG_DEBUG("%s: sampler disabled, target = %f\n", __func__, adapt_p_ctx->target); llama_sample_softmax_impl(nullptr, candidates); return; } @@ -1195,20 +1174,25 @@ void llama_sample_adaptive_p_impl(struct llama_sampling * ctx, llama_token_data_ void llama_prep_adaptive_p_impl( struct llama_sampling * smpl, - float * logits, + llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { - if (adapt_p_ctx->updt_w_cur) { - // update with current probability, original not needed + if (adapt_p_ctx->updt_w_cur // update with current probability, original not needed + || (adapt_p_ctx->target < 0.0f)) { // or disabled return; } constexpr float kDelta = 30.0f; //16.6f; auto t_start = ggml_time_us(); auto & orig_prob = adapt_p_ctx->orig_prob; - - std::copy(logits, logits + orig_prob.size(), orig_prob.begin()); + if (candidates->size != orig_prob.size() || candidates->sorted) { + LLAMA_LOG_ERROR("%s: this function must be called before any other sampler has been applied\n", __func__); + LLAMA_LOG_ERROR("%s: the sampler has been initialized with a vocabulary of %zu, but is being called with %zu candidates\n", + __func__, orig_prob.size(), candidates->size); + GGML_ABORT("Bad candidates in adaptive_p sampler"); + } float max_logit = -INFINITY; - for (int j = 0; j < int(orig_prob.size()); ++j) { + for (int j = 0; j < int(candidates->size); ++j) { + orig_prob[j] = candidates->data[j].logit; max_logit = std::max(max_logit, orig_prob[j]); } adapt_p_ctx->cum_orig_prob = iqk_exp_with_thresh(orig_prob.size(), orig_prob.data(), max_logit, max_logit - kDelta); @@ -1228,8 +1212,6 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab, /* .decay = */ clamped_decay, /* .updt_w_cur = */ updt_w_cur, /* .rng = */ std::mt19937(seed), - // /* .weighted_sum = */ {}, - // /* .total_weight = */ {}, /* .history = */ {}, /* .orig_prob = */ {}, /* .cum_orig_prob = */ 1.0f, @@ -1237,8 +1219,6 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab, /* .max_xform_logit = */ -INFINITY, /* .cum_probs = */ {}, }; - // result->weighted_sum.push_back(target / (1.0f - clamped_decay)); - // result->total_weight.push_back(1.0f / (1.0f - clamped_decay)); result->history.push_back({ target / (1.0f - clamped_decay), // weighted_sum 1.0f / (1.0f - clamped_decay) }); // total_weight diff --git a/src/llama-sampling.h b/src/llama-sampling.h index 6127a50f..80d5a4f2 100644 --- a/src/llama-sampling.h +++ b/src/llama-sampling.h @@ -70,8 +70,6 @@ struct llama_sampler_adaptive_p { const float decay; // EMA decay; history ≈ 1/(1-decay) tokens (0.0 - 0.99) const bool updt_w_cur; // false=original, true=current std::mt19937 rng; // RNG - // std::vector weighted_sum; // [0] = sum(p_n * decay^N) - // std::vector total_weight; // [0] = sum(decay^i), converges to 1/(1-decay) std::vector> history; // // first referenced in prep @@ -94,7 +92,7 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab, void llama_prep_adaptive_p_impl( struct llama_sampling * smpl, - float * logits, + llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); void llama_sample_adaptive_p_impl( @@ -102,7 +100,7 @@ void llama_sample_adaptive_p_impl( llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx); -void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind); +void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status); void llama_sample_repetition_penalties_impl( diff --git a/src/llama.cpp b/src/llama.cpp index 6792145f..1090bee0 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -8281,8 +8281,8 @@ void llama_sample_adaptive_p(llama_context * ctx, llama_sample_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx); } -void llama_prep_adaptive_p(struct llama_context * ctx, float * logits, struct llama_sampler_adaptive_p * adapt_p_ctx) { - llama_prep_adaptive_p_impl(&ctx->sampling, logits, adapt_p_ctx); +void llama_prep_adaptive_p(struct llama_context * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) { + llama_prep_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx); } @@ -8388,8 +8388,8 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p(int n_vocab, const float return llama_init_adaptive_p_impl(n_vocab, target, decay, updt_w_cur, seed); } -void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind) { - llama_review_adaptive_p_impl(adapt_p_ctx, n_rewind); +void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status) { + llama_review_adaptive_p_impl(adapt_p_ctx, n_unsent, rewind_status); }