Adaptive P sampler: update review logic, delete old code comments, put prep stage after logit bias (#1386)
* simpler n_rewind logic, delete old comments * use more consistent names, add updt_w_cur to json schema * align comments * refactor review logic, update struct/variable names * revert cosmetic changes * check enable/disable in llama_prep_adaptive_p_impl() * delete extra whitespaces after statement * show target in debug prints * more concise debug print * delete old comments * update with loop instead of move() * comment out all adaptive p debug prints * more debug prints * move review() variables: common_sampler struct -> common_sampler_review() args * match n_unsent type * fix merge bugs, delete adaptive p references in buffer_and_check_string_ban() * restore accidental erasure * Revert "adaptive p: collect probability before logit bias" This reverts commit 1434878461c49d1a2a9047fc15d5e7b78421fd2a.
This commit is contained in:
dungquixote42
GitHub
parent
a6a1da9a28
commit
be2940f57a
|
|
@ -109,8 +109,6 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, co
|
|||
}
|
||||
}
|
||||
|
||||
result->n_rewind = -1;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -147,12 +145,10 @@ void common_sampler_reset(common_sampler * ctx) {
|
|||
llama_sampler_dry_reset(ctx->smpl);
|
||||
}
|
||||
|
||||
void common_sampler_review(common_sampler * ctx) {
|
||||
const int32_t n_rewind = ctx->n_rewind;
|
||||
|
||||
void common_sampler_review(common_sampler * ctx, const size_t n_unsent, const bool rewind_status) {
|
||||
// add stateful samplers here
|
||||
if (ctx->adapt_p_ctx != nullptr) {
|
||||
llama_review_adaptive_p(ctx->adapt_p_ctx, n_rewind);
|
||||
llama_review_adaptive_p(ctx->adapt_p_ctx, n_unsent, rewind_status);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -425,13 +421,14 @@ static llama_token llama_sampling_sample_impl(
|
|||
id = llama_sample_token_mirostat_v2(ctx_main, &cur_p, mirostat_tau, mirostat_eta, &ctx_sampling->mirostat_mu);
|
||||
} else if (adaptive_target >= 0.0f && ctx_sampling->adapt_p_ctx!=nullptr) {
|
||||
// adaptive p sampling
|
||||
llama_prep_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx);
|
||||
sampler_queue(ctx_main, params, ctx_sampling, cur_p, std::max(1, params.min_keep));
|
||||
id = llama_sample_token_adaptive_p(ctx_main, &cur_p, ctx_sampling->adapt_p_ctx);
|
||||
} else {
|
||||
// temperature sampling
|
||||
size_t min_keep = std::max(1, params.min_keep);
|
||||
|
||||
sampler_queue(ctx_main, params,ctx_sampling, cur_p, min_keep);
|
||||
sampler_queue(ctx_main, params,ctx_sampling, cur_p, min_keep);
|
||||
id = llama_sample_token_with_rng(ctx_main, &cur_p, ctx_sampling->rng);
|
||||
|
||||
}
|
||||
|
|
@ -496,11 +493,6 @@ static llama_token_data_array llama_sampling_prepare_impl(
|
|||
*original_logits = {logits, logits + n_vocab};
|
||||
}
|
||||
|
||||
if ((params.temp > 0) && (params.mirostat == 0) && (params.adaptive_target >= 0) && (ctx_sampling->adapt_p_ctx != nullptr)) {
|
||||
// collect original probability before logit bias is applied
|
||||
llama_prep_adaptive_p(ctx_main, logits, ctx_sampling->adapt_p_ctx);
|
||||
}
|
||||
|
||||
// apply params.logit_bias map
|
||||
for (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {
|
||||
logits[it->first] += it->second;
|
||||
|
|
|
|||
|
|
@ -134,8 +134,6 @@ struct common_sampler {
|
|||
llama_token_data_array cur_p; // current candidates
|
||||
|
||||
std::mt19937 rng;
|
||||
|
||||
int32_t n_rewind; // number of tokens to rewind
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -152,7 +150,7 @@ void common_sampler_reset(common_sampler * ctx);
|
|||
|
||||
// Review stateful samplers
|
||||
// - rewind internal states (maybe)
|
||||
void common_sampler_review(common_sampler * ctx);
|
||||
void common_sampler_review(common_sampler * ctx, const size_t n_unsent, const bool rewind_status);
|
||||
|
||||
// Set the sampler seed
|
||||
void llama_sampling_set_rng_seed(struct common_sampler * ctx, uint32_t seed);
|
||||
|
|
|
|||
|
|
@ -882,6 +882,7 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task)
|
|||
slot.sparams.mirostat_eta = json_value(data, "mirostat_eta", default_sparams.mirostat_eta);
|
||||
slot.sparams.adaptive_target = json_value(data, "adaptive_target", default_sparams.adaptive_target);
|
||||
slot.sparams.adaptive_decay = json_value(data, "adaptive_decay", default_sparams.adaptive_decay);
|
||||
slot.sparams.adaptive_updt_w_cur = json_value(data, "adaptive_updt_w_cur", default_sparams.adaptive_updt_w_cur);
|
||||
slot.sparams.penalize_nl = json_value(data, "penalize_nl", default_sparams.penalize_nl);
|
||||
slot.params.n_keep = json_value(data, "n_keep", slot.params.n_keep);
|
||||
slot.params.n_discard = json_value(data, "n_discard", defaults.n_discard);
|
||||
|
|
@ -1667,6 +1668,7 @@ json server_context::get_formated_generation(const server_slot& slot) const {
|
|||
{"mirostat_eta", slot.sparams.mirostat_eta},
|
||||
{"adaptive_target", slot.sparams.adaptive_target},
|
||||
{"adaptive_decay", slot.sparams.adaptive_decay},
|
||||
{"adaptive_updt_w_cur", slot.sparams.adaptive_updt_w_cur},
|
||||
{"penalize_nl", slot.sparams.penalize_nl},
|
||||
{"stop", slot.params.antiprompt},
|
||||
{"max_tokens", slot.params.n_predict}, // User configured n_predict
|
||||
|
|
@ -3332,7 +3334,7 @@ void server_context::speculative_decoding_accept() {
|
|||
buffer_and_check_string_ban(slot, result);
|
||||
}
|
||||
|
||||
common_sampler_review(slot.ctx_sampling);
|
||||
common_sampler_review(slot.ctx_sampling, slot.token_buffer.size(), slot.rewind_status);
|
||||
}
|
||||
SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int)ids.size() - 1, (int)slot.drafted.size(), slot.n_past);
|
||||
LOG_VERBOSE("speculative decoding result", {
|
||||
|
|
@ -3545,7 +3547,6 @@ void server_context::buffer_and_check_string_ban(server_slot & slot, completion_
|
|||
bool buffer_full = slot.token_buffer.size() >= slot.n_buffer;
|
||||
|
||||
int32_t ban_pos = -1;
|
||||
int32_t n_rewind = 0;
|
||||
bool sent_results = false;
|
||||
|
||||
// Always reset logit bias to base before checking bans
|
||||
|
|
@ -3553,12 +3554,6 @@ void server_context::buffer_and_check_string_ban(server_slot & slot, completion_
|
|||
|
||||
if (slot.ban_phrases.size() > 0 || slot.ban_regex.size() > 0 || slot.ban_regex_ci.size() > 0) {
|
||||
ban_pos = check_ban_phrase(slot);
|
||||
if (ban_pos >= 0 && slot.sparams.adaptive_target >= 0.0f) {
|
||||
int32_t buffer_start_pos = slot.n_past - (int32_t)slot.token_buffer.size() + 1;
|
||||
int32_t n_keep_buffer = ban_pos - buffer_start_pos;
|
||||
if (n_keep_buffer < 0) n_keep_buffer = 0;
|
||||
n_rewind = (int32_t)slot.token_buffer.size() - n_keep_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
bool allow_rewind = true;
|
||||
|
|
@ -3600,17 +3595,11 @@ void server_context::buffer_and_check_string_ban(server_slot & slot, completion_
|
|||
// send 1 token from the front (FIFO)
|
||||
send_token_results(slot.token_buffer, slot, 1);
|
||||
}
|
||||
if (slot.sparams.adaptive_target >= 0.0f) {
|
||||
sent_results = true;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// buffer the result, wait for more tokens to validate string
|
||||
slot.sampled = result.tok;
|
||||
}
|
||||
if (slot.sparams.adaptive_target >= 0.0f) {
|
||||
slot.ctx_sampling->n_rewind = sent_results ? -1 : n_rewind;
|
||||
}
|
||||
}
|
||||
|
||||
void server_context::process_batch_tokens(int32_t & n_batch) {
|
||||
|
|
@ -3761,7 +3750,7 @@ void server_context::process_batch_tokens(int32_t & n_batch) {
|
|||
buffer_and_check_string_ban(slot, result);
|
||||
}
|
||||
|
||||
common_sampler_review(slot.ctx_sampling);
|
||||
common_sampler_review(slot.ctx_sampling, slot.token_buffer.size(), slot.rewind_status);
|
||||
|
||||
slot.i_batch = -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1406,7 +1406,7 @@ LLAMA_API struct llama_grammar* llama_sampler_init_grammar_lazy_patterns(
|
|||
const uint32_t seed);
|
||||
|
||||
void llama_prep_adaptive_p(struct llama_context * ctx,
|
||||
float * logits,
|
||||
llama_token_data_array * candidates,
|
||||
struct llama_sampler_adaptive_p * adapt_p_ctx);
|
||||
|
||||
/// @details Adaptive p sampler described in https://github.com/MrJackSpade/adaptive-p-docs/blob/main/README.md
|
||||
|
|
@ -1414,7 +1414,7 @@ LLAMA_API struct llama_grammar* llama_sampler_init_grammar_lazy_patterns(
|
|||
llama_token_data_array * candidates,
|
||||
struct llama_sampler_adaptive_p * adapt_p_ctx);
|
||||
|
||||
void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind);
|
||||
void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status);
|
||||
|
||||
|
||||
/// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
|
||||
|
|
|
|||
|
|
@ -1046,49 +1046,30 @@ struct llama_sampler_dry* llama_sampler_init_dry_impl(const struct llama_vocab&
|
|||
|
||||
// adaptive p
|
||||
|
||||
void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind) {
|
||||
if ((n_rewind == 0) || (adapt_p_ctx->target < 0.0f)) {
|
||||
return;
|
||||
}
|
||||
// auto & weighted_sum = adapt_p_ctx->weighted_sum;
|
||||
// auto & total_weight = adapt_p_ctx->total_weight;
|
||||
|
||||
const int32_t sz = adapt_p_ctx->history.size();
|
||||
if ((sz <= 0) || (sz <= n_rewind)) {
|
||||
// critically short history. reset to initial state
|
||||
LLAMA_LOG_WARN("%s: sz=%d, n_rewind=%d should not be possible\n", __func__, sz, n_rewind);
|
||||
adapt_p_ctx->history.clear();
|
||||
adapt_p_ctx->history.push_back({
|
||||
adapt_p_ctx->target / adapt_p_ctx->decay, // weighted_sum
|
||||
1.0f / adapt_p_ctx->decay }); // total_weight
|
||||
void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status) {
|
||||
// LLAMA_LOG_DEBUG("%s: n_unsent = %zu, rewind_status = %s\n", __func__, n_unsent, rewind_status ? "true" : "false");
|
||||
if (adapt_p_ctx->target < 0.0f) {
|
||||
// LLAMA_LOG_DEBUG("%s: sampler disabled, target = %f\n", __func__, adapt_p_ctx->target);
|
||||
return;
|
||||
}
|
||||
|
||||
if (n_rewind < 0) {
|
||||
// clear history except most recent
|
||||
adapt_p_ctx->history.front() = adapt_p_ctx->history.back();
|
||||
adapt_p_ctx->history.resize(1);
|
||||
} else {
|
||||
// rewind
|
||||
adapt_p_ctx->history.resize(sz - n_rewind);
|
||||
auto & history = adapt_p_ctx->history;
|
||||
const size_t hsz = history.size();
|
||||
const size_t hsz_next = 1 + n_unsent;
|
||||
// LLAMA_LOG_DEBUG("%s: hsz = %zu, hsz_next = %zu\n", __func__, hsz, hsz_next);
|
||||
if (hsz_next >= hsz >> 1) { return; } // skip small update
|
||||
|
||||
// int32_t sz = weighted_sum.size() - n_rewind;
|
||||
// if (sz > 0) {
|
||||
// weighted_sum.resize(sz);
|
||||
// } else {
|
||||
// LLAMA_LOG_WARN("%s: n_rewind=%d, sz=%d should not be possible\n", __func__, n_rewind, sz);
|
||||
// weighted_sum.clear();
|
||||
// weighted_sum.push_back(adapt_p_ctx->target / adapt_p_ctx->decay); // set to default value
|
||||
// }
|
||||
// sz = total_weight.size() - n_rewind;
|
||||
// if (sz > 0) {
|
||||
// total_weight.resize(sz);
|
||||
// } else {
|
||||
// LLAMA_LOG_WARN("%s: n_rewind=%d, sz=%d should not be possible\n", __func__, n_rewind, sz);
|
||||
// total_weight.clear();
|
||||
// total_weight.push_back(1.0f / adapt_p_ctx->decay); // set to default value
|
||||
// }
|
||||
if (!rewind_status) {
|
||||
// sent results, overwrite old history
|
||||
// LLAMA_LOG_DEBUG("%s: hsz = %zu, hsz_next = %zu\n", __func__, hsz, hsz_next);
|
||||
// LLAMA_LOG_DEBUG("%s: history[hsz-1].first = %f\n", __func__, history[hsz-1].first);
|
||||
const size_t hsz_diff = hsz - hsz_next;
|
||||
for (int j = 0; j < hsz_next; ++j) {
|
||||
history[j] = history[j + hsz_diff];
|
||||
}
|
||||
}
|
||||
history.resize(hsz_next);
|
||||
// LLAMA_LOG_DEBUG("%s: history[hsz_next-1].first = %f\n", __func__, history[hsz_next-1].first);
|
||||
}
|
||||
|
||||
llama_token llama_sample_token_adaptive_p_impl(
|
||||
|
|
@ -1126,8 +1107,6 @@ llama_token llama_sample_token_adaptive_p_impl(
|
|||
ctx->history.push_back({
|
||||
ctx->decay * ctx->history.back().first + update_prob, // weighted_sum
|
||||
ctx->decay * ctx->history.back().second + 1.0f }); // total_weight
|
||||
// ctx->weighted_sum.push_back(ctx->decay * ctx->weighted_sum.back() + update_prob);
|
||||
// ctx->total_weight.push_back(ctx->decay * ctx->total_weight.back() + 1.0f);
|
||||
}
|
||||
|
||||
smpl->t_sample_us += ggml_time_us() - t_start_sample_us;
|
||||
|
|
@ -1139,7 +1118,7 @@ llama_token llama_sample_token_adaptive_p_impl(
|
|||
void llama_sample_adaptive_p_impl(struct llama_sampling * ctx, llama_token_data_array * candidates,
|
||||
struct llama_sampler_adaptive_p * adapt_p_ctx) {
|
||||
if (adapt_p_ctx->target < 0.0f) {
|
||||
// sampler is disabled
|
||||
// LLAMA_LOG_DEBUG("%s: sampler disabled, target = %f\n", __func__, adapt_p_ctx->target);
|
||||
llama_sample_softmax_impl(nullptr, candidates);
|
||||
return;
|
||||
}
|
||||
|
|
@ -1195,20 +1174,25 @@ void llama_sample_adaptive_p_impl(struct llama_sampling * ctx, llama_token_data_
|
|||
|
||||
void llama_prep_adaptive_p_impl(
|
||||
struct llama_sampling * smpl,
|
||||
float * logits,
|
||||
llama_token_data_array * candidates,
|
||||
struct llama_sampler_adaptive_p * adapt_p_ctx) {
|
||||
if (adapt_p_ctx->updt_w_cur) {
|
||||
// update with current probability, original not needed
|
||||
if (adapt_p_ctx->updt_w_cur // update with current probability, original not needed
|
||||
|| (adapt_p_ctx->target < 0.0f)) { // or disabled
|
||||
return;
|
||||
}
|
||||
constexpr float kDelta = 30.0f; //16.6f;
|
||||
auto t_start = ggml_time_us();
|
||||
auto & orig_prob = adapt_p_ctx->orig_prob;
|
||||
|
||||
std::copy(logits, logits + orig_prob.size(), orig_prob.begin());
|
||||
if (candidates->size != orig_prob.size() || candidates->sorted) {
|
||||
LLAMA_LOG_ERROR("%s: this function must be called before any other sampler has been applied\n", __func__);
|
||||
LLAMA_LOG_ERROR("%s: the sampler has been initialized with a vocabulary of %zu, but is being called with %zu candidates\n",
|
||||
__func__, orig_prob.size(), candidates->size);
|
||||
GGML_ABORT("Bad candidates in adaptive_p sampler");
|
||||
}
|
||||
|
||||
float max_logit = -INFINITY;
|
||||
for (int j = 0; j < int(orig_prob.size()); ++j) {
|
||||
for (int j = 0; j < int(candidates->size); ++j) {
|
||||
orig_prob[j] = candidates->data[j].logit;
|
||||
max_logit = std::max(max_logit, orig_prob[j]);
|
||||
}
|
||||
adapt_p_ctx->cum_orig_prob = iqk_exp_with_thresh(orig_prob.size(), orig_prob.data(), max_logit, max_logit - kDelta);
|
||||
|
|
@ -1228,8 +1212,6 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab,
|
|||
/* .decay = */ clamped_decay,
|
||||
/* .updt_w_cur = */ updt_w_cur,
|
||||
/* .rng = */ std::mt19937(seed),
|
||||
// /* .weighted_sum = */ {},
|
||||
// /* .total_weight = */ {},
|
||||
/* .history = */ {},
|
||||
/* .orig_prob = */ {},
|
||||
/* .cum_orig_prob = */ 1.0f,
|
||||
|
|
@ -1237,8 +1219,6 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab,
|
|||
/* .max_xform_logit = */ -INFINITY,
|
||||
/* .cum_probs = */ {},
|
||||
};
|
||||
// result->weighted_sum.push_back(target / (1.0f - clamped_decay));
|
||||
// result->total_weight.push_back(1.0f / (1.0f - clamped_decay));
|
||||
result->history.push_back({
|
||||
target / (1.0f - clamped_decay), // weighted_sum
|
||||
1.0f / (1.0f - clamped_decay) }); // total_weight
|
||||
|
|
|
|||
|
|
@ -70,8 +70,6 @@ struct llama_sampler_adaptive_p {
|
|||
const float decay; // EMA decay; history ≈ 1/(1-decay) tokens (0.0 - 0.99)
|
||||
const bool updt_w_cur; // false=original, true=current
|
||||
std::mt19937 rng; // RNG
|
||||
// std::vector<float> weighted_sum; // [0] = sum(p_n * decay^N)
|
||||
// std::vector<float> total_weight; // [0] = sum(decay^i), converges to 1/(1-decay)
|
||||
std::vector<std::pair<float, float>> history; // <weighted_sum, total_weight>
|
||||
|
||||
// first referenced in prep
|
||||
|
|
@ -94,7 +92,7 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p_impl(int n_vocab,
|
|||
|
||||
void llama_prep_adaptive_p_impl(
|
||||
struct llama_sampling * smpl,
|
||||
float * logits,
|
||||
llama_token_data_array * candidates,
|
||||
struct llama_sampler_adaptive_p * adapt_p_ctx);
|
||||
|
||||
void llama_sample_adaptive_p_impl(
|
||||
|
|
@ -102,7 +100,7 @@ void llama_sample_adaptive_p_impl(
|
|||
llama_token_data_array * candidates,
|
||||
struct llama_sampler_adaptive_p * adapt_p_ctx);
|
||||
|
||||
void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind);
|
||||
void llama_review_adaptive_p_impl(llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status);
|
||||
|
||||
|
||||
void llama_sample_repetition_penalties_impl(
|
||||
|
|
|
|||
|
|
@ -8281,8 +8281,8 @@ void llama_sample_adaptive_p(llama_context * ctx,
|
|||
llama_sample_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx);
|
||||
}
|
||||
|
||||
void llama_prep_adaptive_p(struct llama_context * ctx, float * logits, struct llama_sampler_adaptive_p * adapt_p_ctx) {
|
||||
llama_prep_adaptive_p_impl(&ctx->sampling, logits, adapt_p_ctx);
|
||||
void llama_prep_adaptive_p(struct llama_context * ctx, llama_token_data_array * candidates, struct llama_sampler_adaptive_p * adapt_p_ctx) {
|
||||
llama_prep_adaptive_p_impl(&ctx->sampling, candidates, adapt_p_ctx);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -8388,8 +8388,8 @@ struct llama_sampler_adaptive_p * llama_init_adaptive_p(int n_vocab, const float
|
|||
return llama_init_adaptive_p_impl(n_vocab, target, decay, updt_w_cur, seed);
|
||||
}
|
||||
|
||||
void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const int32_t n_rewind) {
|
||||
llama_review_adaptive_p_impl(adapt_p_ctx, n_rewind);
|
||||
void llama_review_adaptive_p(struct llama_sampler_adaptive_p * adapt_p_ctx, const size_t n_unsent, const bool rewind_status) {
|
||||
llama_review_adaptive_p_impl(adapt_p_ctx, n_unsent, rewind_status);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue