nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix mrope application across chunk boundaries (Fixes #993 and #1902 -- part 2) (#1918) 

* (qwen3vl) Correct calculation for injection point of deepstack image embeddings

INjection point for deepstack embeddings used Hyperparameter n_embd_inp(), which caused the hidden state to be double accounted for, causing an OOB array access. The correct accessor is n_embd()

* Fix m-rope when pipeline parallelism is enabled
This commit is contained in:
Farmadupe 2026-06-05 16:10:02 +01:00 committed by GitHub
parent 1b53a58bf9
commit 6b9de3dbaa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 23 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
common/common.cpp
View File

@ -4055,14 +4055,7 @@ static std::pair<int, int> get_batch_ubatch(const gpt_params & params) {
if (params.n_ctx > 0) {
n_batch = std::min(n_batch, params.n_ctx);
}
if (!params.mmproj.path.empty() && params.mmproj_use_gpu) {
// temporary fix for qwen mtmd (only when mmproj is on GPU)
n_batch = std::max(n_batch, n_ubatch);
n_ubatch = n_batch;
fprintf(stdout, "Adjust batch size for mtmd: u_batch = %d, batch = %d\n", n_ubatch, n_batch);
} else {
n_ubatch = std::min(n_batch, n_ubatch);
}
n_ubatch = std::min(n_batch, n_ubatch);
return {n_batch, n_ubatch};
}

2
examples/mtmd/mtmd-helper.cpp
View File

@ -183,7 +183,7 @@ static int32_t mtmd_helper_decode_image_chunk_impl(
}
const llama_model * model = llama_get_model(lctx);
int n_mmproj_embd = llama_model_n_embd_inp(model);
int n_mmproj_embd = llama_model_n_embd(model);
int n_pos_per_embd = mtmd_decode_use_mrope(ctx) ? 4 : 1;
int32_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk);

22
src/llama.cpp
View File

@ -5156,11 +5156,31 @@ static int llama_decode_internal(
}
}
// Repack the rope buffer for the ubatch depending on type.
// * mrope: (section-major array of rope fields) [t; n][h; n][w; n][extra; n]
// * others: (flat array ) [t; n]
const uint8_t rope_params_per_token = (hparams.rope_type == LLAMA_ROPE_TYPE_MROPE ||
hparams.rope_type == LLAMA_ROPE_TYPE_IMROPE) ? 4 : 1;
llama_pos * u_batch_pos;
if (batch_all.pos && batch_all.embd && rope_params_per_token == 4) {
pos.resize((size_t) n_tokens * rope_params_per_token);
for (uint32_t i = 0; i < n_tokens; ++i) {
pos[0*n_tokens + i] = batch_all.pos[0*n_tokens_all + cur_token + i]; // t
pos[1*n_tokens + i] = batch_all.pos[1*n_tokens_all + cur_token + i]; // h
pos[2*n_tokens + i] = batch_all.pos[2*n_tokens_all + cur_token + i]; // w
pos[3*n_tokens + i] = batch_all.pos[3*n_tokens_all + cur_token + i]; // extra
}
u_batch_pos = pos.data();
} else {
u_batch_pos = batch_all.pos ? batch_all.pos + cur_token : nullptr;
}
llama_batch u_batch = {
/* .n_tokens = */ (int32_t) n_tokens,
/* .token = */ batch_all.token ? batch_all.token + cur_token : nullptr,
/* .embd = */ batch_all.embd ? batch_all.embd + cur_token*n_embd : nullptr,
/* .pos = */ batch_all.pos ? batch_all.pos + cur_token : nullptr,
/* .pos = */ u_batch_pos,
/* .n_seq_id = */ batch_all.n_seq_id ? batch_all.n_seq_id + cur_token : nullptr,
/* .seq_id = */ batch_all.seq_id ? batch_all.seq_id + cur_token : nullptr,
/* .logits = */ batch_all.logits ? batch_all.logits + cur_token : nullptr,