From 6b9de3dbaa21ae95ea80638e5ee836795cc48c93 Mon Sep 17 00:00:00 2001 From: Farmadupe Date: Fri, 5 Jun 2026 16:10:02 +0100 Subject: [PATCH] Fix mrope application across chunk boundaries (Fixes #993 and #1902 -- part 2) (#1918) * (qwen3vl) Correct calculation for injection point of deepstack image embeddings INjection point for deepstack embeddings used Hyperparameter n_embd_inp(), which caused the hidden state to be double accounted for, causing an OOB array access. The correct accessor is n_embd() * Fix m-rope when pipeline parallelism is enabled --- common/common.cpp | 9 +-------- examples/mtmd/mtmd-helper.cpp | 2 +- src/llama.cpp | 22 +++++++++++++++++++++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 7677d614..224f3e0d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -4055,14 +4055,7 @@ static std::pair get_batch_ubatch(const gpt_params & params) { if (params.n_ctx > 0) { n_batch = std::min(n_batch, params.n_ctx); } - if (!params.mmproj.path.empty() && params.mmproj_use_gpu) { - // temporary fix for qwen mtmd (only when mmproj is on GPU) - n_batch = std::max(n_batch, n_ubatch); - n_ubatch = n_batch; - fprintf(stdout, "Adjust batch size for mtmd: u_batch = %d, batch = %d\n", n_ubatch, n_batch); - } else { - n_ubatch = std::min(n_batch, n_ubatch); - } + n_ubatch = std::min(n_batch, n_ubatch); return {n_batch, n_ubatch}; } diff --git a/examples/mtmd/mtmd-helper.cpp b/examples/mtmd/mtmd-helper.cpp index 7a5d0ee5..ab227d47 100644 --- a/examples/mtmd/mtmd-helper.cpp +++ b/examples/mtmd/mtmd-helper.cpp @@ -183,7 +183,7 @@ static int32_t mtmd_helper_decode_image_chunk_impl( } const llama_model * model = llama_get_model(lctx); - int n_mmproj_embd = llama_model_n_embd_inp(model); + int n_mmproj_embd = llama_model_n_embd(model); int n_pos_per_embd = mtmd_decode_use_mrope(ctx) ? 4 : 1; int32_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk); diff --git a/src/llama.cpp b/src/llama.cpp index 73417724..e55d7c20 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -5156,11 +5156,31 @@ static int llama_decode_internal( } } + + // Repack the rope buffer for the ubatch depending on type. + // * mrope: (section-major array of rope fields) [t; n][h; n][w; n][extra; n] + // * others: (flat array ) [t; n] + const uint8_t rope_params_per_token = (hparams.rope_type == LLAMA_ROPE_TYPE_MROPE || + hparams.rope_type == LLAMA_ROPE_TYPE_IMROPE) ? 4 : 1; + llama_pos * u_batch_pos; + if (batch_all.pos && batch_all.embd && rope_params_per_token == 4) { + pos.resize((size_t) n_tokens * rope_params_per_token); + for (uint32_t i = 0; i < n_tokens; ++i) { + pos[0*n_tokens + i] = batch_all.pos[0*n_tokens_all + cur_token + i]; // t + pos[1*n_tokens + i] = batch_all.pos[1*n_tokens_all + cur_token + i]; // h + pos[2*n_tokens + i] = batch_all.pos[2*n_tokens_all + cur_token + i]; // w + pos[3*n_tokens + i] = batch_all.pos[3*n_tokens_all + cur_token + i]; // extra + } + u_batch_pos = pos.data(); + } else { + u_batch_pos = batch_all.pos ? batch_all.pos + cur_token : nullptr; + } + llama_batch u_batch = { /* .n_tokens = */ (int32_t) n_tokens, /* .token = */ batch_all.token ? batch_all.token + cur_token : nullptr, /* .embd = */ batch_all.embd ? batch_all.embd + cur_token*n_embd : nullptr, - /* .pos = */ batch_all.pos ? batch_all.pos + cur_token : nullptr, + /* .pos = */ u_batch_pos, /* .n_seq_id = */ batch_all.n_seq_id ? batch_all.n_seq_id + cur_token : nullptr, /* .seq_id = */ batch_all.seq_id ? batch_all.seq_id + cur_token : nullptr, /* .logits = */ batch_all.logits ? batch_all.logits + cur_token : nullptr,