nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Load standalone Qwen3.5 MTP GGUFs passed with -md (#2328) 

* Load standalone Qwen3.5 MTP GGUFs passed with -md

A predictor-only MTP GGUF reports the full block count (n_main +
nextn_predict_layers) but only ships the NextN block, so loading one
with -md failed:

  check_tensor_dims: tensor 'blk.0.attn_norm.weight' not found

create_qwen35_tensors() and create_qwen35moe_tensors() create every
main block as required. Detect the predictor-only case the same way
create_step35_tensors() does and mark the absent blocks
TENSOR_SKIP|TENSOR_NOT_REQUIRED.

Qwen3.5 also has to use the common MTP package contract, otherwise the
predictor-only GGUF is never classified as a companion, and the target
is not classified TARGET_ONLY - which is what makes it export the
hidden states the companion consumes.

The remaining two hunks cover cases the above newly reaches: a
predictor-only GGUF passed as -m now loads far enough to abort in the
graph builder, and its empty main blocks reach split_recurrent_tensors()
under -sm graph.

* Qwen3.5 MTP: require q_proj in predictor-only GGUFs, check companion arch

Review follow-up.

A dense NextN block loads q_proj as optional because it can be shared
with the last main block. A predictor-only GGUF has no main blocks, so
one built that way loaded with wq == nullptr and then hung. Require the
tensor in that case so the load fails naming it. eh_proj, attn_q and the
MLP are all optional on that block, so the tail probe stays on enorm,
which is required - the comment there said only eh_proj.

Adding Qwen3.5 to the common MTP package contract also made
common_speculative_has_recognized_mtp_companion() accept any GGUF
classified COMPANION, with no architecture check of the kind the Step
and DeepSeek branches have. Add it, plus the predictor count. Dense and
MoE are separate architectures, so the comparison is on the arch itself.
This commit is contained in:
Thireus ☠ 2026-08-24 09:03:27 +01:00 committed by GitHub
parent 66b2f50ce3
commit 477852c1c9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 58 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
common/common.cpp
View File

@ -4072,6 +4072,14 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) {
return iparams;
}
// a predictor-only MTP GGUF has no main blocks, so it cannot be the target model
if (llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_COMPANION) {
fprintf(stderr, "%s: error: '%s' is an MTP companion, pass it with -md instead\n",
__func__, params.model.c_str());
llama_free_model(model);
return iparams;
}
auto cparams = common_context_params_to_llama(params);
llama_context * lctx = llama_init_from_model(model, cparams);

17
common/speculative.cpp
View File

@ -145,7 +145,8 @@ static bool common_speculative_are_compatible(
}
static bool common_speculative_target_has_appended_mtp_contract(const llama_model * model) {
return llama_model_is_step35(model) || llama_model_is_deepseek4(model);
return llama_model_is_step35(model) || llama_model_is_deepseek4(model) ||
llama_model_is_qwen35_family(model);
}
static bool common_speculative_has_recognized_mtp_companion(
@ -2178,6 +2179,20 @@ bool common_speculative_finalize_startup(
__func__, n_heads);
return false;
}
} else if (llama_model_is_qwen35_family(model)) {
// dense and MoE are separate architectures, so compare the arch itself
if (std::strcmp(llama_model_arch_string(model), llama_model_arch_string(companion)) != 0) {
LOG_ERR("%s: Qwen3.5 MTP requires a companion of the same architecture, target is %s and companion is %s\n",
__func__, llama_model_arch_string(model), llama_model_arch_string(companion));
return false;
}
const int32_t n_heads = llama_model_n_nextn_layer(companion);
if (n_heads != 1) {
LOG_ERR("%s: Qwen3.5 MTP companion requires exactly one predictor layer, got %d\n",
__func__, n_heads);
return false;
}
}
if (common_speculative_has_recognized_mtp_companion(model, companion)) {

2
include/llama.h
View File

@ -713,6 +713,8 @@ extern "C" {
LLAMA_API bool llama_model_is_step35(const struct llama_model * model);
LLAMA_API bool llama_model_is_qwen35_family(const struct llama_model * model);
LLAMA_API bool llama_is_gemma4_mtp_file(const char * path);
LLAMA_API bool llama_model_is_split_mode_graph(const struct llama_model * model);

24
src/llama-load-tensors.cpp
View File

@ -1704,6 +1704,12 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) {
const int64_t value_dim = head_v_dim * n_v_heads;
const int64_t conv_dim = key_dim * 2 + value_dim;
// A predictor-only MTP GGUF reports the full block count but only ships the NextN block, skip the rest
const bool mtp_only = hparams.nextn_predict_layers > 0 &&
ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr;
const int trunk_flags = mtp_only
? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0;
for (int i = 0; i < n_layer; ++i) {
const bool is_mtp_layer = hparams.nextn_predict_layers > 0 &&
static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers;
@ -1712,7 +1718,7 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) {
auto & layer = model.layers[i];
int flags = 0;
int flags = is_mtp_layer ? 0 : trunk_flags;
if (!model.mtp && is_mtp_layer) {
flags |= llama_model_loader::TENSOR_SKIP;
}
@ -1813,6 +1819,12 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {
const int64_t value_dim = head_v_dim * n_v_heads;
const int64_t conv_dim = key_dim * 2 + value_dim;
// A predictor-only MTP GGUF reports the full block count but only ships the NextN block, skip the rest
const bool mtp_only = hparams.nextn_predict_layers > 0 &&
ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr;
const int trunk_flags = mtp_only
? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0;
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
@ -1821,7 +1833,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {
ggml_context * ctx_split = ctx_for_layer_split(i);
int flags = 0;
int flags = is_mtp_layer ? 0 : trunk_flags;
// Skip loading MTP layers if the feature is disabled
if (!model.mtp) {
if (is_mtp_layer) {
@ -1829,6 +1841,8 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {
}
}
const int mtp_opt = is_mtp_layer ? llama_model_loader::TENSOR_NOT_REQUIRED : 0;
// q_proj may be shared with the last main block, which a predictor-only GGUF does not have
const int mtp_opt_q = mtp_only ? 0 : mtp_opt;
layer.attn_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags);
layer.attn_post_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, flags);
@ -1836,7 +1850,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) {
if (!hparams.is_recurrent(i)) {
// Attention layers (MTP layer is always standard attention)
layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, flags | mtp_opt);
layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, flags | mtp_opt_q);
layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, flags);
layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, flags);
layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags);
@ -5395,6 +5409,10 @@ bool create_tensors_helper::create_tensors() {
for ([[maybe_unused]] auto mem : mem_used) LLAMA_LOG_DEBUG(" %g", mem/1024./1024.);
LLAMA_LOG_DEBUG("\n");
auto & layer = model.layers[il];
// a predictor-only MTP GGUF has no tensors at all in its main blocks
if (!layer.attn_norm && !layer.wq && !layer.wqkv && !layer.ssm_in && !layer.wo) {
continue;
}
auto ctx_split = ctx_for_layer_split(il);
if (layer.attn_norm) {
prepare_split_tensors(-1, ctx_split, layer.attn_norm, layer.split_attn_norm, mirror, mem_used);

13
src/llama-model.cpp
View File

@ -2319,6 +2319,10 @@ bool llama_model_is_step35(const llama_model * model) {
return model && model->arch == LLM_ARCH_STEP35;
}
bool llama_model_is_qwen35_family(const llama_model * model) {
return model && (model->arch == LLM_ARCH_QWEN35 || model->arch == LLM_ARCH_QWEN35MOE);
}
enum llama_mtp_package llama_model_mtp_package(const llama_model * model) {
if (!model) {
return LLAMA_MTP_PACKAGE_INVALID;
@ -2331,13 +2335,15 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) {
const size_t n_nextn = model->hparams.nextn_predict_layers;
const bool has_common_package_contract =
llama_model_is_step35(model) || llama_model_is_deepseek4(model) ||
llama_model_is_qwen35_family(model) ||
llama_model_is_gemma4_mtp_assistant(model);
if (!has_common_package_contract) {
return n_nextn > 0 ? LLAMA_MTP_PACKAGE_EMBEDDED : LLAMA_MTP_PACKAGE_NONE;
}
if (n_nextn == 0) {
if (llama_model_is_step35(model) || llama_model_is_deepseek4(model)) {
if (llama_model_is_step35(model) || llama_model_is_deepseek4(model) ||
llama_model_is_qwen35_family(model)) {
for (const auto & layer : model->layers) {
if (layer.attn_norm != nullptr) {
return LLAMA_MTP_PACKAGE_TARGET_ONLY;
@ -2353,7 +2359,10 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) {
}
const size_t first = n_layers - n_nextn;
const bool has_tail = model->layers[first].nextn.eh_proj != nullptr;
// A Qwen3.5 NextN block loads eh_proj, attn_q and the MLP as optional, so none of them
// marks a predictor tail on its own; enorm is required and is present in all of them.
const bool has_tail = model->layers[first].nextn.eh_proj != nullptr ||
(llama_model_is_qwen35_family(model) && model->layers[first].nextn.enorm != nullptr);
bool has_trunk = false;
for (size_t il = 0; il < first; ++il) {