diff --git a/common/common.cpp b/common/common.cpp index f241371f..153927bf 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -4072,6 +4072,14 @@ struct llama_init_result llama_init_from_gpt_params(gpt_params & params) { return iparams; } + // a predictor-only MTP GGUF has no main blocks, so it cannot be the target model + if (llama_model_mtp_package(model) == LLAMA_MTP_PACKAGE_COMPANION) { + fprintf(stderr, "%s: error: '%s' is an MTP companion, pass it with -md instead\n", + __func__, params.model.c_str()); + llama_free_model(model); + return iparams; + } + auto cparams = common_context_params_to_llama(params); llama_context * lctx = llama_init_from_model(model, cparams); diff --git a/common/speculative.cpp b/common/speculative.cpp index c81a70e8..bf777391 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -145,7 +145,8 @@ static bool common_speculative_are_compatible( } static bool common_speculative_target_has_appended_mtp_contract(const llama_model * model) { - return llama_model_is_step35(model) || llama_model_is_deepseek4(model); + return llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model); } static bool common_speculative_has_recognized_mtp_companion( @@ -2178,6 +2179,20 @@ bool common_speculative_finalize_startup( __func__, n_heads); return false; } + } else if (llama_model_is_qwen35_family(model)) { + // dense and MoE are separate architectures, so compare the arch itself + if (std::strcmp(llama_model_arch_string(model), llama_model_arch_string(companion)) != 0) { + LOG_ERR("%s: Qwen3.5 MTP requires a companion of the same architecture, target is %s and companion is %s\n", + __func__, llama_model_arch_string(model), llama_model_arch_string(companion)); + return false; + } + + const int32_t n_heads = llama_model_n_nextn_layer(companion); + if (n_heads != 1) { + LOG_ERR("%s: Qwen3.5 MTP companion requires exactly one predictor layer, got %d\n", + __func__, n_heads); + return false; + } } if (common_speculative_has_recognized_mtp_companion(model, companion)) { diff --git a/include/llama.h b/include/llama.h index 177f1a71..6ccb146a 100644 --- a/include/llama.h +++ b/include/llama.h @@ -713,6 +713,8 @@ extern "C" { LLAMA_API bool llama_model_is_step35(const struct llama_model * model); + LLAMA_API bool llama_model_is_qwen35_family(const struct llama_model * model); + LLAMA_API bool llama_is_gemma4_mtp_file(const char * path); LLAMA_API bool llama_model_is_split_mode_graph(const struct llama_model * model); diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index 47300df0..c1e2f3b3 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -1704,6 +1704,12 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; + // A predictor-only MTP GGUF reports the full block count but only ships the NextN block, skip the rest + const bool mtp_only = hparams.nextn_predict_layers > 0 && + ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr; + const int trunk_flags = mtp_only + ? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0; + for (int i = 0; i < n_layer; ++i) { const bool is_mtp_layer = hparams.nextn_predict_layers > 0 && static_cast(i) >= n_layer - hparams.nextn_predict_layers; @@ -1712,7 +1718,7 @@ bool create_tensors_helper::create_qwen35moe_tensors(const LLM_TN & tn) { auto & layer = model.layers[i]; - int flags = 0; + int flags = is_mtp_layer ? 0 : trunk_flags; if (!model.mtp && is_mtp_layer) { flags |= llama_model_loader::TENSOR_SKIP; } @@ -1813,6 +1819,12 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { const int64_t value_dim = head_v_dim * n_v_heads; const int64_t conv_dim = key_dim * 2 + value_dim; + // A predictor-only MTP GGUF reports the full block count but only ships the NextN block, skip the rest + const bool mtp_only = hparams.nextn_predict_layers > 0 && + ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_NORM, "weight", 0).c_str()) == nullptr; + const int trunk_flags = mtp_only + ? llama_model_loader::TENSOR_SKIP | llama_model_loader::TENSOR_NOT_REQUIRED : 0; + for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i]; @@ -1821,7 +1833,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { ggml_context * ctx_split = ctx_for_layer_split(i); - int flags = 0; + int flags = is_mtp_layer ? 0 : trunk_flags; // Skip loading MTP layers if the feature is disabled if (!model.mtp) { if (is_mtp_layer) { @@ -1829,6 +1841,8 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { } } const int mtp_opt = is_mtp_layer ? llama_model_loader::TENSOR_NOT_REQUIRED : 0; + // q_proj may be shared with the last main block, which a predictor-only GGUF does not have + const int mtp_opt_q = mtp_only ? 0 : mtp_opt; layer.attn_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, flags); layer.attn_post_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), { n_embd }, flags); @@ -1836,7 +1850,7 @@ bool create_tensors_helper::create_qwen35_tensors(const LLM_TN & tn) { if (!hparams.is_recurrent(i)) { // Attention layers (MTP layer is always standard attention) - layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, flags | mtp_opt); + layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), { n_embd, n_embd_head_k * n_head * 2 }, flags | mtp_opt_q); layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), { n_embd, n_embd_k_gqa }, flags); layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), { n_embd, n_embd_v_gqa }, flags); layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_k * n_head, n_embd }, flags); @@ -5395,6 +5409,10 @@ bool create_tensors_helper::create_tensors() { for ([[maybe_unused]] auto mem : mem_used) LLAMA_LOG_DEBUG(" %g", mem/1024./1024.); LLAMA_LOG_DEBUG("\n"); auto & layer = model.layers[il]; + // a predictor-only MTP GGUF has no tensors at all in its main blocks + if (!layer.attn_norm && !layer.wq && !layer.wqkv && !layer.ssm_in && !layer.wo) { + continue; + } auto ctx_split = ctx_for_layer_split(il); if (layer.attn_norm) { prepare_split_tensors(-1, ctx_split, layer.attn_norm, layer.split_attn_norm, mirror, mem_used); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 119f0ca3..378f026b 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2319,6 +2319,10 @@ bool llama_model_is_step35(const llama_model * model) { return model && model->arch == LLM_ARCH_STEP35; } +bool llama_model_is_qwen35_family(const llama_model * model) { + return model && (model->arch == LLM_ARCH_QWEN35 || model->arch == LLM_ARCH_QWEN35MOE); +} + enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { if (!model) { return LLAMA_MTP_PACKAGE_INVALID; @@ -2331,13 +2335,15 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { const size_t n_nextn = model->hparams.nextn_predict_layers; const bool has_common_package_contract = llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model) || llama_model_is_gemma4_mtp_assistant(model); if (!has_common_package_contract) { return n_nextn > 0 ? LLAMA_MTP_PACKAGE_EMBEDDED : LLAMA_MTP_PACKAGE_NONE; } if (n_nextn == 0) { - if (llama_model_is_step35(model) || llama_model_is_deepseek4(model)) { + if (llama_model_is_step35(model) || llama_model_is_deepseek4(model) || + llama_model_is_qwen35_family(model)) { for (const auto & layer : model->layers) { if (layer.attn_norm != nullptr) { return LLAMA_MTP_PACKAGE_TARGET_ONLY; @@ -2353,7 +2359,10 @@ enum llama_mtp_package llama_model_mtp_package(const llama_model * model) { } const size_t first = n_layers - n_nextn; - const bool has_tail = model->layers[first].nextn.eh_proj != nullptr; + // A Qwen3.5 NextN block loads eh_proj, attn_q and the MLP as optional, so none of them + // marks a predictor tail on its own; enorm is required and is present in all of them. + const bool has_tail = model->layers[first].nextn.eh_proj != nullptr || + (llama_model_is_qwen35_family(model) && model->layers[first].nextn.enorm != nullptr); bool has_trunk = false; for (size_t il = 0; il < first; ++il) {