From da2293ded3b89459c0818747ef6f87e41c546549 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Fri, 7 Aug 2026 07:51:35 +0300 Subject: [PATCH] Better placement of MoE tensors with -ncmoe and 1 GPU (#2262) --- src/llama-load-tensors.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index 94d3a636..83a528ca 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -264,10 +264,17 @@ create_tensors_helper::create_tensors_helper(llama_model_loader & _ml, llama_mod if (ml.ncmoe > 0) { auto buft = llama_default_buffer_type_cpu(true); if (model.split_mode == LLAMA_SPLIT_MODE_ATTN || model.split_mode == LLAMA_SPLIT_MODE_GRAPH || ml.ncmoe >= n_layer || model.devices.size() < 2) { - int nmax = std::min(ml.ncmoe, n_layer); - for (int i = 0; i < nmax; ++i) { - std::string pattern = "blk\\." + std::to_string(i) + "\\.ffn_(up|down|gate|gate_up)_exps\\.(weight|scale)"; - this->overrides.emplace_back(std::make_pair(std::regex(pattern), buft)); + const auto tn = LLM_TN(model.arch); + int last_layer = n_layer - model.hparams.nextn_predict_layers; + int ndone = 0; + for (int i = last_layer-1; i >= 0; --i) { + auto d_name = tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i); + auto d_meta = ml.get_tensor_meta(d_name.c_str()); + if (d_meta) { + std::string pattern = "blk\\." + std::to_string(i) + "\\.ffn_(up|down|gate|gate_up)_exps\\.(weight|scale)"; + this->overrides.emplace_back(std::make_pair(std::regex(pattern), buft)); + if (++ndone == ml.ncmoe) break; + } } } else if (model.split_mode == LLAMA_SPLIT_MODE_LAYER) {