From d66dc7c0c40793ad661d154149f17e9231fb7b52 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Wed, 25 Mar 2026 17:18:32 +0100 Subject: [PATCH] CUDA: restore pinned memory usage for tensor overrides (#1508) --- common/common.cpp | 3 ++- src/llama-load-tensors.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 0c3b423e..6fcf44c9 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1482,7 +1482,8 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa return true; } if (arg == "--cpu-moe" || arg == "-cmoe") { - params.tensor_buft_overrides.push_back({strdup("\\.ffn_(up|down|gate|gate_up)_exps\\.weight"), ggml_backend_cpu_buffer_type()}); + params.ncmoe = 999; + //params.tensor_buft_overrides.push_back({strdup("\\.ffn_(up|down|gate|gate_up)_exps\\.weight"), ggml_backend_cpu_buffer_type()}); return true; } if (arg == "--n-cpu-moe" || arg == "-ncmoe") { diff --git a/src/llama-load-tensors.cpp b/src/llama-load-tensors.cpp index a64df9b9..f4cebc62 100644 --- a/src/llama-load-tensors.cpp +++ b/src/llama-load-tensors.cpp @@ -217,12 +217,14 @@ create_tensors_helper::create_tensors_helper(llama_model_loader & _ml, llama_mod if (ml.tensor_buft_overrides) { for (const auto * o = ml.tensor_buft_overrides; o->pattern != nullptr; ++o) { - overrides.emplace_back(std::make_pair(std::regex(o->pattern), o->buft)); + auto buft = o->buft; + if (ggml_backend_buft_is_host(buft)) buft = llama_default_buffer_type_cpu(true); + overrides.emplace_back(std::make_pair(std::regex(o->pattern), buft)); } } if (ml.ncmoe > 0) { - auto buft = ggml_backend_cpu_buffer_type(); + auto buft = llama_default_buffer_type_cpu(true); if (model.split_mode == LLAMA_SPLIT_MODE_ATTN || model.split_mode == LLAMA_SPLIT_MODE_GRAPH || ml.ncmoe >= n_layer || model.devices.size() < 2) { int nmax = std::min(ml.ncmoe, n_layer); for (int i = 0; i < nmax; ++i) {