nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CUDA: restore pinned memory usage for tensor overrides (#1508)

This commit is contained in:
Kawrakow 2026-03-25 17:18:32 +01:00 committed by GitHub
parent 1f3e832cb3
commit d66dc7c0c4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
common/common.cpp
View File

@ -1482,7 +1482,8 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
return true;
}
if (arg == "--cpu-moe" || arg == "-cmoe") {
params.tensor_buft_overrides.push_back({strdup("\\.ffn_(up|down|gate|gate_up)_exps\\.weight"), ggml_backend_cpu_buffer_type()});
params.ncmoe = 999;
//params.tensor_buft_overrides.push_back({strdup("\\.ffn_(up|down|gate|gate_up)_exps\\.weight"), ggml_backend_cpu_buffer_type()});
return true;
}
if (arg == "--n-cpu-moe" || arg == "-ncmoe") {

6
src/llama-load-tensors.cpp
View File

@ -217,12 +217,14 @@ create_tensors_helper::create_tensors_helper(llama_model_loader & _ml, llama_mod
if (ml.tensor_buft_overrides) {
for (const auto * o = ml.tensor_buft_overrides; o->pattern != nullptr; ++o) {
overrides.emplace_back(std::make_pair(std::regex(o->pattern), o->buft));
auto buft = o->buft;
if (ggml_backend_buft_is_host(buft)) buft = llama_default_buffer_type_cpu(true);
overrides.emplace_back(std::make_pair(std::regex(o->pattern), buft));
}
}
if (ml.ncmoe > 0) {
auto buft = ggml_backend_cpu_buffer_type();
auto buft = llama_default_buffer_type_cpu(true);
if (model.split_mode == LLAMA_SPLIT_MODE_ATTN || model.split_mode == LLAMA_SPLIT_MODE_GRAPH || ml.ncmoe >= n_layer || model.devices.size() < 2) {
int nmax = std::min(ml.ncmoe, n_layer);
for (int i = 0; i < nmax; ++i) {