nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Gemma4 partial offload (#1657) 

* Fix Gemma4 partial offload

* Also here
This commit is contained in:
Kawrakow 2026-04-19 14:25:05 +02:00 committed by GitHub
parent d6657db245
commit 00ba208a5c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llama-load-tensors.cpp
View File

@ -235,7 +235,7 @@ create_tensors_helper::create_tensors_helper(llama_model_loader & _ml, llama_mod
if (model.split_mode == LLAMA_SPLIT_MODE_ATTN || model.split_mode == LLAMA_SPLIT_MODE_GRAPH || ml.ncmoe >= n_layer || model.devices.size() < 2) {
int nmax = std::min(ml.ncmoe, n_layer);
for (int i = 0; i < nmax; ++i) {
std::string pattern = "blk\\." + std::to_string(i) + "\\.(ffn_(up|down|gate|gate_up)_exps\\.weight)";
std::string pattern = "blk\\." + std::to_string(i) + "\\.ffn_(up|down|gate|gate_up)_exps\\.(weight|scale)";
this->overrides.emplace_back(std::make_pair(std::regex(pattern), buft));
}
}

6
src/llama-model.cpp
View File

@ -1890,14 +1890,14 @@ llm_tensor llm_tensor_type(llm_arch arch, const std::string & tensor_name, int i
for (auto & entry : it->second) {
auto base_name = ::format(entry.second.c_str(), il);
auto this_name = base_name + ".weight";
if (tensor_name.find(this_name) == 0) {
if (tensor_name == this_name) {
return entry.first;
}
this_name = base_name + ".bias";
if (tensor_name.find(this_name) == 0) {
if (tensor_name == this_name) {
return entry.first;
}
if (tensor_name.find(base_name) == 0) {
if (tensor_name == base_name) {
return entry.first;
}
}

10
src/llama.cpp
View File

@ -2119,7 +2119,11 @@ static std::pair<std::vector<double>, double> get_layer_sizes(const llama_model_
continue;
}
result[il] += size;
if (auto pos = name.rfind(".bias"); pos < name.size() && name.size() - pos == 4) {
if (auto pos = name.rfind(".bias"); pos < name.size() && name.size() - pos == 5) {
// bias, we don't need to account for those
continue;
}
if (auto pos = name.rfind(".scale"); pos < name.size() && name.size() - pos == 6) {
// bias, we don't need to account for those
continue;
}
@ -2475,7 +2479,7 @@ static bool llm_load_tensors(
}
if (has_experts) {
LLAMA_LOG_INFO("Adding experts CPU overrides for layer %d\n", il);
std::string pattern = "blk\\." + std::to_string(il) + "\\.(ffn_(up|down|gate|gate_up)_exps\\.weight)";
std::string pattern = "blk\\." + std::to_string(il) + "\\.(ffn_(up|down|gate|gate_up)_exps\\.(weight|scale))";
auto & o = overrides.emplace_back();
o.pattern = strdup(pattern.c_str());
o.buft = buft;
@ -2550,7 +2554,7 @@ static bool llm_load_tensors(
}
if (has_experts) {
LLAMA_LOG_INFO("Adding experts CPU overrides for layer %d in device %d\n", il, id);
std::string pattern = "blk\\." + std::to_string(il) + "\\.(ffn_(up|down|gate|gate_up)_exps\\.weight)";
std::string pattern = "blk\\." + std::to_string(il) + "\\.(ffn_(up|down|gate|gate_up)_exps\\.(weight|scale))";
auto & o = overrides.emplace_back();
o.pattern = strdup(pattern.c_str());
o.buft = buft;