nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Be able to use imatrix computed with merged ffn_gate_up_exps (#1419) 

* Be able to use imatrix computed with merged ffn_gate_up_exps

* Also the other way around
This commit is contained in:
Kawrakow 2026-03-13 17:57:56 +01:00 committed by GitHub
parent 633c1baa94
commit c2b8e95700
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 27 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/llama-quantize.cpp
View File

@ -1448,19 +1448,33 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (imatrix_data) {
auto it = imatrix_data->find(tensor->name);
if (it == imatrix_data->end()) {
// MLA hack: most imatrix files floating around the Internet have been computed with standard attention.
// This means that the imatrix file does not contain data for the *.attn_k_b.weight and *.attn_v_b.weight
// required by MLA. But the *.attn_v_b.weight tensors "see" the exact same activations as the
// *.attn_kv_b.weight tensors used in standard attention. Hence, if we find imatrix data for
// *.attn_kv_b.weight we can use it for *.attn_v_b.weight and vice versa.
std::string name{tensor->name};
static std::array<std::string, 2> alternatives{".attn_v_b.weight", ".attn_kv_b.weight"};
for (int j = 0; j < int(alternatives.size()); ++j) {
if (auto pos = name.find(alternatives[j]); pos != std::string::npos) {
int j1 = (j + 1) % alternatives.size();
auto alternative_name = name.substr(0, pos) + alternatives[j1];
it = imatrix_data->find(alternative_name);
break;
if (auto pos1 = name.find("ffn_up_exps.weight"), pos2 = name.find("ffn_gate_exps.weight"); pos1 != std::string::npos || pos2 != std::string::npos) {
// Merged ffn_up/gate_exps hack
auto pos = pos1 != std::string::npos ? pos1 : pos2;
auto merged_name = name.substr(0, pos) + "ffn_gate_up_exps.weight";
it = imatrix_data->find(merged_name);
if (it == imatrix_data->end()) {
auto up_name = name.substr(0, pos) + "ffn_up_exps.weight";
it = imatrix_data->find(up_name);
}
} else if (auto pos = name.find("ffn_gate_up_exps.weight"); pos != std::string::npos) {
auto not_merged_name = name.substr(0, pos) + "ffn_up_exps.weight";
it = imatrix_data->find(not_merged_name);
} else {
// MLA hack: most imatrix files floating around the Internet have been computed with standard attention.
// This means that the imatrix file does not contain data for the *.attn_k_b.weight and *.attn_v_b.weight
// required by MLA. But the *.attn_v_b.weight tensors "see" the exact same activations as the
// *.attn_kv_b.weight tensors used in standard attention. Hence, if we find imatrix data for
// *.attn_kv_b.weight we can use it for *.attn_v_b.weight and vice versa.
std::string name{tensor->name};
static std::array<std::string, 2> alternatives{".attn_v_b.weight", ".attn_kv_b.weight"};
for (int j = 0; j < int(alternatives.size()); ++j) {
if (auto pos = name.find(alternatives[j]); pos != std::string::npos) {
int j1 = (j + 1) % alternatives.size();
auto alternative_name = name.substr(0, pos) + alternatives[j1];
it = imatrix_data->find(alternative_name);
break;
}
}
}
}