Quantization fudge factors (#2361)
* Fix MMQ check when quant does not support MMQ * WIP: fudge factors * Add to llama-quantize
This commit is contained in:
Kawrakow
GitHub
parent
96ac84c587
commit
7cff686d37
|
|
@ -14,6 +14,7 @@
|
|||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <cmath>
|
||||
|
||||
struct quant_option {
|
||||
|
|
@ -152,7 +153,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
|
|||
//
|
||||
[[noreturn]]
|
||||
static void usage(const char * executable) {
|
||||
printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--per-layer-token-embedding-type] [--extra-output-tensor] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable);
|
||||
printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--per-layer-token-embedding-type] [--extra-output-tensor] [--fudge-factors] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable);
|
||||
printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n");
|
||||
printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n");
|
||||
printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n");
|
||||
|
|
@ -172,6 +173,7 @@ static void usage(const char * executable) {
|
|||
printf(" --repack-pattern Comma separated list of regexs to use for matching tensor names to be repacked.\n\n");
|
||||
printf(" --symmetric-q40 Use [-7:7] range for Q4_0 quantization (turns off imatrix)\n\n");
|
||||
printf(" --slow-iq2ks Use the original very slow IQ2_KS quantization method.\n\n");
|
||||
printf(" --fudge-factors type1=ff1,type2=ff2... Apply scale fudge factors during quantization as specified by type=fudge-factor.\n\n");
|
||||
printf("Additional specific tensor quantization types used in the custom quant scheme 'CQS (default is Q2_K):\n");
|
||||
printf(" --attn-q-type ggml_type: use this ggml_type for the attn_q.weight tensor.\n");
|
||||
printf(" --attn-k-type ggml_type: use this ggml_type for the attn_k.weight tensor.\n");
|
||||
|
|
@ -341,6 +343,32 @@ static bool parse_custom_quants(const std::string& arg, std::vector<CustomQ>& cu
|
|||
return true;
|
||||
}
|
||||
|
||||
static bool parse_fudge_factors(const std::string & arg, std::unordered_map<ggml_type, float> & factors) {
|
||||
for (const auto & item : string_split<std::string>(arg, ',')) {
|
||||
auto pos = item.find('=');
|
||||
if (pos == std::string::npos) {
|
||||
fprintf(stderr, "Invalid fudge factor input %s\n", arg.c_str());
|
||||
return false;
|
||||
}
|
||||
auto type_as_string = item.substr(0, pos);
|
||||
auto value_as_string = item.substr(pos + 1);
|
||||
auto type = parse_ggml_type(type_as_string.c_str());
|
||||
if (type == GGML_TYPE_COUNT) {
|
||||
fprintf(stderr, "Invalid quantization type '%s' in fudge factor input %s\n", type_as_string.c_str(), item.c_str());
|
||||
return false;
|
||||
}
|
||||
std::istringstream stream(value_as_string);
|
||||
float val; stream >> val;
|
||||
if (stream.fail()) {
|
||||
fprintf(stderr, "Invalid fudge factor '%s' in fudge factor input %s\n", value_as_string.c_str(), item.c_str());
|
||||
return false;
|
||||
}
|
||||
factors[type] = val;
|
||||
printf("Adding scale fudge factor = %g for type %s\n", val, ggml_type_name(type));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
if (argc < 3) {
|
||||
usage(argv[0]);
|
||||
|
|
@ -358,6 +386,8 @@ int main(int argc, char ** argv) {
|
|||
|
||||
std::vector<std::string> repack_patterns;
|
||||
|
||||
std::unordered_map<ggml_type, float> fudge_factors;
|
||||
|
||||
bool hide_imatrix = false;
|
||||
|
||||
for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {
|
||||
|
|
@ -466,6 +496,10 @@ int main(int argc, char ** argv) {
|
|||
if (arg_idx == argc-1 || !parse_custom_quants(argv[++arg_idx], custom_quants)) {
|
||||
usage(argv[0]);
|
||||
}
|
||||
} else if (strcmp(argv[arg_idx], "--fudge-factors") == 0) {
|
||||
if (arg_idx == argc-1 || !parse_fudge_factors(argv[++arg_idx], fudge_factors)) {
|
||||
usage(argv[0]);
|
||||
}
|
||||
} else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) {
|
||||
params.allow_requantize = true;
|
||||
} else if (strcmp(argv[arg_idx], "--pure") == 0) {
|
||||
|
|
@ -499,6 +533,12 @@ int main(int argc, char ** argv) {
|
|||
}
|
||||
}
|
||||
|
||||
if (!fudge_factors.empty()) {
|
||||
for (auto [type, factor] : fudge_factors) {
|
||||
ggml_set_quantize_fudge_factor(type, factor);
|
||||
}
|
||||
}
|
||||
|
||||
if (!repack_patterns.empty()) {
|
||||
params.repack_pattern = &repack_patterns;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3353,6 +3353,9 @@ extern "C" {
|
|||
struct ggml_tensor * dst,
|
||||
struct ggml_tensor * src);
|
||||
|
||||
GGML_API void ggml_set_quantize_fudge_factor(enum ggml_type type, float fudge);
|
||||
GGML_API float ggml_get_quantize_fudge_factor(enum ggml_type type);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -677,6 +677,8 @@ void quantize_row_q4_0_ref(const float * restrict x, block_q4_0 * restrict y, in
|
|||
|
||||
const int nb = k / qk;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q4_0);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
float amax = 0.0f; // absolute max
|
||||
float max = 0.0f;
|
||||
|
|
@ -692,7 +694,7 @@ void quantize_row_q4_0_ref(const float * restrict x, block_q4_0 * restrict y, in
|
|||
const float d = max / -8;
|
||||
const float id = d ? 1.0f/d : 0.0f;
|
||||
|
||||
y[i].d = GGML_FP32_TO_FP16(d);
|
||||
y[i].d = GGML_FP32_TO_FP16(fudge*d);
|
||||
|
||||
for (int j = 0; j < qk/2; ++j) {
|
||||
const float x0 = x[i*qk + 0 + j]*id;
|
||||
|
|
@ -761,6 +763,8 @@ void quantize_row_q5_0_ref(const float * restrict x, block_q5_0 * restrict y, in
|
|||
|
||||
const int nb = k / qk;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q5_0);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
float amax = 0.0f; // absolute max
|
||||
float max = 0.0f;
|
||||
|
|
@ -776,7 +780,7 @@ void quantize_row_q5_0_ref(const float * restrict x, block_q5_0 * restrict y, in
|
|||
const float d = max / -16;
|
||||
const float id = d ? 1.0f/d : 0.0f;
|
||||
|
||||
y[i].d = GGML_FP32_TO_FP16(d);
|
||||
y[i].d = GGML_FP32_TO_FP16(fudge * d);
|
||||
|
||||
uint32_t qh = 0;
|
||||
|
||||
|
|
@ -908,6 +912,8 @@ void quantize_row_q8_0_ref(const float * restrict x, block_q8_0 * restrict y, in
|
|||
assert(k % QK8_0 == 0);
|
||||
const int nb = k / QK8_0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_0);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
float amax = 0.0f; // absolute max
|
||||
|
||||
|
|
@ -919,7 +925,7 @@ void quantize_row_q8_0_ref(const float * restrict x, block_q8_0 * restrict y, in
|
|||
const float d = amax / ((1 << 7) - 1);
|
||||
const float id = d ? 1.0f/d : 0.0f;
|
||||
|
||||
y[i].d = GGML_FP32_TO_FP16(d);
|
||||
y[i].d = GGML_FP32_TO_FP16(fudge * d);
|
||||
|
||||
for (int j = 0; j < QK8_0; ++j) {
|
||||
const float x0 = x[i*QK8_0 + j]*id;
|
||||
|
|
@ -2491,6 +2497,8 @@ void quantize_row_q3_K_ref(const float * restrict x, block_q3_K * restrict y, in
|
|||
int8_t L[QK_K];
|
||||
float scales[QK_K / 16];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q3_K);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
|
||||
float max_scale = 0;
|
||||
|
|
@ -2517,7 +2525,7 @@ void quantize_row_q3_K_ref(const float * restrict x, block_q3_K * restrict y, in
|
|||
l >>= 4;
|
||||
y[i].scales[j%4 + 8] |= (l << (2*(j/4)));
|
||||
}
|
||||
y[i].d = GGML_FP32_TO_FP16(1/iscale);
|
||||
y[i].d = GGML_FP32_TO_FP16(fudge/iscale);
|
||||
} else {
|
||||
y[i].d = GGML_FP32_TO_FP16(0.f);
|
||||
}
|
||||
|
|
@ -2624,6 +2632,8 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri
|
|||
float sw[QK_K / 16];
|
||||
int8_t Ls[QK_K / 16];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q3_K);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
|
||||
float sumx2 = 0;
|
||||
|
|
@ -2658,7 +2668,7 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri
|
|||
l >>= 4;
|
||||
y[i].scales[j%4 + 8] |= (l << (2*(j/4)));
|
||||
}
|
||||
y[i].d = GGML_FP32_TO_FP16(d_block);
|
||||
y[i].d = GGML_FP32_TO_FP16(d_block * fudge);
|
||||
|
||||
int8_t sc;
|
||||
for (int j = 0; j < QK_K/16; ++j) {
|
||||
|
|
@ -3165,6 +3175,8 @@ void quantize_row_q6_K_ref(const float * restrict x, block_q6_K * restrict y, in
|
|||
int8_t L[QK_K];
|
||||
float scales[QK_K/16];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_K);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
|
||||
float max_scale = 0;
|
||||
|
|
@ -3191,7 +3203,7 @@ void quantize_row_q6_K_ref(const float * restrict x, block_q6_K * restrict y, in
|
|||
}
|
||||
|
||||
float iscale = -128.f/max_scale;
|
||||
y[i].d = GGML_FP32_TO_FP16(1/iscale);
|
||||
y[i].d = GGML_FP32_TO_FP16(fudge/iscale);
|
||||
for (int ib = 0; ib < QK_K/16; ++ib) {
|
||||
y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));
|
||||
}
|
||||
|
|
@ -3272,6 +3284,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri
|
|||
int8_t L[QK_K];
|
||||
float scales[QK_K/16];
|
||||
//float weights[16];
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_K);
|
||||
|
||||
for (int i = 0; i < nb; i++) {
|
||||
|
||||
|
|
@ -3311,7 +3324,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri
|
|||
}
|
||||
|
||||
float iscale = -128.f/max_scale;
|
||||
y[i].d = GGML_FP32_TO_FP16(1/iscale);
|
||||
y[i].d = GGML_FP32_TO_FP16(fudge/iscale);
|
||||
for (int ib = 0; ib < QK_K/16; ++ib) {
|
||||
y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]));
|
||||
}
|
||||
|
|
@ -3382,6 +3395,8 @@ static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restri
|
|||
for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];
|
||||
float sigma2 = sum_x2/n_per_row;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q4_0);
|
||||
|
||||
const int64_t nb = n_per_row/QK4_0;
|
||||
for (int ib = 0; ib < nb; ++ib) {
|
||||
const float * xb = x + QK4_0 * ib;
|
||||
|
|
@ -3392,7 +3407,7 @@ static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restri
|
|||
for (int j = 0; j < QK4_0; ++j) weight[j] = xb[j]*xb[j];
|
||||
}
|
||||
float d = make_qx_quants(QK4_0, 8, xb, L, 1, weight);
|
||||
y[ib].d = GGML_FP32_TO_FP16(d);
|
||||
y[ib].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
y[ib].qs[j] = L[j] | (L[j+16] << 4);
|
||||
}
|
||||
|
|
@ -3513,6 +3528,7 @@ static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restri
|
|||
float sum_x2 = 0;
|
||||
for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j];
|
||||
float sigma2 = sum_x2/n_per_row;
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q5_0);
|
||||
|
||||
const int64_t nb = n_per_row/QK5_0;
|
||||
for (int ib = 0; ib < nb; ++ib) {
|
||||
|
|
@ -3520,7 +3536,7 @@ static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restri
|
|||
const float * qw = quant_weights + QK5_0 * ib;
|
||||
for (int j = 0; j < QK5_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]);
|
||||
float d = make_qx_quants(QK5_0, 16, xb, L, 1, weight);
|
||||
y[ib].d = GGML_FP32_TO_FP16(d);
|
||||
y[ib].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
|
||||
uint32_t qh = 0;
|
||||
|
||||
|
|
@ -3623,6 +3639,8 @@ static void quantize_row_q6_0_impl(const float * restrict x, block_q6_0 * restri
|
|||
sigma2 = sum_x2/n_per_row;
|
||||
}
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_0);
|
||||
|
||||
const int64_t nb = n_per_row/QK6_0;
|
||||
for (int ib = 0; ib < nb; ++ib) {
|
||||
const float * xb = x + QK6_0 * ib;
|
||||
|
|
@ -3633,7 +3651,7 @@ static void quantize_row_q6_0_impl(const float * restrict x, block_q6_0 * restri
|
|||
for (int j = 0; j < QK6_0; ++j) weight[j] = xb[j]*xb[j];
|
||||
}
|
||||
float d = make_qx_quants(QK6_0, 32, xb, L, 1, weight);
|
||||
y[ib].d = GGML_FP32_TO_FP16(d);
|
||||
y[ib].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
|
||||
memset(y[ib].qh, 0, QK6_0/4);
|
||||
|
||||
|
|
@ -13004,6 +13022,8 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict
|
|||
uint8_t block_signs[4];
|
||||
uint32_t q2[2*(QK_K/32)];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_XXS);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
y[ibl].d = GGML_FP32_TO_FP16(0.f);
|
||||
|
|
@ -13141,7 +13161,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict
|
|||
}
|
||||
|
||||
float d = max_scale/31;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d);
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
float id = 1/d;
|
||||
for (int ib = 0; ib < QK_K/32; ++ib) {
|
||||
int l = nearest_int(0.5f*(id*scales[ib]-1));
|
||||
|
|
@ -13191,6 +13211,8 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v
|
|||
float sumx[17], sumw[17], pairs[32];
|
||||
int * int_pairs = (int *)(pairs + 1);
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_XS);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
y[ibl].d = GGML_FP32_TO_FP16(0.f);
|
||||
|
|
@ -13366,7 +13388,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v
|
|||
}
|
||||
|
||||
float d = max_scale/31;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d);
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
float id = 1/d;
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int ib = 0; ib < QK_K/16; ++ib) {
|
||||
|
|
@ -13395,7 +13417,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v
|
|||
}
|
||||
}
|
||||
memcpy(y[ibl].qs, q2, QK_K/4);
|
||||
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(1.05f*sumqx/sumq2);
|
||||
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -13715,6 +13737,8 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v
|
|||
uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4);
|
||||
uint8_t * qh = q3 + 3*(QK_K/8);
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_XXS);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
dh[0] = GGML_FP32_TO_FP16(0.f);
|
||||
|
|
@ -13865,7 +13889,7 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v
|
|||
}
|
||||
|
||||
float d = max_scale/31;
|
||||
dh[0] = GGML_FP32_TO_FP16(d * 1.0125f); // small improvement via this fudge factor
|
||||
dh[0] = GGML_FP32_TO_FP16(d * fudge);
|
||||
float id = 1/d;
|
||||
for (int ib = 0; ib < QK_K/32; ++ib) {
|
||||
int l = nearest_int(0.5f*(id*scales[ib]-1));
|
||||
|
|
@ -13938,6 +13962,8 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo
|
|||
const int bs4 = block_size/4;
|
||||
const int bs8 = block_size/8;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_S);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq3_s));
|
||||
|
|
@ -14074,7 +14100,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo
|
|||
}
|
||||
|
||||
float d = max_scale/31;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f);
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
float id = 1/d;
|
||||
for (int ib = 0; ib < QK_K/block_size; ib += 2) {
|
||||
int l1 = nearest_int(0.5f*(id*scales[ib+0]-1));
|
||||
|
|
@ -14376,6 +14402,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy
|
|||
//const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA};
|
||||
|
||||
//int * idx = (int *)(pairs + 1);
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ1_S);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
|
|
@ -14413,7 +14440,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy
|
|||
}
|
||||
|
||||
float d = max_scale/15;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.125f is another fudge factor. Don't ask me why it is needed.
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d*fudge);
|
||||
float id = 1/d;
|
||||
for (int ib = 0; ib < QK_K/block_size; ++ib) {
|
||||
int l = nearest_int(0.5f*(id*scales[ib]-1));
|
||||
|
|
@ -14632,6 +14659,8 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy
|
|||
iq1m_scale_t s;
|
||||
const float * xx;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ1_M);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
memset(y[ibl].qs, 0, QK_K/8);
|
||||
memset(y[ibl].qh, 0, QK_K/16);
|
||||
|
|
@ -14720,7 +14749,7 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy
|
|||
}
|
||||
}
|
||||
if (sumq2_f > 0) d = sumqx_f/sumq2_f;
|
||||
s.f16 = GGML_FP32_TO_FP16(d*1.085f); // 1.085f is another fudge factor. Don't ask me why it is needed.
|
||||
s.f16 = GGML_FP32_TO_FP16(d*fudge);
|
||||
sc[0] |= ((s.u16 & 0x000f) << 12);
|
||||
sc[1] |= ((s.u16 & 0x00f0) << 8);
|
||||
sc[2] |= ((s.u16 & 0x0f00) << 4);
|
||||
|
|
@ -14794,6 +14823,8 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block
|
|||
memset(q4, 0, super_block_size/2);
|
||||
dh[0] = GGML_FP32_TO_FP16(0.f);
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(super_block_size/block_size > 1 ? GGML_TYPE_IQ4_XS : GGML_TYPE_IQ4_NL);
|
||||
|
||||
float max_scale = 0, amax_scale = 0;
|
||||
for (int ib = 0; ib < super_block_size/block_size; ++ib) {
|
||||
const float * xb = x + ib*block_size;
|
||||
|
|
@ -14915,7 +14946,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block
|
|||
int nb = super_block_size/block_size;
|
||||
memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t));
|
||||
float d = -max_scale/32;
|
||||
dh[0] = GGML_FP32_TO_FP16(d);
|
||||
dh[0] = GGML_FP32_TO_FP16(d*fudge);
|
||||
float id = d ? 1/d : 0.f;
|
||||
for (int ib = 0; ib < super_block_size/block_size; ++ib) {
|
||||
int l = nearest_int(id*scales[ib]);
|
||||
|
|
@ -14935,7 +14966,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block
|
|||
scales_h[ib/8] |= (l_h << 2*(ib%8));
|
||||
}
|
||||
} else {
|
||||
dh[0] = GGML_FP32_TO_FP16(scales[0]);
|
||||
dh[0] = GGML_FP32_TO_FP16(scales[0] * fudge);
|
||||
if (ntry > 0) {
|
||||
float id = scales[0] ? 1/scales[0] : 0;
|
||||
for (int j = 0; j < super_block_size; ++j) {
|
||||
|
|
@ -15059,6 +15090,8 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy
|
|||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_S);
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq2_s));
|
||||
|
|
@ -15187,7 +15220,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy
|
|||
}
|
||||
|
||||
float d = max_scale/31;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f);
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d * fudge);
|
||||
float id = 1/d;
|
||||
for (int ib = 0; ib < QK_K/16; ++ib) {
|
||||
int l = nearest_int(0.5f*(id*scales[ib]-1));
|
||||
|
|
|
|||
107
ggml/src/ggml.c
107
ggml/src/ggml.c
|
|
@ -1967,6 +1967,113 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
|
|||
},
|
||||
};
|
||||
|
||||
static float fudge_factors[GGML_TYPE_COUNT] = {
|
||||
[GGML_TYPE_I8] = 1.0f,
|
||||
[GGML_TYPE_I16] = 1.0f,
|
||||
[GGML_TYPE_I32] = 1.0f,
|
||||
[GGML_TYPE_I64] = 1.0f,
|
||||
[GGML_TYPE_F64] = 1.0f,
|
||||
[GGML_TYPE_F32] = 1.0f,
|
||||
[GGML_TYPE_F16] = 1.0f,
|
||||
[GGML_TYPE_Q4_0] = 1.0f,
|
||||
[GGML_TYPE_Q4_1] = 1.0f,
|
||||
[4] = 1.0f,
|
||||
[5] = 1.0f,
|
||||
[GGML_TYPE_Q5_0] = 1.0f,
|
||||
[GGML_TYPE_Q5_1] = 1.0f,
|
||||
[GGML_TYPE_Q6_0] = 1.0f,
|
||||
[GGML_TYPE_Q8_0] = 1.0f,
|
||||
[GGML_TYPE_Q8_1] = 1.0f,
|
||||
[GGML_TYPE_Q8_0_X4] = 1.0f,
|
||||
[GGML_TYPE_Q8_1_X4] = 1.0f,
|
||||
[GGML_TYPE_Q8_2_X4] = 1.0f,
|
||||
[GGML_TYPE_Q2_K] = 1.0f,
|
||||
[GGML_TYPE_Q2_K_R4] = 1.0f,
|
||||
[GGML_TYPE_Q3_K] = 1.0f,
|
||||
[GGML_TYPE_Q3_K_R4] = 1.0f,
|
||||
[GGML_TYPE_Q4_K] = 1.0f,
|
||||
[GGML_TYPE_Q4_K_R4] = 1.0f,
|
||||
[GGML_TYPE_Q5_K] = 1.0f,
|
||||
[GGML_TYPE_Q5_K_R4] = 1.0f,
|
||||
[GGML_TYPE_Q6_K] = 1.0f,
|
||||
[GGML_TYPE_Q6_K_R4] = 1.0f,
|
||||
[GGML_TYPE_Q8_K_R8] = 1.0f,
|
||||
[GGML_TYPE_Q8_K_R16] = 1.0f,
|
||||
[GGML_TYPE_IQ2_XXS] = 1.0f, // Note: via the grid we effectively scale with 43/40
|
||||
[GGML_TYPE_IQ2_XXS_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ2_XS] = 1.05f, // Note: via the grid we effective further scale with 43/40
|
||||
[GGML_TYPE_IQ2_XS_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ3_XXS] = 1.0125f, // Note: via the grid we effective further scale with 63/60
|
||||
[GGML_TYPE_IQ3_XXS_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ3_S] = 1.033f,
|
||||
[GGML_TYPE_IQ3_S_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ2_S] = 0.9875f, // Note: via the grid we effective further scale with 43/40
|
||||
[GGML_TYPE_IQ2_S_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ1_S] = 1.125f,
|
||||
[GGML_TYPE_IQ1_S_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ1_M] = 1.085f,
|
||||
[GGML_TYPE_IQ1_M_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ1_BN] = 1.0f,
|
||||
[GGML_TYPE_IQ2_BN] = 1.0f,
|
||||
[GGML_TYPE_IQ2_BN_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ4_NL] = 1.0f,
|
||||
[GGML_TYPE_IQ4_XS] = 1.0f,
|
||||
[GGML_TYPE_MXFP4] = 1.0f,
|
||||
[GGML_TYPE_MXFP4_R8] = 1.0f,
|
||||
[GGML_TYPE_IQ4_KS] = 1.0f,
|
||||
[GGML_TYPE_IQ4_KS_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ5_KS_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ4_KSS] = 1.01f,
|
||||
[GGML_TYPE_IQ5_KS] = 1.0f,
|
||||
[GGML_TYPE_Q8_K] = 1.0f,
|
||||
[GGML_TYPE_Q8_K64] = 1.0f,
|
||||
[GGML_TYPE_Q8_K128] = 1.0f,
|
||||
[GGML_TYPE_Q8_KV] = 1.0f,
|
||||
[GGML_TYPE_Q8_KV_R8] = 1.0f,
|
||||
[GGML_TYPE_Q8_K16] = 1.0f,
|
||||
[GGML_TYPE_Q8_K32] = 1.0f,
|
||||
[GGML_TYPE_Q8_KR8] = 1.0f,
|
||||
[GGML_TYPE_BF16] = 1.0f,
|
||||
[GGML_TYPE_BF16_R16] = 1.0f,
|
||||
[GGML_TYPE_Q4_0_4_4] = 1.0f,
|
||||
[GGML_TYPE_Q4_0_4_8] = 1.0f,
|
||||
[GGML_TYPE_Q4_0_8_8] = 1.0f,
|
||||
[GGML_TYPE_IQ2_K] = 1.03f,
|
||||
[GGML_TYPE_IQ2_K_R4] = 1.0f,
|
||||
// What do I do here? We use 1.03f for the slow path, 1.0f for the default
|
||||
[GGML_TYPE_IQ2_KS] = 1.0f,
|
||||
[GGML_TYPE_IQ1_KT] = 1.07f,
|
||||
[GGML_TYPE_IQ2_KT] = 1.0f, // Note: we apply 1.05f at run time!
|
||||
[GGML_TYPE_IQ3_KT] = 1.0f, // Note: we apply 1.01f at run time!
|
||||
[GGML_TYPE_IQ4_KT] = 1.0f,
|
||||
[GGML_TYPE_Q1_0_G128] = 1.0f,
|
||||
[GGML_TYPE_IQ3_K] = 1.01f,
|
||||
[GGML_TYPE_IQ3_KS] = 1.0f,
|
||||
[GGML_TYPE_IQ2_KL] = 1.025f,
|
||||
[GGML_TYPE_IQ4_K] = 1.0f,
|
||||
[GGML_TYPE_IQ4_K_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ3_K_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ5_K] = 1.0f,
|
||||
[GGML_TYPE_IQ5_K_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ6_K] = 1.0f,
|
||||
[GGML_TYPE_IQ4_NL_R4] = 1.0f,
|
||||
[GGML_TYPE_IQ4_XS_R8] = 1.0f,
|
||||
[GGML_TYPE_Q4_0_R8] = 1.0f,
|
||||
[GGML_TYPE_Q8_0_R8] = 1.0f,
|
||||
[GGML_TYPE_Q5_0_R4] = 1.0f,
|
||||
[GGML_TYPE_Q6_0_R4] = 1.0f,
|
||||
[GGML_TYPE_I2_S] = 1.0f,
|
||||
};
|
||||
|
||||
void ggml_set_quantize_fudge_factor(enum ggml_type type, float fudge) {
|
||||
fudge_factors[type] = fudge;
|
||||
}
|
||||
|
||||
float ggml_get_quantize_fudge_factor(enum ggml_type type) {
|
||||
return fudge_factors[type];
|
||||
}
|
||||
|
||||
|
||||
// For internal test use
|
||||
ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) {
|
||||
GGML_ASSERT(type < GGML_TYPE_COUNT);
|
||||
|
|
|
|||
|
|
@ -1204,6 +1204,8 @@ void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
|
||||
const int8_t * shifted_values = iq2nl_values + 4;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_K);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq2_k));
|
||||
|
|
@ -1327,7 +1329,7 @@ void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
}
|
||||
}
|
||||
}
|
||||
y[ibl].d = GGML_FP32_TO_FP16(1.030f*(sumq2 > 0 ? sumqx/sumq2 : d));
|
||||
y[ibl].d = GGML_FP32_TO_FP16(fudge*(sumq2 > 0 ? sumqx/sumq2 : d));
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -2058,6 +2060,8 @@ void quantize_row_iq2_kl_impl(const float * x, void * vy, int n_per_row, const f
|
|||
|
||||
float max_scale = 0, max_abs_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_KL);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
|
||||
std::memset(&y[ibl], 0, sizeof(block_iq2_kl));
|
||||
auto scales = all_scales + ibl*(QK_K/kBlockSize);
|
||||
|
|
@ -2208,7 +2212,7 @@ void quantize_row_iq2_kl_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
if (sumq2 > 0) d = sumqx/sumq2;
|
||||
|
||||
dptr[0] = GGML_FP32_TO_FP16(1.025f * d);
|
||||
dptr[0] = GGML_FP32_TO_FP16(fudge * d);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -2305,6 +2309,8 @@ static void quantize_row_iq3_k_impl(const float * x, void * vy, int n_per_row, c
|
|||
|
||||
const int8_t * shifted_values = iq3nl_values + 8;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_K);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq3_k));
|
||||
|
|
@ -2504,7 +2510,7 @@ static void quantize_row_iq3_k_impl(const float * x, void * vy, int n_per_row, c
|
|||
}
|
||||
}
|
||||
}
|
||||
y[ibl].d = GGML_FP32_TO_FP16(1.01f*(sumq2 > 0 ? sumqx/sumq2 : d));
|
||||
y[ibl].d = GGML_FP32_TO_FP16(fudge*(sumq2 > 0 ? sumqx/sumq2 : d));
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -2600,6 +2606,8 @@ static void quantize_row_iq3_ks_impl(const int super_block_size, const int block
|
|||
float amax_scale = 0;
|
||||
float max_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_KS);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) {
|
||||
memset(&y[ibl], 0, sizeof(block_iq3_ks));
|
||||
const float * xbl = x + ibl*super_block_size;
|
||||
|
|
@ -2743,7 +2751,7 @@ static void quantize_row_iq3_ks_impl(const int super_block_size, const int block
|
|||
}
|
||||
}
|
||||
}
|
||||
if (sumq2 > 0) *dptr = GGML_FP32_TO_FP16(sumqx/sumq2);
|
||||
if (sumq2 > 0) *dptr = GGML_FP32_TO_FP16(fudge*sumqx/sumq2);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2935,6 +2943,8 @@ static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int b
|
|||
|
||||
const int8_t * shifted_values = values + 16;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_K);
|
||||
|
||||
float max_scale = 0, amax_scale = 0;
|
||||
uint16_t extra = 0;
|
||||
for (int ib = 0; ib < super_block_size/block_size; ++ib) {
|
||||
|
|
@ -3030,7 +3040,7 @@ static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int b
|
|||
}
|
||||
}
|
||||
float d = -max_scale/32;
|
||||
y->d = GGML_FP32_TO_FP16(d);
|
||||
y->d = GGML_FP32_TO_FP16(fudge*d);
|
||||
y->extra = extra;
|
||||
float id = d ? 1/d : 0.f;
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
|
@ -3062,7 +3072,7 @@ static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int b
|
|||
else y->scales_l[ib/2] |= (l_l << 4);
|
||||
scales_h[ib/8] |= (l_h << 2*(ib%8));
|
||||
}
|
||||
if (sumq2 > 0) y->d = GGML_FP32_TO_FP16(sumqx/sumq2);
|
||||
if (sumq2 > 0) y->d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2);
|
||||
|
||||
for (int i = 0; i < super_block_size/32; ++i) {
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
|
|
@ -3245,6 +3255,8 @@ void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
|
||||
const int8_t * shifted_values = iq5nl_values + 32;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ5_K);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq5_k));
|
||||
|
|
@ -3369,7 +3381,7 @@ void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
|
||||
if (!max_abs_scale) continue;
|
||||
float d = -max_scale/32;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d);
|
||||
y[ibl].d = GGML_FP32_TO_FP16(fudge*d);
|
||||
y[ibl].extra = extra;
|
||||
|
||||
float id = 1/d;
|
||||
|
|
@ -3408,7 +3420,7 @@ void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
}
|
||||
}
|
||||
}
|
||||
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2);
|
||||
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -3600,6 +3612,8 @@ void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
float scales[QK_K/16];
|
||||
float weight[16];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ6_K);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq6_k));
|
||||
|
|
@ -3731,7 +3745,7 @@ void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
|
||||
if (!max_abs_scale) continue;
|
||||
float d = -max_scale/127;
|
||||
y[ibl].d = GGML_FP32_TO_FP16(d);
|
||||
y[ibl].d = GGML_FP32_TO_FP16(fudge*d);
|
||||
y[ibl].extra = extra;
|
||||
|
||||
float id = 1/d;
|
||||
|
|
@ -3769,7 +3783,7 @@ void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const fl
|
|||
}
|
||||
}
|
||||
}
|
||||
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2);
|
||||
if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -4382,6 +4396,8 @@ static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int
|
|||
|
||||
float amax_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_KS);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) {
|
||||
memset(&y[ibl], 0, sizeof(block_iq4_ks));
|
||||
const float * xbl = x + ibl*super_block_size;
|
||||
|
|
@ -4480,7 +4496,7 @@ static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int
|
|||
}
|
||||
}
|
||||
float d = amax_scale/127;
|
||||
*dptr = d;
|
||||
*dptr = fudge*d;
|
||||
if (!d) return;
|
||||
float id = d ? 1/d : 0.f;
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
|
@ -4524,7 +4540,7 @@ static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int
|
|||
}
|
||||
}
|
||||
//printf("rmse = %g\n", sqrt(mse/n_per_row));
|
||||
if (sumq2 > 0) *dptr = sumqx/sumq2;
|
||||
if (sumq2 > 0) *dptr = fudge*sumqx/sumq2;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4633,6 +4649,8 @@ static void quantize_row_iq5_ks_impl(const int super_block_size, const int block
|
|||
|
||||
float amax_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ5_KS);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) {
|
||||
memset(&y[ibl], 0, sizeof(block_iq5_ks));
|
||||
const float * xbl = x + ibl*super_block_size;
|
||||
|
|
@ -4731,7 +4749,7 @@ static void quantize_row_iq5_ks_impl(const int super_block_size, const int block
|
|||
}
|
||||
}
|
||||
float d = amax_scale/127;
|
||||
*dptr = d;
|
||||
*dptr = fudge*d;
|
||||
if (!d) return;
|
||||
float id = d ? 1/d : 0.f;
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
|
@ -4767,7 +4785,7 @@ static void quantize_row_iq5_ks_impl(const int super_block_size, const int block
|
|||
}
|
||||
}
|
||||
}
|
||||
if (sumq2 > 0) *dptr = sumqx/sumq2;
|
||||
if (sumq2 > 0) *dptr = fudge*sumqx/sumq2;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4938,6 +4956,8 @@ static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy,
|
|||
|
||||
float amax_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_KSS);
|
||||
|
||||
for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) {
|
||||
memset(&y[ibl], 0, sizeof(block_iq4_kss));
|
||||
const float * xbl = x + ibl*super_block_size;
|
||||
|
|
@ -5053,7 +5073,7 @@ static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy,
|
|||
}
|
||||
}
|
||||
float d = amax_scale/127;
|
||||
*dptr = d;
|
||||
*dptr = fudge*d;
|
||||
if (!d) return;
|
||||
float id = 1/d;
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
|
@ -5129,7 +5149,7 @@ static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy,
|
|||
}
|
||||
}
|
||||
}
|
||||
if (sumq2 > 0) *dptr = sumqx/sumq2 * 1.01f;
|
||||
if (sumq2 > 0) *dptr = fudge*sumqx/sumq2;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -9366,6 +9386,8 @@ void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
}
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ1_KT);
|
||||
|
||||
float id = d ? 1/d : 0.f;
|
||||
for (int ibl = 0; ibl < nblock; ++ibl) {
|
||||
auto scales = all_scales + ibl*Q::kNblock;
|
||||
|
|
@ -9375,7 +9397,7 @@ void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
}
|
||||
|
||||
*dptr = d;
|
||||
*dptr = fudge*d;
|
||||
if (!d) return;
|
||||
|
||||
for (int iloop = 0; iloop < 1; ++iloop) {
|
||||
|
|
@ -9429,7 +9451,7 @@ void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
if (sumq2 > 0) {
|
||||
d = sumqx/sumq2;
|
||||
*dptr = d * 1.07f;
|
||||
*dptr = d * fudge;
|
||||
if (!d) return;
|
||||
} else {
|
||||
break;
|
||||
|
|
@ -9545,6 +9567,8 @@ void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
|
||||
float amax_scale = 0, max_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_KT);
|
||||
|
||||
for (int ibl = 0; ibl < nblock; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq2_kt));
|
||||
|
|
@ -9632,7 +9656,7 @@ void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
}
|
||||
|
||||
*dptr = d;
|
||||
*dptr = d * fudge;
|
||||
if (!d) return;
|
||||
|
||||
for (int iloop = 0; iloop < 1; ++iloop) {
|
||||
|
|
@ -9686,7 +9710,7 @@ void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
if (sumq2 > 0) {
|
||||
d = sumqx/sumq2;
|
||||
*dptr = d;
|
||||
*dptr = d * fudge;
|
||||
if (!d) return;
|
||||
} else {
|
||||
break;
|
||||
|
|
@ -9841,6 +9865,8 @@ void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
|
||||
float xaux[Q::kBlockSize];
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_KT);
|
||||
|
||||
for (int ibl = 0; ibl < nblock; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq3_kt));
|
||||
|
|
@ -9945,7 +9971,7 @@ void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
}
|
||||
|
||||
*dptr = d;
|
||||
*dptr = d * fudge;
|
||||
|
||||
for (int iloop = 0; iloop < 1; ++iloop) {
|
||||
|
||||
|
|
@ -9983,7 +10009,7 @@ void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
if (sumq2 > 0) {
|
||||
d = sumqx/sumq2;
|
||||
*dptr = d;
|
||||
*dptr = d * fudge;
|
||||
if (!d) break;
|
||||
} else {
|
||||
break;
|
||||
|
|
@ -10133,6 +10159,8 @@ void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
|
||||
float amax_scale = 0, max_scale = 0;
|
||||
|
||||
const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_KT);
|
||||
|
||||
for (int ibl = 0; ibl < nblock; ++ibl) {
|
||||
|
||||
memset(&y[ibl], 0, sizeof(block_iq4_kt));
|
||||
|
|
@ -10199,7 +10227,7 @@ void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
|
||||
float d = -max_scale/64;
|
||||
|
||||
dptr[0] = d;
|
||||
dptr[0] = d * fudge;
|
||||
if (!d) return;
|
||||
|
||||
constexpr int kNumGroups = Q::kSuperBlockSize/Q::kGroupSize;
|
||||
|
|
@ -10248,7 +10276,7 @@ void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const f
|
|||
}
|
||||
if (sumq2 > 0) {
|
||||
d = sumqx/sumq2;
|
||||
dptr[0] = d;
|
||||
dptr[0] = d * fudge;
|
||||
if (!d) break;
|
||||
} else {
|
||||
break;
|
||||
|
|
|
|||
Loading…
Reference in New Issue