diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index dc5c7653..77a6727d 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include struct quant_option { @@ -152,7 +153,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp // [[noreturn]] static void usage(const char * executable) { - printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--per-layer-token-embedding-type] [--extra-output-tensor] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); + printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--per-layer-token-embedding-type] [--extra-output-tensor] [--fudge-factors] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); @@ -172,6 +173,7 @@ static void usage(const char * executable) { printf(" --repack-pattern Comma separated list of regexs to use for matching tensor names to be repacked.\n\n"); printf(" --symmetric-q40 Use [-7:7] range for Q4_0 quantization (turns off imatrix)\n\n"); printf(" --slow-iq2ks Use the original very slow IQ2_KS quantization method.\n\n"); + printf(" --fudge-factors type1=ff1,type2=ff2... Apply scale fudge factors during quantization as specified by type=fudge-factor.\n\n"); printf("Additional specific tensor quantization types used in the custom quant scheme 'CQS (default is Q2_K):\n"); printf(" --attn-q-type ggml_type: use this ggml_type for the attn_q.weight tensor.\n"); printf(" --attn-k-type ggml_type: use this ggml_type for the attn_k.weight tensor.\n"); @@ -341,6 +343,32 @@ static bool parse_custom_quants(const std::string& arg, std::vector& cu return true; } +static bool parse_fudge_factors(const std::string & arg, std::unordered_map & factors) { + for (const auto & item : string_split(arg, ',')) { + auto pos = item.find('='); + if (pos == std::string::npos) { + fprintf(stderr, "Invalid fudge factor input %s\n", arg.c_str()); + return false; + } + auto type_as_string = item.substr(0, pos); + auto value_as_string = item.substr(pos + 1); + auto type = parse_ggml_type(type_as_string.c_str()); + if (type == GGML_TYPE_COUNT) { + fprintf(stderr, "Invalid quantization type '%s' in fudge factor input %s\n", type_as_string.c_str(), item.c_str()); + return false; + } + std::istringstream stream(value_as_string); + float val; stream >> val; + if (stream.fail()) { + fprintf(stderr, "Invalid fudge factor '%s' in fudge factor input %s\n", value_as_string.c_str(), item.c_str()); + return false; + } + factors[type] = val; + printf("Adding scale fudge factor = %g for type %s\n", val, ggml_type_name(type)); + } + return true; +} + int main(int argc, char ** argv) { if (argc < 3) { usage(argv[0]); @@ -358,6 +386,8 @@ int main(int argc, char ** argv) { std::vector repack_patterns; + std::unordered_map fudge_factors; + bool hide_imatrix = false; for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) { @@ -466,6 +496,10 @@ int main(int argc, char ** argv) { if (arg_idx == argc-1 || !parse_custom_quants(argv[++arg_idx], custom_quants)) { usage(argv[0]); } + } else if (strcmp(argv[arg_idx], "--fudge-factors") == 0) { + if (arg_idx == argc-1 || !parse_fudge_factors(argv[++arg_idx], fudge_factors)) { + usage(argv[0]); + } } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) { params.allow_requantize = true; } else if (strcmp(argv[arg_idx], "--pure") == 0) { @@ -499,6 +533,12 @@ int main(int argc, char ** argv) { } } + if (!fudge_factors.empty()) { + for (auto [type, factor] : fudge_factors) { + ggml_set_quantize_fudge_factor(type, factor); + } + } + if (!repack_patterns.empty()) { params.repack_pattern = &repack_patterns; } diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d956b8d4..91a46c52 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -3353,6 +3353,9 @@ extern "C" { struct ggml_tensor * dst, struct ggml_tensor * src); + GGML_API void ggml_set_quantize_fudge_factor(enum ggml_type type, float fudge); + GGML_API float ggml_get_quantize_fudge_factor(enum ggml_type type); + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 71bc9fd3..1df61fec 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -677,6 +677,8 @@ void quantize_row_q4_0_ref(const float * restrict x, block_q4_0 * restrict y, in const int nb = k / qk; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q4_0); + for (int i = 0; i < nb; i++) { float amax = 0.0f; // absolute max float max = 0.0f; @@ -692,7 +694,7 @@ void quantize_row_q4_0_ref(const float * restrict x, block_q4_0 * restrict y, in const float d = max / -8; const float id = d ? 1.0f/d : 0.0f; - y[i].d = GGML_FP32_TO_FP16(d); + y[i].d = GGML_FP32_TO_FP16(fudge*d); for (int j = 0; j < qk/2; ++j) { const float x0 = x[i*qk + 0 + j]*id; @@ -761,6 +763,8 @@ void quantize_row_q5_0_ref(const float * restrict x, block_q5_0 * restrict y, in const int nb = k / qk; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q5_0); + for (int i = 0; i < nb; i++) { float amax = 0.0f; // absolute max float max = 0.0f; @@ -776,7 +780,7 @@ void quantize_row_q5_0_ref(const float * restrict x, block_q5_0 * restrict y, in const float d = max / -16; const float id = d ? 1.0f/d : 0.0f; - y[i].d = GGML_FP32_TO_FP16(d); + y[i].d = GGML_FP32_TO_FP16(fudge * d); uint32_t qh = 0; @@ -908,6 +912,8 @@ void quantize_row_q8_0_ref(const float * restrict x, block_q8_0 * restrict y, in assert(k % QK8_0 == 0); const int nb = k / QK8_0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_0); + for (int i = 0; i < nb; i++) { float amax = 0.0f; // absolute max @@ -919,7 +925,7 @@ void quantize_row_q8_0_ref(const float * restrict x, block_q8_0 * restrict y, in const float d = amax / ((1 << 7) - 1); const float id = d ? 1.0f/d : 0.0f; - y[i].d = GGML_FP32_TO_FP16(d); + y[i].d = GGML_FP32_TO_FP16(fudge * d); for (int j = 0; j < QK8_0; ++j) { const float x0 = x[i*QK8_0 + j]*id; @@ -2491,6 +2497,8 @@ void quantize_row_q3_K_ref(const float * restrict x, block_q3_K * restrict y, in int8_t L[QK_K]; float scales[QK_K / 16]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q3_K); + for (int i = 0; i < nb; i++) { float max_scale = 0; @@ -2517,7 +2525,7 @@ void quantize_row_q3_K_ref(const float * restrict x, block_q3_K * restrict y, in l >>= 4; y[i].scales[j%4 + 8] |= (l << (2*(j/4))); } - y[i].d = GGML_FP32_TO_FP16(1/iscale); + y[i].d = GGML_FP32_TO_FP16(fudge/iscale); } else { y[i].d = GGML_FP32_TO_FP16(0.f); } @@ -2624,6 +2632,8 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri float sw[QK_K / 16]; int8_t Ls[QK_K / 16]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q3_K); + for (int i = 0; i < nb; i++) { float sumx2 = 0; @@ -2658,7 +2668,7 @@ static void quantize_row_q3_K_impl(const float * restrict x, block_q3_K * restri l >>= 4; y[i].scales[j%4 + 8] |= (l << (2*(j/4))); } - y[i].d = GGML_FP32_TO_FP16(d_block); + y[i].d = GGML_FP32_TO_FP16(d_block * fudge); int8_t sc; for (int j = 0; j < QK_K/16; ++j) { @@ -3165,6 +3175,8 @@ void quantize_row_q6_K_ref(const float * restrict x, block_q6_K * restrict y, in int8_t L[QK_K]; float scales[QK_K/16]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_K); + for (int i = 0; i < nb; i++) { float max_scale = 0; @@ -3191,7 +3203,7 @@ void quantize_row_q6_K_ref(const float * restrict x, block_q6_K * restrict y, in } float iscale = -128.f/max_scale; - y[i].d = GGML_FP32_TO_FP16(1/iscale); + y[i].d = GGML_FP32_TO_FP16(fudge/iscale); for (int ib = 0; ib < QK_K/16; ++ib) { y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib])); } @@ -3272,6 +3284,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri int8_t L[QK_K]; float scales[QK_K/16]; //float weights[16]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_K); for (int i = 0; i < nb; i++) { @@ -3311,7 +3324,7 @@ static void quantize_row_q6_K_impl(const float * restrict x, block_q6_K * restri } float iscale = -128.f/max_scale; - y[i].d = GGML_FP32_TO_FP16(1/iscale); + y[i].d = GGML_FP32_TO_FP16(fudge/iscale); for (int ib = 0; ib < QK_K/16; ++ib) { y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib])); } @@ -3382,6 +3395,8 @@ static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restri for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; float sigma2 = sum_x2/n_per_row; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q4_0); + const int64_t nb = n_per_row/QK4_0; for (int ib = 0; ib < nb; ++ib) { const float * xb = x + QK4_0 * ib; @@ -3392,7 +3407,7 @@ static void quantize_row_q4_0_impl(const float * restrict x, block_q4_0 * restri for (int j = 0; j < QK4_0; ++j) weight[j] = xb[j]*xb[j]; } float d = make_qx_quants(QK4_0, 8, xb, L, 1, weight); - y[ib].d = GGML_FP32_TO_FP16(d); + y[ib].d = GGML_FP32_TO_FP16(d * fudge); for (int j = 0; j < 16; ++j) { y[ib].qs[j] = L[j] | (L[j+16] << 4); } @@ -3513,6 +3528,7 @@ static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restri float sum_x2 = 0; for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; float sigma2 = sum_x2/n_per_row; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q5_0); const int64_t nb = n_per_row/QK5_0; for (int ib = 0; ib < nb; ++ib) { @@ -3520,7 +3536,7 @@ static void quantize_row_q5_0_impl(const float * restrict x, block_q5_0 * restri const float * qw = quant_weights + QK5_0 * ib; for (int j = 0; j < QK5_0; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); float d = make_qx_quants(QK5_0, 16, xb, L, 1, weight); - y[ib].d = GGML_FP32_TO_FP16(d); + y[ib].d = GGML_FP32_TO_FP16(d * fudge); uint32_t qh = 0; @@ -3623,6 +3639,8 @@ static void quantize_row_q6_0_impl(const float * restrict x, block_q6_0 * restri sigma2 = sum_x2/n_per_row; } + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_Q6_0); + const int64_t nb = n_per_row/QK6_0; for (int ib = 0; ib < nb; ++ib) { const float * xb = x + QK6_0 * ib; @@ -3633,7 +3651,7 @@ static void quantize_row_q6_0_impl(const float * restrict x, block_q6_0 * restri for (int j = 0; j < QK6_0; ++j) weight[j] = xb[j]*xb[j]; } float d = make_qx_quants(QK6_0, 32, xb, L, 1, weight); - y[ib].d = GGML_FP32_TO_FP16(d); + y[ib].d = GGML_FP32_TO_FP16(d * fudge); memset(y[ib].qh, 0, QK6_0/4); @@ -13004,6 +13022,8 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict uint8_t block_signs[4]; uint32_t q2[2*(QK_K/32)]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_XXS); + for (int ibl = 0; ibl < nbl; ++ibl) { y[ibl].d = GGML_FP32_TO_FP16(0.f); @@ -13141,7 +13161,7 @@ static void quantize_row_iq2_xxs_impl(const float * restrict x, void * restrict } float d = max_scale/31; - y[ibl].d = GGML_FP32_TO_FP16(d); + y[ibl].d = GGML_FP32_TO_FP16(d * fudge); float id = 1/d; for (int ib = 0; ib < QK_K/32; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); @@ -13191,6 +13211,8 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v float sumx[17], sumw[17], pairs[32]; int * int_pairs = (int *)(pairs + 1); + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_XS); + for (int ibl = 0; ibl < nbl; ++ibl) { y[ibl].d = GGML_FP32_TO_FP16(0.f); @@ -13366,7 +13388,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v } float d = max_scale/31; - y[ibl].d = GGML_FP32_TO_FP16(d); + y[ibl].d = GGML_FP32_TO_FP16(d * fudge); float id = 1/d; float sumqx = 0, sumq2 = 0; for (int ib = 0; ib < QK_K/16; ++ib) { @@ -13395,7 +13417,7 @@ static void quantize_row_iq2_xs_impl(const float * restrict x, void * restrict v } } memcpy(y[ibl].qs, q2, QK_K/4); - if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(1.05f*sumqx/sumq2); + if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2); } } @@ -13715,6 +13737,8 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K/4); uint8_t * qh = q3 + 3*(QK_K/8); + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_XXS); + for (int ibl = 0; ibl < nbl; ++ibl) { dh[0] = GGML_FP32_TO_FP16(0.f); @@ -13865,7 +13889,7 @@ static void quantize_row_iq3_xxs_impl(int grid_size, const float * restrict x, v } float d = max_scale/31; - dh[0] = GGML_FP32_TO_FP16(d * 1.0125f); // small improvement via this fudge factor + dh[0] = GGML_FP32_TO_FP16(d * fudge); float id = 1/d; for (int ib = 0; ib < QK_K/32; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); @@ -13938,6 +13962,8 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo const int bs4 = block_size/4; const int bs8 = block_size/8; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_S); + for (int ibl = 0; ibl < nbl; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq3_s)); @@ -14074,7 +14100,7 @@ static void quantize_row_iq3_s_impl(int block_size, const float * restrict x, vo } float d = max_scale/31; - y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f); + y[ibl].d = GGML_FP32_TO_FP16(d * fudge); float id = 1/d; for (int ib = 0; ib < QK_K/block_size; ib += 2) { int l1 = nearest_int(0.5f*(id*scales[ib+0]-1)); @@ -14376,6 +14402,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy //const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA}; //int * idx = (int *)(pairs + 1); + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ1_S); for (int ibl = 0; ibl < nbl; ++ibl) { @@ -14413,7 +14440,7 @@ static void quantize_row_iq1_s_impl(const float * restrict x, void * restrict vy } float d = max_scale/15; - y[ibl].d = GGML_FP32_TO_FP16(d*1.125f); // 1.125f is another fudge factor. Don't ask me why it is needed. + y[ibl].d = GGML_FP32_TO_FP16(d*fudge); float id = 1/d; for (int ib = 0; ib < QK_K/block_size; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); @@ -14632,6 +14659,8 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy iq1m_scale_t s; const float * xx; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ1_M); + for (int ibl = 0; ibl < nbl; ++ibl) { memset(y[ibl].qs, 0, QK_K/8); memset(y[ibl].qh, 0, QK_K/16); @@ -14720,7 +14749,7 @@ static void quantize_row_iq1_m_impl(const float * restrict x, void * restrict vy } } if (sumq2_f > 0) d = sumqx_f/sumq2_f; - s.f16 = GGML_FP32_TO_FP16(d*1.085f); // 1.085f is another fudge factor. Don't ask me why it is needed. + s.f16 = GGML_FP32_TO_FP16(d*fudge); sc[0] |= ((s.u16 & 0x000f) << 12); sc[1] |= ((s.u16 & 0x00f0) << 8); sc[2] |= ((s.u16 & 0x0f00) << 4); @@ -14794,6 +14823,8 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block memset(q4, 0, super_block_size/2); dh[0] = GGML_FP32_TO_FP16(0.f); + const float fudge = ggml_get_quantize_fudge_factor(super_block_size/block_size > 1 ? GGML_TYPE_IQ4_XS : GGML_TYPE_IQ4_NL); + float max_scale = 0, amax_scale = 0; for (int ib = 0; ib < super_block_size/block_size; ++ib) { const float * xb = x + ib*block_size; @@ -14915,7 +14946,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block int nb = super_block_size/block_size; memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t)); float d = -max_scale/32; - dh[0] = GGML_FP32_TO_FP16(d); + dh[0] = GGML_FP32_TO_FP16(d*fudge); float id = d ? 1/d : 0.f; for (int ib = 0; ib < super_block_size/block_size; ++ib) { int l = nearest_int(id*scales[ib]); @@ -14935,7 +14966,7 @@ static void quantize_row_iq4_nl_impl(const int super_block_size, const int block scales_h[ib/8] |= (l_h << 2*(ib%8)); } } else { - dh[0] = GGML_FP32_TO_FP16(scales[0]); + dh[0] = GGML_FP32_TO_FP16(scales[0] * fudge); if (ntry > 0) { float id = scales[0] ? 1/scales[0] : 0; for (int j = 0; j < super_block_size; ++j) { @@ -15059,6 +15090,8 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy bool is_on_grid_aux[2]; uint8_t block_signs[2]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_S); + for (int ibl = 0; ibl < nbl; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq2_s)); @@ -15187,7 +15220,7 @@ static void quantize_row_iq2_s_impl(const float * restrict x, void * restrict vy } float d = max_scale/31; - y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f); + y[ibl].d = GGML_FP32_TO_FP16(d * fudge); float id = 1/d; for (int ib = 0; ib < QK_K/16; ++ib) { int l = nearest_int(0.5f*(id*scales[ib]-1)); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 2d72b3f4..74e9d866 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1967,6 +1967,113 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = { }, }; +static float fudge_factors[GGML_TYPE_COUNT] = { + [GGML_TYPE_I8] = 1.0f, + [GGML_TYPE_I16] = 1.0f, + [GGML_TYPE_I32] = 1.0f, + [GGML_TYPE_I64] = 1.0f, + [GGML_TYPE_F64] = 1.0f, + [GGML_TYPE_F32] = 1.0f, + [GGML_TYPE_F16] = 1.0f, + [GGML_TYPE_Q4_0] = 1.0f, + [GGML_TYPE_Q4_1] = 1.0f, + [4] = 1.0f, + [5] = 1.0f, + [GGML_TYPE_Q5_0] = 1.0f, + [GGML_TYPE_Q5_1] = 1.0f, + [GGML_TYPE_Q6_0] = 1.0f, + [GGML_TYPE_Q8_0] = 1.0f, + [GGML_TYPE_Q8_1] = 1.0f, + [GGML_TYPE_Q8_0_X4] = 1.0f, + [GGML_TYPE_Q8_1_X4] = 1.0f, + [GGML_TYPE_Q8_2_X4] = 1.0f, + [GGML_TYPE_Q2_K] = 1.0f, + [GGML_TYPE_Q2_K_R4] = 1.0f, + [GGML_TYPE_Q3_K] = 1.0f, + [GGML_TYPE_Q3_K_R4] = 1.0f, + [GGML_TYPE_Q4_K] = 1.0f, + [GGML_TYPE_Q4_K_R4] = 1.0f, + [GGML_TYPE_Q5_K] = 1.0f, + [GGML_TYPE_Q5_K_R4] = 1.0f, + [GGML_TYPE_Q6_K] = 1.0f, + [GGML_TYPE_Q6_K_R4] = 1.0f, + [GGML_TYPE_Q8_K_R8] = 1.0f, + [GGML_TYPE_Q8_K_R16] = 1.0f, + [GGML_TYPE_IQ2_XXS] = 1.0f, // Note: via the grid we effectively scale with 43/40 + [GGML_TYPE_IQ2_XXS_R4] = 1.0f, + [GGML_TYPE_IQ2_XS] = 1.05f, // Note: via the grid we effective further scale with 43/40 + [GGML_TYPE_IQ2_XS_R4] = 1.0f, + [GGML_TYPE_IQ3_XXS] = 1.0125f, // Note: via the grid we effective further scale with 63/60 + [GGML_TYPE_IQ3_XXS_R4] = 1.0f, + [GGML_TYPE_IQ3_S] = 1.033f, + [GGML_TYPE_IQ3_S_R4] = 1.0f, + [GGML_TYPE_IQ2_S] = 0.9875f, // Note: via the grid we effective further scale with 43/40 + [GGML_TYPE_IQ2_S_R4] = 1.0f, + [GGML_TYPE_IQ1_S] = 1.125f, + [GGML_TYPE_IQ1_S_R4] = 1.0f, + [GGML_TYPE_IQ1_M] = 1.085f, + [GGML_TYPE_IQ1_M_R4] = 1.0f, + [GGML_TYPE_IQ1_BN] = 1.0f, + [GGML_TYPE_IQ2_BN] = 1.0f, + [GGML_TYPE_IQ2_BN_R4] = 1.0f, + [GGML_TYPE_IQ4_NL] = 1.0f, + [GGML_TYPE_IQ4_XS] = 1.0f, + [GGML_TYPE_MXFP4] = 1.0f, + [GGML_TYPE_MXFP4_R8] = 1.0f, + [GGML_TYPE_IQ4_KS] = 1.0f, + [GGML_TYPE_IQ4_KS_R4] = 1.0f, + [GGML_TYPE_IQ5_KS_R4] = 1.0f, + [GGML_TYPE_IQ4_KSS] = 1.01f, + [GGML_TYPE_IQ5_KS] = 1.0f, + [GGML_TYPE_Q8_K] = 1.0f, + [GGML_TYPE_Q8_K64] = 1.0f, + [GGML_TYPE_Q8_K128] = 1.0f, + [GGML_TYPE_Q8_KV] = 1.0f, + [GGML_TYPE_Q8_KV_R8] = 1.0f, + [GGML_TYPE_Q8_K16] = 1.0f, + [GGML_TYPE_Q8_K32] = 1.0f, + [GGML_TYPE_Q8_KR8] = 1.0f, + [GGML_TYPE_BF16] = 1.0f, + [GGML_TYPE_BF16_R16] = 1.0f, + [GGML_TYPE_Q4_0_4_4] = 1.0f, + [GGML_TYPE_Q4_0_4_8] = 1.0f, + [GGML_TYPE_Q4_0_8_8] = 1.0f, + [GGML_TYPE_IQ2_K] = 1.03f, + [GGML_TYPE_IQ2_K_R4] = 1.0f, + // What do I do here? We use 1.03f for the slow path, 1.0f for the default + [GGML_TYPE_IQ2_KS] = 1.0f, + [GGML_TYPE_IQ1_KT] = 1.07f, + [GGML_TYPE_IQ2_KT] = 1.0f, // Note: we apply 1.05f at run time! + [GGML_TYPE_IQ3_KT] = 1.0f, // Note: we apply 1.01f at run time! + [GGML_TYPE_IQ4_KT] = 1.0f, + [GGML_TYPE_Q1_0_G128] = 1.0f, + [GGML_TYPE_IQ3_K] = 1.01f, + [GGML_TYPE_IQ3_KS] = 1.0f, + [GGML_TYPE_IQ2_KL] = 1.025f, + [GGML_TYPE_IQ4_K] = 1.0f, + [GGML_TYPE_IQ4_K_R4] = 1.0f, + [GGML_TYPE_IQ3_K_R4] = 1.0f, + [GGML_TYPE_IQ5_K] = 1.0f, + [GGML_TYPE_IQ5_K_R4] = 1.0f, + [GGML_TYPE_IQ6_K] = 1.0f, + [GGML_TYPE_IQ4_NL_R4] = 1.0f, + [GGML_TYPE_IQ4_XS_R8] = 1.0f, + [GGML_TYPE_Q4_0_R8] = 1.0f, + [GGML_TYPE_Q8_0_R8] = 1.0f, + [GGML_TYPE_Q5_0_R4] = 1.0f, + [GGML_TYPE_Q6_0_R4] = 1.0f, + [GGML_TYPE_I2_S] = 1.0f, +}; + +void ggml_set_quantize_fudge_factor(enum ggml_type type, float fudge) { + fudge_factors[type] = fudge; +} + +float ggml_get_quantize_fudge_factor(enum ggml_type type) { + return fudge_factors[type]; +} + + // For internal test use ggml_type_traits_t ggml_internal_get_type_traits(enum ggml_type type) { GGML_ASSERT(type < GGML_TYPE_COUNT); diff --git a/ggml/src/iqk/iqk_quantize.cpp b/ggml/src/iqk/iqk_quantize.cpp index 015205c9..3ef2bcec 100644 --- a/ggml/src/iqk/iqk_quantize.cpp +++ b/ggml/src/iqk/iqk_quantize.cpp @@ -1204,6 +1204,8 @@ void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const fl const int8_t * shifted_values = iq2nl_values + 4; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_K); + for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq2_k)); @@ -1327,7 +1329,7 @@ void quantize_row_iq2_k_impl(const float * x, void * vy, int n_per_row, const fl } } } - y[ibl].d = GGML_FP32_TO_FP16(1.030f*(sumq2 > 0 ? sumqx/sumq2 : d)); + y[ibl].d = GGML_FP32_TO_FP16(fudge*(sumq2 > 0 ? sumqx/sumq2 : d)); } } @@ -2058,6 +2060,8 @@ void quantize_row_iq2_kl_impl(const float * x, void * vy, int n_per_row, const f float max_scale = 0, max_abs_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_KL); + for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { std::memset(&y[ibl], 0, sizeof(block_iq2_kl)); auto scales = all_scales + ibl*(QK_K/kBlockSize); @@ -2208,7 +2212,7 @@ void quantize_row_iq2_kl_impl(const float * x, void * vy, int n_per_row, const f } if (sumq2 > 0) d = sumqx/sumq2; - dptr[0] = GGML_FP32_TO_FP16(1.025f * d); + dptr[0] = GGML_FP32_TO_FP16(fudge * d); } } @@ -2305,6 +2309,8 @@ static void quantize_row_iq3_k_impl(const float * x, void * vy, int n_per_row, c const int8_t * shifted_values = iq3nl_values + 8; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_K); + for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq3_k)); @@ -2504,7 +2510,7 @@ static void quantize_row_iq3_k_impl(const float * x, void * vy, int n_per_row, c } } } - y[ibl].d = GGML_FP32_TO_FP16(1.01f*(sumq2 > 0 ? sumqx/sumq2 : d)); + y[ibl].d = GGML_FP32_TO_FP16(fudge*(sumq2 > 0 ? sumqx/sumq2 : d)); } } @@ -2600,6 +2606,8 @@ static void quantize_row_iq3_ks_impl(const int super_block_size, const int block float amax_scale = 0; float max_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_KS); + for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq3_ks)); const float * xbl = x + ibl*super_block_size; @@ -2743,7 +2751,7 @@ static void quantize_row_iq3_ks_impl(const int super_block_size, const int block } } } - if (sumq2 > 0) *dptr = GGML_FP32_TO_FP16(sumqx/sumq2); + if (sumq2 > 0) *dptr = GGML_FP32_TO_FP16(fudge*sumqx/sumq2); } } @@ -2935,6 +2943,8 @@ static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int b const int8_t * shifted_values = values + 16; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_K); + float max_scale = 0, amax_scale = 0; uint16_t extra = 0; for (int ib = 0; ib < super_block_size/block_size; ++ib) { @@ -3030,7 +3040,7 @@ static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int b } } float d = -max_scale/32; - y->d = GGML_FP32_TO_FP16(d); + y->d = GGML_FP32_TO_FP16(fudge*d); y->extra = extra; float id = d ? 1/d : 0.f; float sumqx = 0, sumq2 = 0; @@ -3062,7 +3072,7 @@ static void quantize_row_iq4_k_impl_bs16(const int super_block_size, const int b else y->scales_l[ib/2] |= (l_l << 4); scales_h[ib/8] |= (l_h << 2*(ib%8)); } - if (sumq2 > 0) y->d = GGML_FP32_TO_FP16(sumqx/sumq2); + if (sumq2 > 0) y->d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2); for (int i = 0; i < super_block_size/32; ++i) { for (int j = 0; j < 16; ++j) { @@ -3245,6 +3255,8 @@ void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const fl const int8_t * shifted_values = iq5nl_values + 32; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ5_K); + for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq5_k)); @@ -3369,7 +3381,7 @@ void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const fl if (!max_abs_scale) continue; float d = -max_scale/32; - y[ibl].d = GGML_FP32_TO_FP16(d); + y[ibl].d = GGML_FP32_TO_FP16(fudge*d); y[ibl].extra = extra; float id = 1/d; @@ -3408,7 +3420,7 @@ void quantize_row_iq5_k_impl(const float * x, void * vy, int n_per_row, const fl } } } - if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2); + if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2); } @@ -3600,6 +3612,8 @@ void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const fl float scales[QK_K/16]; float weight[16]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ6_K); + for (int ibl = 0; ibl < n_per_row/QK_K; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq6_k)); @@ -3731,7 +3745,7 @@ void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const fl if (!max_abs_scale) continue; float d = -max_scale/127; - y[ibl].d = GGML_FP32_TO_FP16(d); + y[ibl].d = GGML_FP32_TO_FP16(fudge*d); y[ibl].extra = extra; float id = 1/d; @@ -3769,7 +3783,7 @@ void quantize_row_iq6_k_impl(const float * x, void * vy, int n_per_row, const fl } } } - if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(sumqx/sumq2); + if (sumq2 > 0) y[ibl].d = GGML_FP32_TO_FP16(fudge*sumqx/sumq2); } } @@ -4382,6 +4396,8 @@ static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int float amax_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_KS); + for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq4_ks)); const float * xbl = x + ibl*super_block_size; @@ -4480,7 +4496,7 @@ static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int } } float d = amax_scale/127; - *dptr = d; + *dptr = fudge*d; if (!d) return; float id = d ? 1/d : 0.f; float sumqx = 0, sumq2 = 0; @@ -4524,7 +4540,7 @@ static void quantize_row_iq4_k_impl_bs128(const int super_block_size, const int } } //printf("rmse = %g\n", sqrt(mse/n_per_row)); - if (sumq2 > 0) *dptr = sumqx/sumq2; + if (sumq2 > 0) *dptr = fudge*sumqx/sumq2; } } @@ -4633,6 +4649,8 @@ static void quantize_row_iq5_ks_impl(const int super_block_size, const int block float amax_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ5_KS); + for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq5_ks)); const float * xbl = x + ibl*super_block_size; @@ -4731,7 +4749,7 @@ static void quantize_row_iq5_ks_impl(const int super_block_size, const int block } } float d = amax_scale/127; - *dptr = d; + *dptr = fudge*d; if (!d) return; float id = d ? 1/d : 0.f; float sumqx = 0, sumq2 = 0; @@ -4767,7 +4785,7 @@ static void quantize_row_iq5_ks_impl(const int super_block_size, const int block } } } - if (sumq2 > 0) *dptr = sumqx/sumq2; + if (sumq2 > 0) *dptr = fudge*sumqx/sumq2; } } @@ -4938,6 +4956,8 @@ static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy, float amax_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_KSS); + for (int ibl = 0; ibl < n_per_row/super_block_size; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq4_kss)); const float * xbl = x + ibl*super_block_size; @@ -5053,7 +5073,7 @@ static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy, } } float d = amax_scale/127; - *dptr = d; + *dptr = fudge*d; if (!d) return; float id = 1/d; float sumqx = 0, sumq2 = 0; @@ -5129,7 +5149,7 @@ static void quantize_row_iq4_kss_impl(int n_per_row, const float * x, char * cy, } } } - if (sumq2 > 0) *dptr = sumqx/sumq2 * 1.01f; + if (sumq2 > 0) *dptr = fudge*sumqx/sumq2; } } @@ -9366,6 +9386,8 @@ void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const f } } + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ1_KT); + float id = d ? 1/d : 0.f; for (int ibl = 0; ibl < nblock; ++ibl) { auto scales = all_scales + ibl*Q::kNblock; @@ -9375,7 +9397,7 @@ void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const f } } - *dptr = d; + *dptr = fudge*d; if (!d) return; for (int iloop = 0; iloop < 1; ++iloop) { @@ -9429,7 +9451,7 @@ void quantize_row_iq1_kt_impl(const float * x, void * vy, int n_per_row, const f } if (sumq2 > 0) { d = sumqx/sumq2; - *dptr = d * 1.07f; + *dptr = d * fudge; if (!d) return; } else { break; @@ -9545,6 +9567,8 @@ void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const f float amax_scale = 0, max_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ2_KT); + for (int ibl = 0; ibl < nblock; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq2_kt)); @@ -9632,7 +9656,7 @@ void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const f } } - *dptr = d; + *dptr = d * fudge; if (!d) return; for (int iloop = 0; iloop < 1; ++iloop) { @@ -9686,7 +9710,7 @@ void quantize_row_iq2_kt_impl(const float * x, void * vy, int n_per_row, const f } if (sumq2 > 0) { d = sumqx/sumq2; - *dptr = d; + *dptr = d * fudge; if (!d) return; } else { break; @@ -9841,6 +9865,8 @@ void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const f float xaux[Q::kBlockSize]; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ3_KT); + for (int ibl = 0; ibl < nblock; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq3_kt)); @@ -9945,7 +9971,7 @@ void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const f } } - *dptr = d; + *dptr = d * fudge; for (int iloop = 0; iloop < 1; ++iloop) { @@ -9983,7 +10009,7 @@ void quantize_row_iq3_kt_impl(const float * x, void * vy, int n_per_row, const f } if (sumq2 > 0) { d = sumqx/sumq2; - *dptr = d; + *dptr = d * fudge; if (!d) break; } else { break; @@ -10133,6 +10159,8 @@ void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const f float amax_scale = 0, max_scale = 0; + const float fudge = ggml_get_quantize_fudge_factor(GGML_TYPE_IQ4_KT); + for (int ibl = 0; ibl < nblock; ++ibl) { memset(&y[ibl], 0, sizeof(block_iq4_kt)); @@ -10199,7 +10227,7 @@ void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const f float d = -max_scale/64; - dptr[0] = d; + dptr[0] = d * fudge; if (!d) return; constexpr int kNumGroups = Q::kSuperBlockSize/Q::kGroupSize; @@ -10248,7 +10276,7 @@ void quantize_row_iq4_kt_impl(const float * x, void * vy, int n_per_row, const f } if (sumq2 > 0) { d = sumqx/sumq2; - dptr[0] = d; + dptr[0] = d * fudge; if (!d) break; } else { break;