diff --git a/examples/quantize/quantize.cpp b/examples/quantize/quantize.cpp index 694b70cf..51a66a8d 100644 --- a/examples/quantize/quantize.cpp +++ b/examples/quantize/quantize.cpp @@ -151,12 +151,14 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp // [[noreturn]] static void usage(const char * executable) { - printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); + printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--hide-imatrix] [--ignore-imatrix-rules] [--dry-run] [--include-weights] [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--ffn-gate-inp-type] [--attn-q-type] [--attn-k-type] [--attn-v-type] [--attn-qkv-type] [--attn-output-type] [--ffn-gate-type] [--ffn-down-type] [--ffn-up-type] [--repack] [--repack-pattern] [--keep-split] [--partial-requant] [--override-kv] model-f32.gguf [model-quant.gguf] type [nthreads]\n\n", executable); printf(" --allow-requantize: Allows requantizing tensors that have already been quantized. Warning: This can severely reduce quality compared to quantizing from 16bit or 32bit\n"); printf(" --leave-output-tensor: Will leave output.weight un(re)quantized. Increases model size but may also increase quality, especially when requantizing\n"); printf(" --pure: Disable k-quant mixtures and quantize all tensors to the same type\n"); printf(" --imatrix file_name: use data in file_name as importance matrix for quant optimizations\n"); printf(" --hide-imatrix: do not store imatrix details in the quantized model\n"); + printf(" --ignore-imatrix-rules: ignore importance matrix rules when quantizing\n"); + printf(" --dry-run: show what would be quantized without actually writing the output file\n"); printf(" --include-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --exclude-weights tensor_name: use importance matrix for this/these tensor(s)\n"); printf(" --output-tensor-type ggml_type: use this ggml_type for the output.weight tensor.\n"); diff --git a/ggml/src/iqk/iqk_flash_attn.cpp b/ggml/src/iqk/iqk_flash_attn.cpp index 02509095..c0ece49d 100644 --- a/ggml/src/iqk/iqk_flash_attn.cpp +++ b/ggml/src/iqk/iqk_flash_attn.cpp @@ -169,15 +169,19 @@ extern "C" IQK_API bool iqk_flash_attn_noalibi(int type_q, int type_mask, float if (auto type_k = ggml_type(int_type_k_in), type_v = ggml_type(int_type_v); !are_kv_types_supported(type_k, type_v)) { if (ith == 0) { - fprintf(stderr, "\n==================== KV cache types %s, %s are not supported on the CPU\n", + fprintf(stderr, "\n==================== K cache %s coupled with V cache %s is not a supported combination on the CPU backend.\n", ggml_type_name(type_k), ggml_type_name(type_v)); auto & supported = supported_kv_types(); - fprintf(stderr, "Sopprted types are:\n"); + fprintf(stderr, "Supported types are:\n"); for (auto type : supported) { fprintf(stderr, " %s\n", ggml_type_name(type)); } + fprintf(stderr, " Warning: ik_llama.cpp does not support Q5_0 or Q5_1 KV cache on the CPU.\n"); #ifdef __AVX512BF16__ fprintf(stderr, " %s, but only if K and V are both %s\n", ggml_type_name(GGML_TYPE_BF16), ggml_type_name(GGML_TYPE_BF16)); +#endif +#ifndef GGML_IQK_FA_ALL_QUANTS + fprintf(stderr, " To enable q4_0, q4_1, and iq4_nl KV cache types, recompile with -DGGML_IQK_FA_ALL_QUANTS=ON\n"); #endif } barrier(barrier_data); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 250e749f..3bf2e1ec 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -72,3 +72,7 @@ if (BUILD_SHARED_LIBS) set_target_properties(llama PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_definitions(llama PRIVATE LLAMA_SHARED LLAMA_BUILD) endif() + +if (GGML_MAX_CONTEXTS) + add_compile_definitions(GGML_MAX_CONTEXTS=${GGML_MAX_CONTEXTS}) +endif()