diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 549a6127..a0857907 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -16865,17 +16865,6 @@ static int ggml_compute_forward_mul_mat( // compute by src0 rows #if GGML_USE_IQK_MULMAT - if (ith == 0) { - static bool first_time = true; - if (first_time) { - first_time = false; -#ifdef HAVE_FANCY_SIMD - printf("======================================= HAVE_FANCY_SIMD is defined\n"); -#else - printf("======================================= HAVE_FANCY_SIMD is NOT defined\n"); -#endif - } - } if (dst->type == GGML_TYPE_F32) { if (iqk_mul_mat_4d(ne01, ne11, ne00, ne02, ne03, ne12, ne13, nb02, nb03, nb12, nb13, nb2/sizeof(float), nb3/sizeof(float), diff --git a/ggml/src/iqk/iqk_cpu_ops.cpp b/ggml/src/iqk/iqk_cpu_ops.cpp index c5ca7ece..4a0e8eee 100644 --- a/ggml/src/iqk/iqk_cpu_ops.cpp +++ b/ggml/src/iqk/iqk_cpu_ops.cpp @@ -22,6 +22,14 @@ #include #endif +bool iqk_has_fancy_simd(void) { +#ifdef HAVE_FANCY_SIMD + return true; +#else + return false; +#endif +} + namespace { // Playing around with group scores: use sum of probabilities in the group inline float group_score(int n_per_group, const float * data) { diff --git a/ggml/src/iqk/iqk_cpu_ops.h b/ggml/src/iqk/iqk_cpu_ops.h index 4b98d885..cd3c641e 100644 --- a/ggml/src/iqk/iqk_cpu_ops.h +++ b/ggml/src/iqk/iqk_cpu_ops.h @@ -14,6 +14,8 @@ extern "C" { struct ggml_tensor; +bool iqk_has_fancy_simd(void); + void iqk_sumrows_div(struct ggml_tensor * div, int ith, int nth); void iqk_grouped_top_k(struct ggml_tensor * dst, int ith, int nth); diff --git a/src/llama.cpp b/src/llama.cpp index 1ac45444..e9ce56ef 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -25,8 +25,9 @@ #include "ggml-alloc.h" #include "ggml-backend.h" -// TODO: fix this include +// TODO: fix these includes #include "iqk/iqk_quantize.h" +#include "iqk/iqk_cpu_ops.h" #define IK_PRINT_TIMING 0 @@ -2082,6 +2083,12 @@ static bool llm_load_tensors( } } + if (iqk_has_fancy_simd()) { + LLAMA_LOG_INFO("======================================= HAVE_FANCY_SIMD is defined\n"); + } else { + LLAMA_LOG_INFO("======================================= HAVE_FANCY_SIMD is NOT defined\n"); + } + model.split_mode = split_mode; model.main_gpu = main_gpu; model.max_gpu = max_gpu; @@ -4713,7 +4720,7 @@ struct llama_model * llama_model_load_from_file( if (has_rpc) { for (auto& it : model->rpc_servers) { device_names.push_back(create_rpc_name(it.endpoint, it.device)); - } + } } device_names.insert(device_names.end(), gpu_names.begin(), gpu_names.end()); } @@ -4726,7 +4733,6 @@ struct llama_model * llama_model_load_from_file( } } - // no gpu used, so set layers offload to be 0 if (!model->devices.size()) { params.n_gpu_layers = 0; @@ -4737,7 +4743,7 @@ struct llama_model * llama_model_load_from_file( const char* name = ggml_backend_buft_name(buft); const char* description = name; size_t description_size = llama_get_device_memory(*model, i); - LLAMA_LOG_INFO("%s: using device %s - %zu MiB free\n", + LLAMA_LOG_INFO("%s: using device %s - %zu MiB free\n", name, description, description_size / 1024 / 1024); }