From c49f7db34aacd8374b4321cd1998acd785ca38b9 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Tue, 25 Aug 2026 13:44:32 +0200 Subject: [PATCH] Fix MMQ check when quant does not support MMQ (#2356) --- ggml/src/ggml-cuda.cu | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu index 9dd74fd5..aaf1b73b 100644 --- a/ggml/src/ggml-cuda.cu +++ b/ggml/src/ggml-cuda.cu @@ -2577,7 +2577,8 @@ static int ggml_cuda_mul_mat_q(ggml_backend_cuda_context & ctx, const ggml_tenso || dst->op == GGML_OP_PERMUTE || dst->op == GGML_OP_NONE) { ++node_n; continue; } - if (dst->op != GGML_OP_MUL_MAT || dst->src[1] != src1 || !ggml_is_quantized(dst->src[0]->type)) break; + if (dst->op != GGML_OP_MUL_MAT || dst->src[1] != src1 || !ggml_is_quantized(dst->src[0]->type) || + !ggml_cuda_should_use_mmq(dst->src[0]->type, ggml_cuda_info().devices[ctx.device].cc, src1->ne[1])) break; if (!is_gemv && mmq_get_q8_1_ds_layout(src0->type) != mmq_get_q8_1_ds_layout(dst->src[0]->type)) break; if (is_gemv) { if (fusion && node_n + 2 < cgraph->n_nodes &&