From bee545824a20f3b7b486a6544b360f54eec6a287 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Mon, 3 Aug 2026 08:01:57 +0300 Subject: [PATCH] Allow concatenating quantized tensors (#2232) * Allow concatenating quantized tensors * Missed this assert * Allow K to be f32 in ggml_cuda_op_indexer_topk --- ggml/src/ggml-cuda/indexer_topk.cu | 3 ++- ggml/src/ggml.c | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cuda/indexer_topk.cu b/ggml/src/ggml-cuda/indexer_topk.cu index 523ff556..0bdd7ea8 100644 --- a/ggml/src/ggml-cuda/indexer_topk.cu +++ b/ggml/src/ggml-cuda/indexer_topk.cu @@ -69,7 +69,8 @@ void ggml_cuda_op_indexer_topk(ggml_backend_cuda_context & ctx, ggml_tensor * ds int n_top_k = dst->ne[0]; int n_kv = k->ne[1]; int n_head = q->ne[1]; - GGML_ASSERT(k->type == GGML_TYPE_F16 || ggml_is_quantized(k->type)); + //if (k->type != GGML_TYPE_F16 && !ggml_is_quantized(k->type)) printf("%s: K is %s?\n", __func__, ggml_type_name(k->type)); + GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32 || ggml_is_quantized(k->type)); GGML_ASSERT(k->ne[2] == 1 || k->ne[3] == 1); GGML_ASSERT(k->ne[1] > n_top_k); GGML_ASSERT(k->ne[1] == m->ne[0]); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 00c83a5a..0cba5310 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -15469,10 +15469,13 @@ static bool ggml_compute_forward_concat_any_opt( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - if (ggml_is_quantized(src0->type)) return false; + if (ggml_is_quantized(src0->type)) { + size_t row_meta = type_traits[src0->type].row_meta_size; + if (row_meta > 0) return false; // We cannot concatenate quants that has per row meta data + } + //if (ggml_is_quantized(src0->type)) return false; GGML_ASSERT(src0->type == src1->type && src0->type == dst->type); - GGML_ASSERT(!ggml_is_quantized(src0->type)); const int ith = params->ith; const int nth = params->nth; @@ -15521,7 +15524,7 @@ static bool ggml_compute_forward_concat_any_opt( if (d > 0) nrows *= dst->ne[d]; } size_t row_size = ggml_row_size(dst->type, dst->ne[0]); - if (src0->nb[1] == row_size && src1->nb[1] == row_size) { + if (src0->nb[1] >= row_size && src1->nb[1] >= row_size) { int npt = (nrows + nth - 1)/nth; int first = ith*npt; int last = MIN(first + npt, nrows);