nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allow concatenating quantized tensors (#2232) 

* Allow concatenating quantized tensors

* Missed this assert

* Allow K to be f32 in ggml_cuda_op_indexer_topk
This commit is contained in:
Kawrakow 2026-08-03 08:01:57 +03:00 committed by GitHub
parent c3b075f069
commit bee545824a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ggml/src/ggml-cuda/indexer_topk.cu
View File

@ -69,7 +69,8 @@ void ggml_cuda_op_indexer_topk(ggml_backend_cuda_context & ctx, ggml_tensor * ds
int n_top_k = dst->ne[0]; int n_top_k = dst->ne[0];
int n_kv = k->ne[1]; int n_kv = k->ne[1];
int n_head = q->ne[1]; int n_head = q->ne[1];
GGML_ASSERT(k->type == GGML_TYPE_F16 || ggml_is_quantized(k->type)); //if (k->type != GGML_TYPE_F16 && !ggml_is_quantized(k->type)) printf("%s: K is %s?\n", __func__, ggml_type_name(k->type));
GGML_ASSERT(k->type == GGML_TYPE_F16 || k->type == GGML_TYPE_F32 || ggml_is_quantized(k->type));
GGML_ASSERT(k->ne[2] == 1 || k->ne[3] == 1); GGML_ASSERT(k->ne[2] == 1 || k->ne[3] == 1);
GGML_ASSERT(k->ne[1] > n_top_k); GGML_ASSERT(k->ne[1] > n_top_k);
GGML_ASSERT(k->ne[1] == m->ne[0]); GGML_ASSERT(k->ne[1] == m->ne[0]);

9
ggml/src/ggml.c
View File

@ -15469,10 +15469,13 @@ static bool ggml_compute_forward_concat_any_opt(
const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1]; const struct ggml_tensor * src1 = dst->src[1];
if (ggml_is_quantized(src0->type)) return false; if (ggml_is_quantized(src0->type)) {
size_t row_meta = type_traits[src0->type].row_meta_size;
if (row_meta > 0) return false; // We cannot concatenate quants that has per row meta data
}
//if (ggml_is_quantized(src0->type)) return false;
GGML_ASSERT(src0->type == src1->type && src0->type == dst->type); GGML_ASSERT(src0->type == src1->type && src0->type == dst->type);
GGML_ASSERT(!ggml_is_quantized(src0->type));
const int ith = params->ith; const int ith = params->ith;
const int nth = params->nth; const int nth = params->nth;
@ -15521,7 +15524,7 @@ static bool ggml_compute_forward_concat_any_opt(
if (d > 0) nrows *= dst->ne[d]; if (d > 0) nrows *= dst->ne[d];
} }
size_t row_size = ggml_row_size(dst->type, dst->ne[0]); size_t row_size = ggml_row_size(dst->type, dst->ne[0]);
if (src0->nb[1] == row_size && src1->nb[1] == row_size) { if (src0->nb[1] >= row_size && src1->nb[1] >= row_size) {
int npt = (nrows + nth - 1)/nth; int npt = (nrows + nth - 1)/nth;
int first = ith*npt; int first = ith*npt;
int last = MIN(first + npt, nrows); int last = MIN(first + npt, nrows);