From ab6d81681827fc9d4afa3cfea5e862f8a007e488 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 31 Aug 2026 19:35:17 +0300 Subject: [PATCH] CUDA DSA: fix v_offset for quantized K/V caches (#2387) v_offset is used as a column index into the dequantized f16 buffer, but was computed as a half-pointer difference. That is only correct for f16 K/V. With q8_0 (34 bytes per 32 elements) it yields the wrong column, so the sparse attention path reads V from the wrong positions and generation degenerates. --- ggml/src/ggml-cuda/dsa_attn.cu | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda/dsa_attn.cu b/ggml/src/ggml-cuda/dsa_attn.cu index 0a0f7ec5..f7db91af 100644 --- a/ggml/src/ggml-cuda/dsa_attn.cu +++ b/ggml/src/ggml-cuda/dsa_attn.cu @@ -296,7 +296,9 @@ bool ggml_cuda_dsa_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) ggml_cuda_pool_alloc v16(ctx.pool()); size_t v_offset = 0; if (is_k_view) { - v_offset = (const half *)V->data - (const half *)K->data; + // Using /sizeof(half) is only correct for f16 K/V, for a quantized cache(i.e. q8_0) it yields the wrong column + const size_t v_byte_off = (const char *)V->data - (const char *)K->data; + v_offset = v_byte_off / ggml_type_size(K->type) * ggml_blck_size(K->type); } else { v16.alloc(v_cache_size); }