From 6e7378f6167b120057137dc411bbd2590c5d7729 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Thu, 13 Aug 2026 15:25:28 +0200 Subject: [PATCH] Another minor optimization on CUDA for split mode graph (#2298) * CUDA: fuse rms -> add -> rms * Another minor optimization on CUDA for split mode graph --- ggml/src/ggml-cuda.cu | 2 +- src/llama-build-context.cpp | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu index d68ccefd..232f765a 100644 --- a/ggml/src/ggml-cuda.cu +++ b/ggml/src/ggml-cuda.cu @@ -3934,7 +3934,7 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg i += 4; } else if (fusion && i + 2 < cgraph->n_nodes && - cgraph->nodes[i+1]->op == GGML_OP_VIEW && + (cgraph->nodes[i+1]->op == GGML_OP_VIEW || cgraph->nodes[i+1]->op == GGML_OP_RESHAPE) && cgraph->nodes[i+2]->op == GGML_OP_FUSED_RMS_NORM && dst->ne[2] == 1 && cgraph->nodes[i+2]->ne[2] == 1) { ggml_cuda_op_fused_rms_rms_norm(ctx, dst, cgraph->nodes[i+2]); diff --git a/src/llama-build-context.cpp b/src/llama-build-context.cpp index 7e2b8a7d..18a4c905 100644 --- a/src/llama-build-context.cpp +++ b/src/llama-build-context.cpp @@ -3175,6 +3175,8 @@ ggml_tensor * llm_build_context::build_std_attention(ggml_cgraph * gf, ggml_tens cb(Vcur, "Vcur_scales", il_cb); } } + ggml_build_forward_expand(gf, Qcur); + ggml_build_forward_expand(gf, Kcur); auto rope_factors = rope_factors_in; if (rope_factors) { GGML_ASSERT(rope_factors->extra); @@ -3200,6 +3202,8 @@ ggml_tensor * llm_build_context::build_std_attention(ggml_cgraph * gf, ggml_tens ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l); + ggml_build_forward_expand(gf, Qcur); + ggml_build_forward_expand(gf, Kcur); } } cb(Qcur, "Qcur", il_cb);