nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Another minor optimization on CUDA for split mode graph (#2298) 

* CUDA: fuse rms -> add -> rms

* Another minor optimization on CUDA for split mode graph
This commit is contained in:
Kawrakow 2026-08-13 15:25:28 +02:00 committed by GitHub
parent 8b276c08ef
commit 6e7378f616
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ggml/src/ggml-cuda.cu
View File

@ -3934,7 +3934,7 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
i += 4;
}
else if (fusion && i + 2 < cgraph->n_nodes &&
cgraph->nodes[i+1]->op == GGML_OP_VIEW &&
(cgraph->nodes[i+1]->op == GGML_OP_VIEW || cgraph->nodes[i+1]->op == GGML_OP_RESHAPE) &&
cgraph->nodes[i+2]->op == GGML_OP_FUSED_RMS_NORM &&
dst->ne[2] == 1 && cgraph->nodes[i+2]->ne[2] == 1) {
ggml_cuda_op_fused_rms_rms_norm(ctx, dst, cgraph->nodes[i+2]);

4
src/llama-build-context.cpp
View File

@ -3175,6 +3175,8 @@ ggml_tensor * llm_build_context::build_std_attention(ggml_cgraph * gf, ggml_tens
cb(Vcur, "Vcur_scales", il_cb);
}
}
ggml_build_forward_expand(gf, Qcur);
ggml_build_forward_expand(gf, Kcur);
auto rope_factors = rope_factors_in;
if (rope_factors) {
GGML_ASSERT(rope_factors->extra);
@ -3200,6 +3202,8 @@ ggml_tensor * llm_build_context::build_std_attention(ggml_cgraph * gf, ggml_tens
ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,
ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
ggml_build_forward_expand(gf, Qcur);
ggml_build_forward_expand(gf, Kcur);
}
}
cb(Qcur, "Qcur", il_cb);