nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

qwen4exp: PP/TG opts for Ampere+Zen3 (merged experts, fused PLE taps, Zen3 tiling)

This commit is contained in:
Marvin 2026-09-05 20:14:29 -03:00
parent fe215a8ccd
commit e2728c8547
5 changed files with 27 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ggml/src/ggml.c
View File

@ -18324,7 +18324,9 @@ static void ggml_compute_forward_mul_mat_id(
const void * wdata_mm = (src1->type == vec_dot_type) ? src1->data : params->wdata;
const size_t row_size_mm = ggml_row_size(vec_dot_type, ne10);
const int chunks_per_expert = MAX(1, MIN(nth, (int)(ne01 / 32)));
// 64 for Zen3 5900XT (2-CCX): halves atomics/barriers vs /32 on narrow
// qwen4exp experts; keeps MIN(nth,...) cap so TG load balance is safe.
const int chunks_per_expert = MAX(1, MIN(nth, (int)(ne01 / 64)));
int total_chunks = 0;
for (int a = 0; a < n_as; a++) {
@ -18655,7 +18657,8 @@ static void ggml_compute_forward_mul_mat_id_up_gate(
const size_t row_size_ug = ggml_row_size(vec_dot_type, ne10);
const int64_t nr0_base = src0_2 ? ne01 : ne01/2;
const int chunks_per_expert_ug = MAX(1, MIN(nth, (int)(nr0_base / 32)));
// See above: /64 for Zen3 to cut scheduling overhead on PP-heavy MoE.
const int chunks_per_expert_ug = MAX(1, MIN(nth, (int)(nr0_base / 64)));
int total_chunks_ug = 0;
for (int a = 0; a < n_as; a++) {

6
ggml/src/iqk/iqk_gemm_kquants.cpp
View File

@ -646,6 +646,12 @@ static void mul_mat_qX_K_q8_K_T(int n, const void * vx, size_t bx, const DataInf
for (int i = 0; i < nb; ++i) {
// DDR4 latency hiding on Zen3 TG: weight stream is ~270B/block,
// prefetch 4 blocks ahead (~1KB) stays in L1 streamer window.
if (i + 4 < nb) {
__builtin_prefetch(deq.x + i + 4, 0, 3);
}
auto all_scales = deq.new_block(i, q8, accd);
__m256i sumi[nrc_y];

7
ggml/src/iqk/iqk_mul_mat.cpp
View File

@ -61,7 +61,9 @@ struct MulMat {
#ifdef __aarch64__
constexpr int k_x_step = 64; //8192; // Tiling does not seem to help on my M2 Max (but difference to tiling is small)
#else
constexpr int k_x_step = 64; // This works best on my Ryzen-7950X (but differences to other tile size are small)
// 32 for Zen3 (2x32MB L3, DDR4): 64x8x256xf32 reuse set thrashes CCX on 5900XT.
// Glove-fit for znver3-only builds; Zen4/Intel still fine at 32 (small diff).
constexpr int k_x_step = 32;
#endif
if (func16 && nrc_y >= 16) {
int n_step = (nrc_y - info.cur_y)/16;
@ -139,7 +141,8 @@ struct MulMat {
#ifdef __aarch64__
constexpr int k_x_step = 64; //8192; // Tiling does not seem to help on my M2 Max (but difference to tiling is small)
#else
constexpr int k_x_step = 64; // This works best on my Ryzen-7950X (but differences to other tile size are small)
// See mul_mat_NxM above: 32 for Zen3 CCX locality.
constexpr int k_x_step = 32;
#endif
auto op = ggml_unary_op(unary_op);
float tmp[k_x_step*16];

7
src/graphs/build_qwen4exp.cpp
View File

@ -135,13 +135,14 @@ static ggml_tensor * qwen4exp_ple_conv(
ggml_view_2d(ctx0, model.layers[il].ple_conv1d, 1, hc_dim,
model.layers[il].ple_conv1d->nb[1],
k * model.layers[il].ple_conv1d->nb[0]));
wk = ggml_reshape_1d(ctx0, wk, hc_dim);
// [1, hc_dim] broadcasts over rows: same math as transpose-mul-transpose
// but saves 2 transposes + 2 conts per tap (2*K nodes per PLE layer).
wk = ggml_reshape_2d(ctx0, wk, 1, hc_dim);
if (wk->type != GGML_TYPE_F32) {
wk = ggml_cast(ctx0, wk, GGML_TYPE_F32);
}
ggml_tensor * term = ggml_mul(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, shifted)), wk);
term = ggml_cont(ctx0, ggml_transpose(ctx0, term)); // [n_tokens, hc_dim]
ggml_tensor * term = ggml_mul(ctx0, shifted, wk); // [n_tokens, hc_dim]
conv_out = conv_out ? ggml_add(ctx0, conv_out, term) : term;
}

9
src/llama-load-tensors.cpp
View File

@ -4820,7 +4820,11 @@ bool create_tensors_helper::create_std_ffn_exps(int64_t n_embd, const LLM_TN & t
if (ug_meta) {
layer.ffn_up_gate_exps = create_tensor(ffn_ctx, ug_name, { n_embd, 2*n_ff_exp, n_expert}, flags);
} else {
merged = flags == 0 && ml.merge_up_gate_exps && merge_up_gate_exps(tn, i, 0);
// qwen4exp (Qwen3.8-Flash-Next): merge up/gate by default for fused MoE
// (halves expert GEMM launches on both CUDA mmq_id and CPU IQK paths).
// Falls back gracefully if types/shapes differ inside merge_up_gate_exps().
const bool want_merge = ml.merge_up_gate_exps || model.arch == LLM_ARCH_QWEN4EXP;
merged = flags == 0 && want_merge && merge_up_gate_exps(tn, i, 0);
if (!merged) {
layer.ffn_up_exps = create_tensor(ffn_ctx, tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, flags);
layer.ffn_gate_exps = create_tensor(ffn_ctx, tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, flags);
@ -4843,7 +4847,8 @@ bool create_tensors_helper::create_std_ffn_exps_from_meta(const LLM_TN & tn, int
if (ug_meta) {
layer.ffn_up_gate_exps = create_tensor(ffn_ctx, ug_name, { ug_meta->ne[0], ug_meta->ne[1], ug_meta->ne[2]}, flags);
} else {
merged = flags == 0 && ml.merge_up_gate_exps && merge_up_gate_exps(tn, i, 0);
const bool want_merge = ml.merge_up_gate_exps || model.arch == LLM_ARCH_QWEN4EXP;
merged = flags == 0 && want_merge && merge_up_gate_exps(tn, i, 0);
if (!merged) {
auto u_name = tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i);
auto g_name = tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i);