From 37d82c23132c55584e7b3978fe803cff7edeb932 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Thu, 13 Aug 2026 17:47:37 +0200 Subject: [PATCH] Change the default amb value from 0 to 256 (#2312) --- common/common.h | 2 +- src/llama.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/common.h b/common/common.h index 086d98d6..3bb51367 100644 --- a/common/common.h +++ b/common/common.h @@ -421,7 +421,7 @@ struct gpt_params { bool cont_batching = true; // insert new sequences for decoding on-the-fly bool flash_attn = true; // flash attention int mla_attn = 3; // MLA 0: standard, 1: MLA with K and V^T cache, 2: MLA with just K cache, 3: the best of both worlds - int attn_max_batch = 0; // Max batch size to use when computing attention (only applicable if flash_attn = false) + int attn_max_batch = 256; // Max batch size to use when computing attention (only applicable if flash_attn = false) bool fused_moe_up_gate = true; // fused up*unary(gate) op for MoE models bool fused_up_gate = true; // fused up*unary(gate) op bool fused_mmad = true; // fused mul+multi_add op diff --git a/src/llama.cpp b/src/llama.cpp index 61831a38..4e477a55 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -7520,7 +7520,7 @@ struct llama_model_params llama_model_default_params() { /*.max_ctx_size =*/ 0, /*.n_seq_max =*/ 1, /*.n_ubatch =*/ 512, - /*.amb =*/ 0, + /*.amb =*/ 256, /*.fit_margin =*/ 0, /*.fit =*/ false, /*.worst_graph_tokens =*/ 0, @@ -7606,7 +7606,7 @@ struct llama_context_params llama_context_default_params() { /*.offload_kqv =*/ true, /*.flash_attn =*/ true, /*.mla_attn =*/ 3, - /*.attn_max_batch =*/ 0, + /*.attn_max_batch =*/ 256, /*.fused_moe_up_gate =*/ true, /*.grouped_expert_routing =*/ false, /*.fused_up_gate =*/ true,