nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change the default amb value from 0 to 256 (#2312)

This commit is contained in:
Kawrakow 2026-08-13 17:47:37 +02:00 committed by GitHub
parent 6e7378f616
commit 37d82c2313
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common/common.h
View File

@ -421,7 +421,7 @@ struct gpt_params {
bool cont_batching = true; // insert new sequences for decoding on-the-fly
bool flash_attn = true; // flash attention
int mla_attn = 3; // MLA 0: standard, 1: MLA with K and V^T cache, 2: MLA with just K cache, 3: the best of both worlds
int attn_max_batch = 0; // Max batch size to use when computing attention (only applicable if flash_attn = false)
int attn_max_batch = 256; // Max batch size to use when computing attention (only applicable if flash_attn = false)
bool fused_moe_up_gate = true; // fused up*unary(gate) op for MoE models
bool fused_up_gate = true; // fused up*unary(gate) op
bool fused_mmad = true; // fused mul+multi_add op

4
src/llama.cpp
View File

@ -7520,7 +7520,7 @@ struct llama_model_params llama_model_default_params() {
/*.max_ctx_size =*/ 0,
/*.n_seq_max =*/ 1,
/*.n_ubatch =*/ 512,
/*.amb =*/ 0,
/*.amb =*/ 256,
/*.fit_margin =*/ 0,
/*.fit =*/ false,
/*.worst_graph_tokens =*/ 0,
@ -7606,7 +7606,7 @@ struct llama_context_params llama_context_default_params() {
/*.offload_kqv =*/ true,
/*.flash_attn =*/ true,
/*.mla_attn =*/ 3,
/*.attn_max_batch =*/ 0,
/*.attn_max_batch =*/ 256,
/*.fused_moe_up_gate =*/ true,
/*.grouped_expert_routing =*/ false,
/*.fused_up_gate =*/ true,