nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

model: add Cohere2-MoE North Mini Code support (#1945) 

* Add Cohere2 MoE North Mini Code support

* Fix Cohere2 MoE expert tensor emission

* Enhance Cohere2-MoE support by modifying tensor handling and configuration logic

* Fix Cohere2-MoE graph split reduce handling

---------

Co-authored-by: Joel Farthing <262452229+joelfarthing@users.noreply.github.com>
This commit is contained in:
Joel Farthing 2026-06-10 08:28:27 -05:00 committed by GitHub
parent e6f8112f3b
commit 4a1e2eaa69
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 289 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
convert_hf_to_gguf.py
View File

@ -594,6 +594,9 @@ class Model:
if chkhsh == "9c2227e4dd922002fb81bde4fc02b0483ca4f12911410dee2255e4987644e3f8":
# ref: https://huggingface.co/CohereForAI/c4ai-command-r-v01
res = "command-r"
if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e":
# ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0
res = "cohere2_moe"
if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea":
# ref: https://huggingface.co/Qwen/Qwen1.5-7B
res = "qwen2"
@ -1561,7 +1564,12 @@ class LlamaModel(Model):
special_vocab.add_to_gguf(self.gguf_writer)
def set_gguf_parameters(self):
saved_intermediate_size = self.hparams.get("intermediate_size")
saved_num_experts_per_tok = self.hparams.pop("num_experts_per_tok")
self.hparams["intermediate_size"] = self.hparams["prefix_dense_intermediate_size"]
super().set_gguf_parameters()
self.hparams["intermediate_size"] = saved_intermediate_size
self.hparams["num_experts_per_tok"] = saved_num_experts_per_tok
hparams = self.hparams
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
@ -3692,6 +3700,86 @@ class CommandR2Model(Model):
self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
@Model.register("Cohere2MoeForCausalLM")
class Cohere2MoeModel(Model):
model_arch = gguf.MODEL_ARCH.COHERE2_MOE
_experts: list[dict[str, Tensor]] | None = None
def set_gguf_parameters(self):
saved_intermediate_size = self.hparams["intermediate_size"]
saved_num_experts_per_tok = self.hparams.pop("num_experts_per_tok")
self.hparams["intermediate_size"] = self.hparams["prefix_dense_intermediate_size"]
super().set_gguf_parameters()
self.hparams["intermediate_size"] = saved_intermediate_size
self.hparams["num_experts_per_tok"] = saved_num_experts_per_tok
hparams = self.hparams
self.gguf_writer.add_vocab_size(hparams["vocab_size"])
self.gguf_writer.add_logit_scale(hparams.get("logit_scale", 1.0))
self.gguf_writer.add_sliding_window(hparams["sliding_window"])
self.gguf_writer.add_sliding_window_pattern([
layer_type == "sliding_attention"
for layer_type in hparams["layer_types"]
])
self.gguf_writer.add_rope_dimension_count(hparams.get("head_dim", hparams["hidden_size"] // hparams["num_attention_heads"]))
self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
self.gguf_writer.add_expert_feed_forward_length(hparams["intermediate_size"])
self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
self.gguf_writer.add_expert_count(hparams["num_experts"])
self.gguf_writer.add_expert_used_count(hparams["num_experts_per_tok"])
self.gguf_writer.add_expert_weights_norm(bool(hparams.get("norm_topk_prob", False)))
expert_selection_fn = hparams.get("expert_selection_fn", "softmax")
if expert_selection_fn != "sigmoid":
raise ValueError(f"Unsupported Cohere2-MoE expert_selection_fn={expert_selection_fn!r}")
self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
if hparams.get("num_shared_experts", 0) != 0:
raise ValueError("Cohere2-MoE shared experts are not supported in this GGUF converter yet")
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
# Cohere2-MoE HF tensors already use the interleaved RoPE layout expected here.
if ".mlp.experts." in name:
n_experts = self.hparams["num_experts"]
assert bid is not None
if self._experts is None:
self._experts = [{} for _ in range(self.block_count)]
self._experts[bid][name] = data_torch
if len(self._experts[bid]) < n_experts * 3:
return []
tensors: list[tuple[str, Tensor]] = []
for src, dst in [
("gate_proj", "gate_proj"),
("down_proj", "down_proj"),
("up_proj", "up_proj"),
]:
datas: list[Tensor] = []
for xid in range(n_experts):
ename = f"model.layers.{bid}.mlp.experts.{xid}.{src}.weight"
datas.append(self._experts[bid][ename])
del self._experts[bid][ename]
merged_name = f"model.layers.{bid}.mlp.experts.{dst}.weight"
tensors.append((self.map_tensor_name(merged_name), torch.stack(datas, dim=0)))
yield from tensors
return
if name == "model.embed_tokens.weight":
yield self.map_tensor_name(name), data_torch
if self.tensor_names is None or "lm_head.weight" not in self.tensor_names:
yield self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT, suffix=".weight"), data_torch
return
yield self.map_tensor_name(name), data_torch
@Model.register("OlmoForCausalLM")
@Model.register("OLMoForCausalLM")
class OlmoModel(Model):

19
gguf-py/gguf/constants.py
View File

@ -250,6 +250,7 @@ class MODEL_ARCH(IntEnum):
MAMBA = auto()
XVERSE = auto()
COMMAND_R = auto()
COHERE2_MOE = auto()
DBRX = auto()
OLMO = auto()
OPENELM = auto()
@ -419,6 +420,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.MAMBA: "mamba",
MODEL_ARCH.XVERSE: "xverse",
MODEL_ARCH.COMMAND_R: "command-r",
MODEL_ARCH.COHERE2_MOE: "cohere2_moe",
MODEL_ARCH.DBRX: "dbrx",
MODEL_ARCH.OLMO: "olmo",
MODEL_ARCH.OPENELM: "openelm",
@ -1136,6 +1138,23 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.ATTN_K_NORM,
MODEL_TENSOR.ATTN_Q_NORM,
],
MODEL_ARCH.COHERE2_MOE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
MODEL_TENSOR.OUTPUT,
MODEL_TENSOR.ATTN_NORM,
MODEL_TENSOR.ATTN_Q,
MODEL_TENSOR.ATTN_K,
MODEL_TENSOR.ATTN_V,
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.FFN_GATE,
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
MODEL_TENSOR.FFN_GATE_INP,
MODEL_TENSOR.FFN_GATE_EXP,
MODEL_TENSOR.FFN_DOWN_EXP,
MODEL_TENSOR.FFN_UP_EXP,
],
MODEL_ARCH.DBRX: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,

8
gguf-py/gguf/vocab.py
View File

@ -278,7 +278,13 @@ class SpecialVocab:
elif chat_template_json.is_file():
with open(chat_template_json, encoding = 'utf-8') as f:
chat_template_alt = json.load(f).get('chat_template')
chat_template = tokenizer_config.get('chat_template', chat_template_alt)
prefer_chat_template_alt = False
if chat_template_alt is not None:
config_file = path / 'config.json'
if config_file.is_file():
with open(config_file, encoding = 'utf-8') as f:
prefer_chat_template_alt = json.load(f).get('model_type') == 'cohere2_moe'
chat_template = chat_template_alt if prefer_chat_template_alt else tokenizer_config.get('chat_template', chat_template_alt)
if chat_template is None or isinstance(chat_template, (str, list)):
self.chat_template = chat_template
else:

1
src/CMakeLists.txt
View File

@ -109,6 +109,7 @@ add_library(llama
graphs/build_glm4.cpp
graphs/build_bitnet.cpp
graphs/build_cohere2.cpp
graphs/build_cohere2_moe.cpp
graphs/build_t5.cpp
graphs/build_jais.cpp
graphs/build_chatglm.cpp

82
src/graphs/build_cohere2_moe.cpp Normal file
View File

@ -0,0 +1,82 @@
#include "../llama-build-context.h"
#include "../llama-model.h"
#include "../llama-context.h"
ggml_cgraph * llm_build_context::build_cohere2_moe() {
ggml_cgraph * gf = new_graph_custom();
const int64_t n_embd_head = hparams.n_embd_head_v(0);
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k(0));
const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
ggml_tensor * inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
ggml_tensor * inp_pos = build_inp_pos();
ggml_tensor * KQ_mask = build_inp_KQ_mask();
ggml_tensor * KQ_mask_swa = build_inp_KQ_mask_swa();
for (int il = 0; il < n_layer; ++il) {
const bool is_sliding = hparams.swa_layers[il];
const bool force_rope = il < (int) hparams.n_layer_dense_lead;
ggml_tensor * KQ_mask_l = is_sliding ? KQ_mask_swa : KQ_mask;
ggml_tensor * attn_out = build_std_attention(gf, model.layers[il].attn_norm, inpL, inp_pos, nullptr, nullptr,
KQ_mask_l, nullptr, nullptr, kq_scale, 0.f,
is_sliding ? hparams.n_swa : 0, il, is_sliding || force_rope, false, true, false);
cb(attn_out, "attn_out", il);
if (il == n_layer - 1 && n_tokens > 1) {
ggml_tensor * inp_out_ids = build_inp_out_ids();
attn_out = ggml_get_rows(ctx0, attn_out, inp_out_ids);
inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
}
ggml_tensor * cur;
if (model.layers[il].ffn_gate_inp == nullptr) {
attn_out->op_params[3] = 1;
cur = llm_build_ffn(ctx0, lctx, model.layers[il].attn_norm, inpL,
model.layers[il].ffn_up, nullptr, nullptr,
model.layers[il].ffn_gate, nullptr, nullptr,
model.layers[il].ffn_down, nullptr, nullptr,
nullptr, LLM_FFN_SILU, LLM_FFN_PAR,
cb, il, gf, false, false, attn_out);
} else {
cur = llm_build_std_moe_ffn(ctx0, lctx, model.layers[il].attn_norm, inpL,
model.layers[il].ffn_gate_inp, nullptr,
model.layers[il].ffn_up_exps, nullptr,
model.layers[il].ffn_gate_exps, nullptr,
model.layers[il].ffn_down_exps, nullptr,
nullptr,
nullptr, nullptr,
nullptr, nullptr,
nullptr, nullptr,
n_expert, n_expert_used,
LLM_FFN_SILU, hparams.expert_weights_norm, false, 0.0f,
(llm_expert_gating_func_type) hparams.expert_gating_func,
LLM_FFN_SILU, cb, il, gf, false, model.layers[il].ffn_up_gate_exps, nullptr, nullptr);
cur = ggml_add(ctx0, cur, attn_out);
}
cb(cur, "ffn_out", il);
cur = lctx.cvec.apply_to(ctx0, cur, il);
cb(cur, "l_out", il);
inpL = cur;
}
ggml_tensor * cur = inpL;
cur = llm_build_norm(ctx0, cur, hparams, model.output_norm, nullptr, LLM_NORM_RMS, cb, -1);
cb(cur, "result_norm", -1);
if (hparams.f_logit_scale) {
cur = ggml_scale(ctx0, cur, hparams.f_logit_scale);
cb(cur, "result_norm_scaled", -1);
}
cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
cb(cur, "result_output", -1);
ggml_build_forward_expand(gf, cur);
return gf;
}

1
src/llama-arch.cpp
View File

@ -64,6 +64,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_GRANITE, "granite" },
{ LLM_ARCH_GRANITE_MOE, "granitemoe" },
{ LLM_ARCH_COHERE2, "cohere2" },
{ LLM_ARCH_COHERE2_MOE, "cohere2_moe" },
{ LLM_ARCH_DOTS1, "dots1" },
{ LLM_ARCH_ERNIE4_5, "ernie4_5" },
{ LLM_ARCH_ERNIE4_5_MOE, "ernie4_5-moe" },

1
src/llama-arch.h
View File

@ -63,6 +63,7 @@ enum llm_arch {
LLM_ARCH_GRANITE,
LLM_ARCH_GRANITE_MOE,
LLM_ARCH_COHERE2,
LLM_ARCH_COHERE2_MOE,
LLM_ARCH_DOTS1,
LLM_ARCH_ERNIE4_5,
LLM_ARCH_ERNIE4_5_MOE,

8
src/llama-build-context.cpp
View File

@ -1595,6 +1595,7 @@ static ggml_tensor * llm_build_kqv(
|| model.arch == LLM_ARCH_GPTNEOX
|| model.arch == LLM_ARCH_QWEN2
|| model.arch == LLM_ARCH_COHERE2
|| model.arch == LLM_ARCH_COHERE2_MOE
|| model.arch == LLM_ARCH_COMMAND_R
|| model.arch == LLM_ARCH_GLM4
|| model.arch == LLM_ARCH_GLM4_MOE
@ -1738,7 +1739,7 @@ static ggml_tensor * llm_build_kqv(
auto q_i = ggml_view_3d(ctx, q, q->ne[0], q->ne[1], this_ne12, q->nb[1], q->nb[2], q->nb[2]*i12);
auto kq_i = ggml_mul_mat(ctx, k_i, q_i);
if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 ||
model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_COMMAND_R || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) {
model.arch == LLM_ARCH_COHERE2 || model.arch == LLM_ARCH_COHERE2_MOE || model.arch == LLM_ARCH_COMMAND_R || model.arch == LLM_ARCH_GLM4 || model.arch == LLM_ARCH_GLM4_MOE) {
ggml_mul_mat_set_prec(kq_i, GGML_PREC_F32);
}
if (model.arch == LLM_ARCH_GROK) {
@ -2448,6 +2449,10 @@ ggml_cgraph * llm_build_context::llama_build_graph(
{
result = llm.build_cohere2();
} break;
case LLM_ARCH_COHERE2_MOE:
{
result = llm.build_cohere2_moe();
} break;
case LLM_ARCH_T5:
{
if (lctx.is_encoding) {
@ -2572,6 +2577,7 @@ ggml_tensor * llm_build_context::build_std_attention(ggml_cgraph * gf, ggml_tens
|| model.arch == LLM_ARCH_GPTNEOX
|| model.arch == LLM_ARCH_QWEN2
|| model.arch == LLM_ARCH_COHERE2
|| model.arch == LLM_ARCH_COHERE2_MOE
|| model.arch == LLM_ARCH_COMMAND_R
|| model.arch == LLM_ARCH_GLM4
// || model.arch == LLM_ARCH_GLM4_MOE

1
src/llama-build-context.h
View File

@ -287,6 +287,7 @@ struct llm_build_context {
ggml_cgraph * build_bitnet_158();
ggml_cgraph * build_cohere2();
ggml_cgraph * build_cohere2_moe();
ggml_cgraph * build_t5_encoder();

18
src/llama-hparams.cpp
View File

@ -1170,6 +1170,24 @@ void llm_load_hparams(
default: model.type = e_model::MODEL_UNKNOWN;
}
} break;
case LLM_ARCH_COHERE2_MOE:
{
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_layers, hparams.n_layer);
ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);
ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);
ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);
ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);
if (hparams.expert_gating_func == LLM_EXPERT_GATING_FUNC_TYPE_NONE) {
hparams.expert_gating_func = LLM_EXPERT_GATING_FUNC_SIGMOID;
}
switch (hparams.n_layer) {
case 49: model.type = e_model::MODEL_30B; break;
default: model.type = e_model::MODEL_UNKNOWN;
}
} break;
case LLM_ARCH_BAILINGMOE2:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);

39
src/llama-load-tensors.cpp
View File

@ -137,6 +137,7 @@ struct create_tensors_helper : public create_tensors_helper_interface {
bool create_chatglm_tensors(const LLM_TN & tn);
bool create_cohere2_tensors(const LLM_TN & tn);
bool create_cohere2_moe_tensors(const LLM_TN & tn);
bool create_dots1_tensors(const LLM_TN & tn);
@ -3213,6 +3214,42 @@ bool create_tensors_helper::create_cohere2_tensors(const LLM_TN & tn) {
return use_mmap_buffer;
}
bool create_tensors_helper::create_cohere2_moe_tensors(const LLM_TN & tn) {
LOADING_PRELUDE
model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
model.output_norm = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
model.output = create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
if (model.output == nullptr) {
model.output = create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
}
for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];
ggml_context * ctx_split = ctx_for_layer_split(i);
layer.attn_norm = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_NORM, "weight", i), { n_embd }, 0);
layer.wq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
layer.wk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
layer.wv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
layer.wo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
layer.bq = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd_head_k * n_head}, llama_model_loader::TENSOR_NOT_REQUIRED);
layer.bk = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_k_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
layer.bv = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_v_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
layer.bo = create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
if (i < (int) hparams.n_layer_dense_lead) {
create_std_ffn(i, tn, layer, n_ff, n_embd, ctx_split);
} else {
layer.ffn_gate_inp = create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
use_mmap_buffer &= !create_std_ffn_exps(n_embd, tn, i, 0, hparams.n_ff_exp);
}
}
return use_mmap_buffer;
}
bool create_tensors_helper::create_glm4_tensors(const LLM_TN & tn) {
LOADING_PRELUDE
model.tok_embd = create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
@ -4358,6 +4395,8 @@ bool create_tensors_helper::create_tensors() {
use_mmap_buffer = create_chatglm_tensors(tn); break;
case LLM_ARCH_COHERE2:
use_mmap_buffer = create_cohere2_tensors(tn); break;
case LLM_ARCH_COHERE2_MOE:
use_mmap_buffer = create_cohere2_moe_tensors(tn); break;
case LLM_ARCH_GLM4:
use_mmap_buffer = create_glm4_tensors(tn); break;
case LLM_ARCH_DOTS1:

21
src/llama-model.cpp
View File

@ -1340,6 +1340,27 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
},
},
{
LLM_ARCH_COHERE2_MOE,
{
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
{ LLM_TENSOR_OUTPUT, "output" },
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
{ LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
{ LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
{ LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
{ LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" },
{ LLM_TENSOR_FFN_GATE_UP_EXPS,"blk.%d.ffn_gate_up_exps" },
},
},
{
LLM_ARCH_DOTS1,
{

3
src/llama-vocab.cpp
View File

@ -2020,7 +2020,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "refact") {
pre_type = LLAMA_VOCAB_PRE_TYPE_REFACT;
} else if (
tokenizer_pre == "command-r") {
tokenizer_pre == "command-r" ||
tokenizer_pre == "cohere2_moe") {
pre_type = LLAMA_VOCAB_PRE_TYPE_COMMAND_R;
clean_spaces = false;
} else if (

2
src/llama.cpp
View File

@ -3061,6 +3061,7 @@ static bool is_model_split_supported(const llama_model & model) {
LLM_ARCH_MISTRAL3,
LLM_ARCH_COMMAND_R,
LLM_ARCH_COHERE2,
LLM_ARCH_COHERE2_MOE,
LLM_ARCH_MIMO2,
LLM_ARCH_QWEN3,
LLM_ARCH_QWEN3VL,
@ -7395,6 +7396,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
case LLM_ARCH_GRANITE:
case LLM_ARCH_GRANITE_MOE:
case LLM_ARCH_COHERE2:
case LLM_ARCH_COHERE2_MOE:
case LLM_ARCH_ERNIE4_5:
case LLM_ARCH_ERNIE4_5_MOE:
case LLM_ARCH_SMOLLM3: