From eaf83865a132f66e8f49efe0e78491625942f068 Mon Sep 17 00:00:00 2001 From: Kawrakow Date: Thu, 16 Apr 2026 17:26:31 +0200 Subject: [PATCH] Vision support for Gemma4 (#1635) * WIP: Gemma4 vision Crashes on the GPU because of rms_norm requiring ne0 to be multiple of warp_size. Runs on the CPU, but produces garbage. * Remove unnecessary assert in CUDA rms_norm * GLU was not advertised as supported on CUDA * Still not working * This seems to work --- examples/mtmd/clip-impl.h | 12 ++ examples/mtmd/clip.cpp | 321 +++++++++++++++++++++++++++++++++++-- examples/mtmd/mtmd-cli.cpp | 8 +- examples/mtmd/mtmd.cpp | 12 +- ggml/src/ggml-cuda.cu | 13 ++ ggml/src/ggml-cuda/norm.cu | 3 +- ggml/src/ggml.c | 41 +++-- src/llama.cpp | 61 ++++++- 8 files changed, 433 insertions(+), 38 deletions(-) diff --git a/examples/mtmd/clip-impl.h b/examples/mtmd/clip-impl.h index 38c426f7..1253b93d 100644 --- a/examples/mtmd/clip-impl.h +++ b/examples/mtmd/clip-impl.h @@ -80,6 +80,9 @@ #define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm #define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale #define TN_LS_2 "%s.blk.%d.ls2.%s" // layer scale +#define TN_LS_OUT "%s.blk.%d.out_scale.%s" // layer out scale (gemma4) +#define TN_ATTN_POST_NORM "%s.blk.%d.attn_post_norm.%s" // post-attn norm (gemma4) +#define TN_FFN_POST_NORM "%s.blk.%d.ffn_post_norm.%s" // post-FFN norm (gemma4) #define TN_LN_PRE "%s.pre_ln.%s" #define TN_LN_POST "%s.post_ln.%s" #define TN_LLAVA_PROJ "mm.%d.%s" @@ -122,6 +125,10 @@ #define TN_MM_NORM_PRE "mm.a.norm_pre.%s" #define TN_MM_NORM_MID "mm.a.norm_mid.%s" +// gemma4 +#define TN_STD_BIAS "v.std_bias" +#define TN_STD_SCALE "v.std_scale" + // cogvlm #define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s" #define TN_MM_H_TO_4H "mm.up.%s" @@ -143,6 +150,8 @@ enum projector_type { PROJECTOR_TYPE_QWEN2VL, PROJECTOR_TYPE_QWEN3VL, PROJECTOR_TYPE_GEMMA3, + PROJECTOR_TYPE_GEMMA4V, + PROJECTOR_TYPE_GEMMA4A, PROJECTOR_TYPE_IDEFICS3, PROJECTOR_TYPE_PIXTRAL, PROJECTOR_TYPE_QWEN25VL, @@ -159,6 +168,7 @@ enum projector_type { PROJECTOR_TYPE_COGVLM, PROJECTOR_TYPE_JANUS_PRO, PROJECTOR_TYPE_UNKNOWN, + }; static std::map PROJECTOR_TYPE_NAMES = { @@ -171,6 +181,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"}, { PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"}, { PROJECTOR_TYPE_GEMMA3, "gemma3"}, + { PROJECTOR_TYPE_GEMMA4V, "gemma4v"}, + { PROJECTOR_TYPE_GEMMA4A, "gemma4a"}, { PROJECTOR_TYPE_IDEFICS3, "idefics3"}, { PROJECTOR_TYPE_PIXTRAL, "pixtral"}, { PROJECTOR_TYPE_ULTRAVOX, "ultravox"}, diff --git a/examples/mtmd/clip.cpp b/examples/mtmd/clip.cpp index 946c93fc..3a9dc6a6 100644 --- a/examples/mtmd/clip.cpp +++ b/examples/mtmd/clip.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #define DEFAULT_INTERPOLATION_MODE ((int)GGML_SCALE_MODE_BILINEAR | (int)GGML_SCALE_FLAG_ALIGN_CORNERS) @@ -162,6 +163,10 @@ static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u } #endif +static inline bool string_ends_with(std::string_view str, std::string_view suffix) { + return str.size() >= suffix.size() && str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; +} + // // clip layers @@ -273,6 +278,10 @@ struct clip_layer { // layer scale (no bias) ggml_tensor * ls_1_w = nullptr; ggml_tensor * ls_2_w = nullptr; + ggml_tensor * ls_out_w = nullptr; // gemma4 + + ggml_tensor * ff_post_norm_w = nullptr; + ggml_tensor * attn_post_norm_w = nullptr; // qwen3vl deepstack merger ggml_tensor * deepstack_norm_w = nullptr; @@ -414,6 +423,18 @@ struct clip_model { ggml_tensor * mm_boi = nullptr; ggml_tensor * mm_eoi = nullptr; + // gemma4 + ggml_tensor * std_bias = nullptr; + ggml_tensor * std_scale = nullptr; + // Gemma4ClippableLinear + struct clamp_info { + float inp_max; + float inp_min; + float out_max; + float out_min; + }; + std::map clamp_info_map; + bool audio_has_avgpool() const { return proj_type == PROJECTOR_TYPE_QWEN2A || proj_type == PROJECTOR_TYPE_VOXTRAL; @@ -536,7 +557,7 @@ struct clip_graph { const int d_head; const int n_layer; const float eps; - const float kq_scale; + float kq_scale; ggml_context_ptr ctx0_ptr; ggml_context * ctx0; @@ -1117,6 +1138,156 @@ struct clip_graph { return gf; } + ggml_tensor * build_mm_gemma4(ggml_tensor * w, ggml_tensor * x) const { + // Gemma4ClippableLinear + + auto it = model.clamp_info_map.find(w->name); + if (it == model.clamp_info_map.end()) { + return ggml_mul_mat(ctx0, w, x); + } else { + const auto & clamp_info = it->second; + ggml_tensor * clamped = ggml_clamp(ctx0, x, clamp_info.inp_min, clamp_info.inp_max); + ggml_tensor * out = ggml_mul_mat(ctx0, w, clamped); + out = ggml_clamp(ctx0, out, clamp_info.out_min, clamp_info.out_max); + return out; + } + } + + // Gemma4 + ggml_cgraph * build_gemma4() { + ggml_tensor * inp_raw = build_inp_raw(); + + // patches = 2 * (patches - 0.5) + // equivalent to: patches * 2 - 1 + inp_raw = ggml_scale_bias(ctx0, inp_raw, 2.0f, -1.0f); + ggml_set_name(inp_raw, "inp_raw_scaled"); + + ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); + inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd); + inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + ggml_set_name(inp, "inp"); + // note: no patch bias + + ggml_tensor * pos_x = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_x, "pos_x"); + ggml_set_input(pos_x); + + ggml_tensor * pos_y = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_y, "pos_y"); + ggml_set_input(pos_y); + + { + const int64_t pos_size = model.position_embeddings->ne[1]; + const size_t nb1 = ggml_row_size(model.position_embeddings->type, n_embd); + + // positional embeddings are stored as lookup tables (one for x, one for y) + ggml_tensor * tbl_x = ggml_view_2d(ctx0, model.position_embeddings, + n_embd, pos_size, nb1, 0); + ggml_tensor * tbl_y = ggml_view_2d(ctx0, model.position_embeddings, + n_embd, pos_size, nb1, pos_size * nb1); + + // ggml_get_rows: [n_embd, n_patches] + ggml_tensor * emb_x = ggml_get_rows(ctx0, tbl_x, pos_x); + ggml_tensor * emb_y = ggml_get_rows(ctx0, tbl_y, pos_y); + + inp = ggml_add(ctx0, inp, emb_x); + inp = ggml_add(ctx0, inp, emb_y); + cb(inp, "pos_embd", -1); + } + + // similar to build_rope_2d, but use neox ordering + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + const int64_t n_dim = cur->ne[0]; + const int64_t n_head = cur->ne[1]; + const int64_t n_pos = cur->ne[2]; + + // first half + ggml_tensor * first; + { + first = ggml_view_3d(ctx0, cur, + n_dim/2, n_head, n_pos, + cur->nb[1], + cur->nb[2], + 0); + first = ggml_rope_ext( + ctx0, + first, + pos_x, // positions + nullptr, // freq factors + n_dim/2, // n_dims + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); + } + + // second half + ggml_tensor * second; + { + second = ggml_view_3d(ctx0, cur, + n_dim/2, n_head, n_pos, + cur->nb[1], + cur->nb[2], + n_dim/2 * ggml_element_size(cur)); + second = ggml_rope_ext( + ctx0, + second, + pos_y, // positions + nullptr, // freq factors + n_dim/2, // n_dims + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, + 1.0f, 0.0f, 1.0f, 0.0f, 0.0f + ); + } + + cur = ggml_concat(ctx0, first, second, 0); + return cur; + }; + + kq_scale = 1.0f; + ggml_tensor * cur = build_vit( + inp, n_patches, + NORM_TYPE_RMS, + hparams.ffn_op, + nullptr, // pos embd is already handled above + add_pos); + + // Gemma4VisionPooler + { + const int kernel_size = hparams.n_merge; + GGML_ASSERT(kernel_size > 0); + + // [n_embd, n_patches] -> [n_patches_x, n_patches_y, n_embd, 1] + cur = ggml_cont_4d(ctx0, ggml_transpose(ctx0, cur), n_patches_x, n_patches_y, n_embd, 1); + cur = ggml_pool_2d(ctx0, cur, GGML_OP_POOL_AVG, + kernel_size, kernel_size, kernel_size, kernel_size, 0, 0); + const int out_x = n_patches_x / kernel_size; + const int out_y = n_patches_y / kernel_size; + // [out_x, out_y, n_embd, 1] -> [n_embd, out_x * out_y] + cur = ggml_reshape_3d(ctx0, cur, out_x * out_y, n_embd, 1); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_scale(ctx0, cur, sqrtf((float)n_embd)); + cb(cur, "pooled", -1); + } + + // hidden_states = (hidden_states - self.std_bias) * self.std_scale + if (model.std_bias && model.std_scale) { + cur = ggml_sub(ctx0, cur, model.std_bias); + cur = ggml_mul(ctx0, cur, model.std_scale); + cb(cur, "std_scaled", -1); + } + + // Gemma4MultimodalEmbedder + cur = build_mm_gemma4(model.mm_input_proj_w, cur); + cb(cur, "projected", -1); + + // embedding_post_projection_norm + cur = ggml_rms_norm(ctx0, cur, hparams.eps); + cb(cur, "projected_normed", -1); + + ggml_build_forward_expand(gf, cur); + return gf; + } + ggml_cgraph * build_minicpmv() { const int batch_size = 1; @@ -2164,6 +2335,17 @@ private: // TODO: q/k norm requires row size == n_embd, while here it's d_head // we can add support in the future if needed GGML_ASSERT(layer.q_norm == nullptr && layer.k_norm == nullptr); + if (layer.q_norm) { + GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); + Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); + cb(Qcur, "Qcur_norm", il); + } + + if (layer.k_norm) { + GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]); + Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); + cb(Kcur, "Kcur_norm", il); + } } else { // separate q, k, v @@ -2182,19 +2364,35 @@ private: Vcur = ggml_add(ctx0, Vcur, layer.v_b); } - if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); - cb(Qcur, "Qcur_norm", il); - } + bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head; - if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); - cb(Kcur, "Kcur_norm", il); + if (!norm_per_head) { + if (layer.q_norm) { + Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); + cb(Qcur, "Qcur_norm", il); + } + + if (layer.k_norm) { + Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); + cb(Kcur, "Kcur_norm", il); + } } Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos); Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos); Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos); + + if (norm_per_head) { + if (layer.q_norm) { + Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); + cb(Qcur, "Qcur_norm_per_head", il); + } + if (layer.k_norm) { + Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); + cb(Kcur, "Kcur_norm_per_head", il); + } + } + } cb(Qcur, "Qcur", il); @@ -2208,6 +2406,11 @@ private: cb(Kcur, "Kcur_pos", il); } + if (model.proj_type == PROJECTOR_TYPE_GEMMA4V) { + Vcur = ggml_rms_norm(ctx0, Vcur, eps); + cb(Vcur, "Vcur_normed", il); + } + cur = build_attn(layer.o_w, layer.o_b, Qcur, Kcur, Vcur, nullptr, kq_scale, il); cb(cur, "attn_out", il); @@ -2218,6 +2421,11 @@ private: cb(cur, "attn_out_scaled", il); } + if (layer.attn_post_norm_w) { + cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il); + cb(cur, "attn_post_normed", il); + } + // re-add the layer input, e.g., residual cur = ggml_add(ctx0, cur, inpL); @@ -2238,6 +2446,11 @@ private: cb(cur, "ffn_out", il); + if (layer.ff_post_norm_w) { + cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il); + cb(cur, "ffn_post_normed", il); + } + if (layer.ls_2_w) { cur = ggml_mul(ctx0, cur, layer.ls_2_w); cb(cur, "ffn_out_scaled", il); @@ -2247,6 +2460,11 @@ private: cur = ggml_add(ctx0, inpL, cur); cb(cur, "layer_out", il); + if (layer.ls_out_w) { + cur = ggml_mul(ctx0, cur, layer.ls_out_w); + cb(cur, "layer_out_scaled", il); + } + inpL = cur; } @@ -2658,6 +2876,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32 { res = graph.build_qwen3vl(); } break; + case PROJECTOR_TYPE_GEMMA4V: + { + res = graph.build_gemma4(); + } break; case PROJECTOR_TYPE_MINICPMV: { res = graph.build_minicpmv(); @@ -3018,6 +3240,16 @@ struct clip_model_loader { LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); } } break; + case PROJECTOR_TYPE_GEMMA4V: + { + hparams.rope_theta = 100.0f; + hparams.n_merge = 3; // pooling_kernel_size + //hparams.image_resize_algo = img_tool::RESIZE_ALGO_BILINEAR; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + // @ngxson : the model performs quite poor with small images, we need to bump minimum image tokens to 40 to avoid that + hparams.set_limit_image_tokens(252, 280); + hparams.set_warmup_n_tokens(256); // avoid OOM on warmup + } break; case PROJECTOR_TYPE_LLAMA4: { hparams.rope_theta = 10000.0f; @@ -3086,6 +3318,11 @@ struct clip_model_loader { std::map tensor_offset; std::vector tensors_to_load; + auto fin = std::ifstream(fname, std::ios::binary); + if (!fin) { + throw std::runtime_error(string_format("%s: failed to open %s\n", __func__, fname.c_str())); + } + // TODO @ngxson : support both audio and video in the future const char * prefix = model.modality == CLIP_MODALITY_AUDIO ? "a" : "v"; @@ -3122,6 +3359,19 @@ struct clip_model_loader { return cur; }; + auto get_scalar = [&](const std::string & name, float default_val) { + auto it = tensor_offset.find(name); + if (it == tensor_offset.end()) { + return default_val; + } + size_t offset = it->second; + fin.seekg(offset, std::ios::beg); + float value; + fin.read(reinterpret_cast(&value), sizeof(float)); + return value; + }; + + model.class_embedding = get_tensor(TN_CLASS_EMBD, false); model.pre_ln_w = get_tensor(string_format(TN_LN_PRE, prefix, "weight"), false); @@ -3151,6 +3401,9 @@ struct clip_model_loader { layer.ln_2_w = get_tensor(string_format(TN_LN_2, prefix, il, "weight"), false); layer.ls_1_w = get_tensor(string_format(TN_LS_1, prefix, il, "weight"), false); // no bias layer.ls_2_w = get_tensor(string_format(TN_LS_2, prefix, il, "weight"), false); // no bias + layer.ls_out_w = get_tensor(string_format(TN_LS_OUT, prefix, il, "weight"), false); // no bias + layer.attn_post_norm_w = get_tensor(string_format(TN_ATTN_POST_NORM, prefix, il, "weight"), false); // no bias + layer.ff_post_norm_w = get_tensor(string_format(TN_FFN_POST_NORM, prefix, il, "weight"), false); // no bias layer.k_b = get_tensor(string_format(TN_ATTN_K, prefix, il, "bias"), false); layer.q_b = get_tensor(string_format(TN_ATTN_Q, prefix, il, "bias"), false); @@ -3327,6 +3580,32 @@ struct clip_model_loader { model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); model.mm_soft_emb_norm_w = get_tensor(TN_MM_SOFT_EMB_N); } break; + case PROJECTOR_TYPE_GEMMA4V: + { + model.mm_input_proj_w = get_tensor(TN_MM_INP_PROJ); + model.std_bias = get_tensor(TN_STD_BIAS, false); + model.std_scale = get_tensor(TN_STD_SCALE, false); + // load scalar for Gemma4ClippableLinear + for (auto * tensor : tensors_to_load) { + std::string name = tensor->name; + if (string_ends_with(name, ".weight")) { + std::string name_inp_max = name; + std::string name_inp_min = name; + std::string name_out_max = name; + std::string name_out_min = name; + string_replace_all(name_inp_max, ".weight", ".input_max"); + string_replace_all(name_inp_min, ".weight", ".input_min"); + string_replace_all(name_out_max, ".weight", ".output_max"); + string_replace_all(name_out_min, ".weight", ".output_min"); + model.clamp_info_map[name] = { + get_scalar(name_inp_max, FLT_MAX), + get_scalar(name_inp_min, -FLT_MAX), + get_scalar(name_out_max, FLT_MAX), + get_scalar(name_out_min, -FLT_MAX) + }; + } + } + } break; case PROJECTOR_TYPE_IDEFICS3: { model.projection = get_tensor(TN_MM_PROJECTOR); @@ -3703,14 +3982,18 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params try { clip_model_loader loader(fname); + bool skip_audio = false; if (loader.has_vision) { ctx_vision = new clip_ctx(ctx_params); loader.load_hparams(ctx_vision->model, CLIP_MODALITY_VISION); loader.load_tensors(*ctx_vision); loader.warmup(*ctx_vision); + if (ctx_vision->model.proj_type == PROJECTOR_TYPE_GEMMA4V) { + skip_audio = true; + } } - if (loader.has_audio) { + if (!skip_audio && loader.has_audio) { ctx_audio = new clip_ctx(ctx_params); loader.load_hparams(ctx_audio->model, CLIP_MODALITY_AUDIO); loader.load_tensors(*ctx_audio); @@ -4415,12 +4698,14 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_GEMMA4V: { GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0); clip_image_u8 resized; + const int cur_merge = params.n_merge == 0 ? 1 : params.n_merge; const clip_image_size new_size = img_tool::calc_size_preserved_ratio( original_size, - params.patch_size * 2, + params.patch_size * cur_merge, params.image_min_pixels, params.image_max_pixels); img_tool::resize(*img, resized, new_size, img_tool::RESIZE_ALGO_BILINEAR, false); @@ -4749,6 +5034,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im case PROJECTOR_TYPE_IDEFICS3: case PROJECTOR_TYPE_INTERNVL: case PROJECTOR_TYPE_LLAMA4: + case PROJECTOR_TYPE_GEMMA4V: { // both X and Y are downscaled by the scale factor int scale_factor = ctx->model.hparams.n_merge; @@ -5073,6 +5359,19 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima set_input_i32("positions", positions); } break; + case PROJECTOR_TYPE_GEMMA4V: + { + // set (col, row) patch positions for learned positional embedding + // what about image_size_height? + const int n_cols = image_size_width / patch_size; + std::vector pos_x(num_patches), pos_y(num_patches); + for (int i = 0; i < num_patches; i++) { + pos_x[i] = i % n_cols; + pos_y[i] = i / n_cols; + } + set_input_i32("pos_x", pos_x); + set_input_i32("pos_y", pos_y); + } break; case PROJECTOR_TYPE_QWEN25VL: { // pw * ph = number of tokens output by ViT after apply patch merger @@ -5300,6 +5599,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers); case PROJECTOR_TYPE_GEMMA3: return ctx->model.mm_input_proj_w->ne[0]; + case PROJECTOR_TYPE_GEMMA4V: + return ctx->model.mm_input_proj_w->ne[1]; case PROJECTOR_TYPE_IDEFICS3: return ctx->model.projection->ne[1]; case PROJECTOR_TYPE_ULTRAVOX: diff --git a/examples/mtmd/mtmd-cli.cpp b/examples/mtmd/mtmd-cli.cpp index 55a9a14c..496a1e53 100644 --- a/examples/mtmd/mtmd-cli.cpp +++ b/examples/mtmd/mtmd-cli.cpp @@ -252,11 +252,11 @@ static int generate_response(mtmd_cli_context & ctx, int n_predict) { return 0; } -static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, bool add_bos = false) { +static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg, bool add_bos = false, bool use_jinja = false) { common_chat_templates_inputs tmpl_inputs; tmpl_inputs.messages = {msg}; tmpl_inputs.add_generation_prompt = true; - tmpl_inputs.use_jinja = false; // jinja is buggy here + tmpl_inputs.use_jinja = use_jinja; // jinja is buggy here auto formatted_chat = common_chat_templates_apply(ctx.tmpls.get(), tmpl_inputs); LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.prompt.c_str()); @@ -362,7 +362,7 @@ int main(int argc, char ** argv) { return 1; // error is already printed by libmtmd } } - if (eval_message(ctx, msg, true)) { + if (eval_message(ctx, msg, true, params.use_jinja)) { return 1; } if (!g_is_interrupted && generate_response(ctx, n_predict)) { @@ -427,7 +427,7 @@ int main(int argc, char ** argv) { common_chat_msg msg; msg.role = "user"; msg.content = content; - int ret = eval_message(ctx, msg, is_first_msg); + int ret = eval_message(ctx, msg, is_first_msg, params.use_jinja); if (ret) { return 1; } diff --git a/examples/mtmd/mtmd.cpp b/examples/mtmd/mtmd.cpp index 530b334d..e89edc5d 100644 --- a/examples/mtmd/mtmd.cpp +++ b/examples/mtmd/mtmd.cpp @@ -301,6 +301,12 @@ struct mtmd_context { img_end = "<|im_end|>"; } + else if (proj == PROJECTOR_TYPE_GEMMA4V) { + // <|image> ... (image embeddings) ... + img_beg = "<|image>"; + img_end = ""; + //image_preproc = std::make_unique(ctx_v); + } } void init_audio() { @@ -831,8 +837,10 @@ float * mtmd_get_output_embd(mtmd_context * ctx) { } bool mtmd_decode_use_non_causal(mtmd_context * ctx) { - if (ctx->ctx_v && clip_get_projector_type(ctx->ctx_v) == PROJECTOR_TYPE_GEMMA3) { - return true; + if (ctx->ctx_v) { + if (auto type = clip_get_projector_type(ctx->ctx_v); type == PROJECTOR_TYPE_GEMMA3 || type == PROJECTOR_TYPE_GEMMA4V) { + return true; + } } return false; } diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu index d1abfc13..f8d65397 100644 --- a/ggml/src/ggml-cuda.cu +++ b/ggml/src/ggml-cuda.cu @@ -4454,6 +4454,19 @@ GGML_CALL static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, cons return false; } break; + case GGML_OP_GLU: + switch (ggml_get_glu_op(op)) { + case GGML_GLU_OP_REGLU: + case GGML_GLU_OP_GEGLU: + case GGML_GLU_OP_SWIGLU: + case GGML_GLU_OP_SWIGLU_OAI: + case GGML_GLU_OP_GEGLU_ERF: + case GGML_GLU_OP_GEGLU_QUICK: + return ggml_is_contiguous_1(op->src[0]); + default: + return false; + } + break; case GGML_OP_FUSED_MUL_UNARY: return ggml_is_contiguous(op->src[0]); case GGML_OP_MUL_MAT: case GGML_OP_MUL_MAT_ID: diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index ebc1cce7..52a59be2 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -438,7 +438,8 @@ static void group_norm_f32_cuda(const float * x, float * dst, const int num_grou } static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) { - GGML_ASSERT(ncols % WARP_SIZE == 0); + // Why did we have this assert? + //GGML_ASSERT(ncols % WARP_SIZE == 0); constexpr int kBlockSize = 256; if (ncols < 1024) { const dim3 block_dims(kBlockSize, 1, 1); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index bc049720..23d0cc60 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -6439,7 +6439,7 @@ static struct ggml_tensor * ggml_sub_impl( struct ggml_tensor * a, struct ggml_tensor * b, bool inplace) { - GGML_ASSERT(ggml_are_same_shape(a, b)); + GGML_ASSERT(ggml_can_repeat(b, a)); bool is_node = false; @@ -13637,7 +13637,8 @@ static void ggml_compute_forward_sub_f32( const struct ggml_tensor * src0 = dst->src[0]; const struct ggml_tensor * src1 = dst->src[1]; - assert(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst)); + GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst)); + GGML_ASSERT(src0->ne[0] == src1->ne[0]); int ith = params->ith; int nth = params->nth; @@ -13655,21 +13656,24 @@ static void ggml_compute_forward_sub_f32( if (nb10 == sizeof(float)) { for (int ir = first; ir < last; ++ir) { // src0, src1 and dst are same shape => same indices - const int i3 = ir/(ne2*ne1); - const int i2 = (ir - i3*ne2*ne1)/ne1; - const int i1 = (ir - i3*ne2*ne1 - i2*ne1); + const int i03 = ir/(ne2*ne1); + const int i02 = (ir - i03*ne2*ne1)/ne1; + const int i01 = (ir - i03*ne2*ne1 - i02*ne1); + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; #ifdef GGML_USE_ACCELERATE vDSP_vsub( - (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11), 1, - (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), 1, - (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), 1, + (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), 1, + (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01), 1, + (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ), 1, ne0); #else ggml_vec_sub_f32(ne0, - (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ), - (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01), - (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11)); + (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ), + (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01), + (float *) ((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11)); #endif // } // } @@ -13678,13 +13682,16 @@ static void ggml_compute_forward_sub_f32( // src1 is not contiguous for (int ir = 0; ir < nr; ++ir) { // src0, src1 and dst are same shape => same indices - const int i3 = ir/(ne2*ne1); - const int i2 = (ir - i3*ne2*ne1)/ne1; - const int i1 = (ir - i3*ne2*ne1 - i2*ne1); + const int i03 = ir/(ne2*ne1); + const int i02 = (ir - i03*ne2*ne1)/ne1; + const int i01 = (ir - i03*ne2*ne1 - i02*ne1); + const int64_t i13 = i03 % ne13; + const int64_t i12 = i02 % ne12; + const int64_t i11 = i01 % ne11; - float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 ); - float * src0_ptr = (float *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01); - char * src1_ptr = (char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11; + float * dst_ptr = (float *) ((char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1 ); + float * src0_ptr = (float *) ((char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01); + char * src1_ptr = (char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11; for (int i0 = 0; i0 < ne0; i0++) { dst_ptr[i0] = src0_ptr[i0] - *(float *)(src1_ptr + i0*nb10); } diff --git a/src/llama.cpp b/src/llama.cpp index da1c8a12..afab028e 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -3397,7 +3397,28 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); - float * data = (float *) lctx.inp_KQ_mask->data; + float * data = nullptr; + float * data_swa = nullptr; + ggml_half * data_f16 = nullptr; + ggml_half * data_swa_f16 = nullptr; + + if (lctx.inp_KQ_mask) { + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask->buffer)); + if (cparams.flash_attn) { + data_f16 = (ggml_half *)lctx.inp_KQ_mask->data; + } else { + data = (float *) lctx.inp_KQ_mask->data; + } + } + + if (lctx.inp_KQ_mask_swa) { + GGML_ASSERT(ggml_backend_buffer_is_host(lctx.inp_KQ_mask_swa->buffer)); + if (cparams.flash_attn) { + data_swa_f16 = (ggml_half *) lctx.inp_KQ_mask_swa->data; + } else { + data_swa = (float *) lctx.inp_KQ_mask_swa->data; + } + } for (int h = 0; h < 1; ++h) { for (int j = 0; j < n_tokens; ++j) { @@ -3416,11 +3437,43 @@ static void llama_set_inputs(llama_context & lctx, const llama_batch & batch) { } } - data[h*(n_tokens*n_tokens) + j*n_stride + i] = f; + if (data) { + data[h*(n_tokens*n_tokens) + j*n_stride + i] = f; + } + if (data_f16) { + data_f16[h*(n_tokens*n_tokens) + j*n_tokens + i] = ggml_fp32_to_fp16(f); + } + + if (data_swa || data_swa_f16) { + if (hparams.n_attn_chunk) { + llama_pos pos_chunk_start = (batch.pos[i] / hparams.n_attn_chunk) * hparams.n_attn_chunk; + if (lctx.kv_self.cells[i].pos < pos_chunk_start || batch.pos[i] < pos_chunk_start) { + f = -INFINITY; + } + } else { + if (batch.pos[i] - kv_self.cells[i].pos >= (int32_t)hparams.n_swa) { + f = -INFINITY; + } + } + if (data_swa) { + data_swa[h*(n_tokens*n_tokens) + j*n_tokens + i] = f; + } + if (data_swa_f16) { + data_swa_f16[h*(n_tokens*n_tokens) + j*n_tokens + i] = ggml_fp32_to_fp16(f); + } + } } - for (int i = n_tokens; i < n_stride; ++i) { - data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY; + if (data) { + for (int i = n_tokens; i < n_stride; ++i) { + data[h*(n_tokens*n_tokens) + j*n_stride + i] = -INFINITY; + } + } + if (data_f16) { + auto h_inf = ggml_fp32_to_fp16(-INFINITY); + for (int i = n_tokens; i < n_stride; ++i) { + data_f16[h*(n_tokens*n_tokens) + j*n_stride + i] = h_inf; + } } } }