diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index b52ecd2e..5e061f63 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -862,6 +862,7 @@ extern "C" { // computation graph struct ggml_cgraph { + uint64_t uid; int size; int n_nodes; int n_leafs; diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index a6a394c4..083eaea2 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1435,6 +1435,11 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru } } +static inline uint64_t get_next_graph_uid() { + static std::atomic counter = 1; + return counter.fetch_add(1, std::memory_order_relaxed); +} + // assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) { // reset splits @@ -1443,6 +1448,8 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg sched->is_reset = false; sched->has_reduce = false; + graph->uid = get_next_graph_uid(); + struct ggml_init_params params = { /* .mem_size = */ sched->context_buffer_size, /* .mem_buffer = */ sched->context_buffer, @@ -1966,6 +1973,8 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg sched->leaf_backend_ids[graph_copy->n_leafs] = tensor_backend_id(leaf); graph_copy->leafs[graph_copy->n_leafs++] = leaf; } + + for (int i = 0; i < sched->n_splits; ++i) sched->splits[i].graph.uid = get_next_graph_uid(); } static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { diff --git a/ggml/src/ggml-cuda.cu b/ggml/src/ggml-cuda.cu index 232f765a..78996594 100644 --- a/ggml/src/ggml-cuda.cu +++ b/ggml/src/ggml-cuda.cu @@ -4506,6 +4506,8 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_cuda_graph graph->use_cpy_indirection = true; // copy pointers to GPU so they can be accessed via indirection within CUDA graph ggml_cuda_cpy_dest_ptrs_copy(graph, graph->cpy_dest_ptrs.data(), graph->cpy_dest_ptrs.size(), stream); + } else { + graph->use_cpy_indirection = false; } return use_cuda_graph; @@ -4514,6 +4516,9 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_cuda_graph static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) { graph_node_properties->node_address = node->data; graph_node_properties->node_op = node->op; + graph_node_properties->type = node->type; + graph_node_properties->view_src = node->view_src; + graph_node_properties->view_offs = node->view_offs; for (int i = 0; i < GGML_MAX_DIMS; i++) { graph_node_properties->ne[i] = node->ne[i]; graph_node_properties->nb[i] = node->nb[i]; @@ -4524,46 +4529,13 @@ static void set_ggml_graph_node_properties(ggml_tensor * node, ggml_graph_node_p memcpy(graph_node_properties->op_params, node->op_params, GGML_MAX_OP_PARAMS); } -static bool ggml_graph_node_has_matching_properties(ggml_tensor * node, ggml_graph_node_properties * graph_node_properties) { - if (node->data != graph_node_properties->node_address && - node->op != GGML_OP_CPY && - node->op != GGML_OP_VIEW) { - return false; - } - - if (node->op != graph_node_properties->node_op) { - return false; - } - - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->ne[i] != graph_node_properties->ne[i]) { - return false; - } - if (node->nb[i] != graph_node_properties->nb[i]) { - return false; - } - } - - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (node->src[i] && - node->src[i]->data != graph_node_properties->src_address[i] && - node->op != GGML_OP_VIEW && - !(node->op == GGML_OP_CPY && i == 1) - ) { - return false; - } - } - - if ((node->op == GGML_OP_SCALE || node->op == GGML_OP_LATENT_ATTN) && - memcmp(graph_node_properties->op_params, node->op_params, GGML_MAX_OP_PARAMS) != 0) { - return false; - } - - return true; -} - static bool is_cuda_graph_update_required(ggml_cuda_graph * graph, ggml_cgraph * cgraph) { + if (cgraph->uid != 0 && graph->uid == cgraph->uid) { + GGML_ASSERT(graph->ggml_graph_properties.size() == (size_t)cgraph->n_nodes); + return false; + } + bool cuda_graph_update_required = false; if (graph->instance == nullptr) { @@ -4579,16 +4551,16 @@ static bool is_cuda_graph_update_required(ggml_cuda_graph * graph, ggml_cgraph * // Loop over nodes in GGML graph to determine if CUDA graph update is required // and store properties to allow this comparison for the next token for (int i = 0; i < cgraph->n_nodes; i++) { - bool has_matching_properties = true; - if (!cuda_graph_update_required) { - has_matching_properties = ggml_graph_node_has_matching_properties(cgraph->nodes[i], &graph->ggml_graph_properties[i]); - } - if (!has_matching_properties) { + ggml_graph_node_properties new_props; + set_ggml_graph_node_properties(cgraph->nodes[i], &new_props); + if (memcmp(&graph->ggml_graph_properties[i], &new_props, sizeof(new_props)) != 0) { cuda_graph_update_required = true; + memcpy(&graph->ggml_graph_properties[i], &new_props, sizeof(new_props)); } - set_ggml_graph_node_properties(cgraph->nodes[i], &graph->ggml_graph_properties[i]); } + graph->uid = cgraph->uid; + return cuda_graph_update_required; } diff --git a/ggml/src/ggml-cuda/graph.cuh b/ggml/src/ggml-cuda/graph.cuh index 1b3a30ff..0165c892 100644 --- a/ggml/src/ggml-cuda/graph.cuh +++ b/ggml/src/ggml-cuda/graph.cuh @@ -4,7 +4,10 @@ struct ggml_graph_node_properties { void * node_address; - ggml_op node_op; + ggml_op node_op; + ggml_type type; + void * view_src; + size_t view_offs; int64_t ne[GGML_MAX_DIMS]; size_t nb[GGML_MAX_DIMS]; void * src_address[GGML_MAX_SRC]; @@ -21,6 +24,7 @@ struct ggml_cuda_graph { CUDA_CHECK(cudaGraphDestroy(graph)); } } + uint64_t uid = 0; cudaGraph_t graph = nullptr; cudaGraphExec_t instance = nullptr; size_t num_nodes = 0; diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 189add9b..b9f9d16a 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -28271,6 +28271,7 @@ struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t siz assert(obj_size == (size_t)((char *)p - (char *)cgraph)); *cgraph = (struct ggml_cgraph) { + /*.uid =*/ 0ull, /*.size =*/ size, /*.n_nodes =*/ 0, /*.n_leafs =*/ 0, @@ -28293,6 +28294,7 @@ struct ggml_cgraph * ggml_new_graph(struct ggml_context * ctx) { struct ggml_cgraph ggml_graph_view(struct ggml_cgraph * cgraph0, int i0, int i1) { struct ggml_cgraph cgraph = { + /*.uid =*/ 0ull, /*.size =*/ 0, /*.n_nodes =*/ i1 - i0, /*.n_leafs =*/ 0, diff --git a/src/graphs/build_deepseek4.cpp b/src/graphs/build_deepseek4.cpp index d3fefc8b..720abe51 100644 --- a/src/graphs/build_deepseek4.cpp +++ b/src/graphs/build_deepseek4.cpp @@ -1138,6 +1138,7 @@ static ggml_tensor * ds4_attention(ggml_cgraph * gf, ggml_context * ctx0, llm_bu raw_k->nb[1], raw_k->nb[2], raw_k->nb[3], raw_k->nb[2]*first); raw_mask = ggml_view_4d(ctx0, raw_mask, nton, raw_mask->ne[1], raw_mask->ne[2], raw_mask->ne[3], raw_mask->nb[1], raw_mask->nb[2], raw_mask->nb[3], raw_mask->nb[0]*first); + raw_mask = ggml_cont(ctx0, raw_mask); } }