From 73ad16269b56a721e8533a450a7aeab0e0417ebb Mon Sep 17 00:00:00 2001 From: Coenie Beyers Date: Wed, 26 Aug 2026 17:02:43 +0200 Subject: [PATCH] rpc: fix crash running GLM-5.2 (glm-dsa) split over RPC (#2360) * rpc: disable unsafe memcmp graph cache Same-shape prefill micro-batches compared equal and took the GRAPH_RECOMPUTE path, re-running a stored graph against a grown KV context; the GLM-5.2 DSA indexer then read past its buffers and crashed the server. Always send the full graph. Upstream retired this cache design in ggml-org/llama.cpp#22701. * rpc: use 64-bit ne/nb in rpc_tensor wire struct ggml_tensor holds int64 ne and size_t nb; the wire struct stored them as uint32, truncating any stride >= 4 GiB. The GLM-5.2 DSA indexer query stride crosses that at ~26k tokens. Bump RPC_PROTO_MAJOR (wire-format change). * ggml: use 64-bit locals in ggml_permute Permuted strides were built in int locals, truncating any stride > 2 GiB before it reached result->nb (size_t). Affects any permuted tensor over ~2 GiB. --- ggml/include/ggml-rpc.h | 6 +++--- ggml/src/ggml-rpc.cpp | 26 +++++++++++++++----------- ggml/src/ggml.c | 5 +++-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index e5db37d3..5df2a1c6 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -7,9 +7,9 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 3 -#define RPC_PROTO_MINOR_VERSION 5 -#define RPC_PROTO_PATCH_VERSION 2 +#define RPC_PROTO_MAJOR_VERSION 4 +#define RPC_PROTO_MINOR_VERSION 0 +#define RPC_PROTO_PATCH_VERSION 0 #define GGML_RPC_MAX_SERVERS 16 // backend API diff --git a/ggml/src/ggml-rpc.cpp b/ggml/src/ggml-rpc.cpp index cfb0f94d..0f0888b8 100644 --- a/ggml/src/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc.cpp @@ -73,8 +73,12 @@ struct rpc_tensor { uint64_t id; uint32_t type; uint64_t buffer; - uint32_t ne[GGML_MAX_DIMS]; - uint32_t nb[GGML_MAX_DIMS]; + // 64-bit to match ggml_tensor (int64_t ne, size_t nb). A 32-bit nb silently + // truncated any stride >= 4 GiB on the wire; the GLM-5.2 DSA indexer query's + // per-token stride crosses that at ~26k tokens, corrupting the stride on the + // remote server. + uint64_t ne[GGML_MAX_DIMS]; + uint64_t nb[GGML_MAX_DIMS]; uint32_t op; int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; int32_t flags; @@ -237,15 +241,15 @@ struct ggml_backend_rpc_buffer_type_context { struct graph_cache { bool is_cached(const ggml_cgraph * cgraph) { - if ((int)last_graph.size() != cgraph->n_nodes) { - return false; - } - for (int i = 0; i < cgraph->n_nodes; i++) { - if (memcmp(&last_graph[i], cgraph->nodes[i], sizeof(ggml_tensor)) != 0) { - return false; - } - } - return true; + // Disabled: comparing graphs by memcmp of the tensor structs treats successive + // same-shape prefill micro-batches as identical, so they take the + // GRAPH_RECOMPUTE path and re-run a stored graph against a grown KV context. + // Stale KV/mask views then make context-dependent ops (e.g. the GLM-5.2 DSA + // indexer) read past their buffers -> server crash. Always send the full graph + // instead (metadata only; tensor data stays in RPC buffers). Upstream + // llama.cpp retired this cache design in ggml-org/llama.cpp#22701. + (void)cgraph; + return false; } void add(const ggml_cgraph * cgraph) { diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index ef687db8..2d72b3f4 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -8846,8 +8846,9 @@ struct ggml_tensor * ggml_permute( //ggml_format_name(result, "%s (permuted)", a->name); ggml_format_name_fast(a->name, " (permuted)", 11, result->name); - int ne[GGML_MAX_DIMS]; - int nb[GGML_MAX_DIMS]; + // 64-bit: nb is size_t in ggml_tensor; a 32-bit int truncated any stride > 2 GiB. + int64_t ne[GGML_MAX_DIMS]; + size_t nb[GGML_MAX_DIMS]; ne[axis0] = a->ne[0]; ne[axis1] = a->ne[1];