nicolas
/
ik_llama_opt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests: no-alloc ctx for CUDA backend path (fix GGML_ASSERT on alloc)

This commit is contained in:
Marvin 2026-09-05 21:53:48 -03:00
parent a34feeb0f9
commit c1a36daada
1 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tests/test-delta-chunk.cpp
View File

@ -248,36 +248,36 @@ static bool run_ggml_op_cuda(const Case & c, const Tensors & t, Result & r, std:
const int hd = c.hd; const int hd = c.hd;
const size_t out_size = (size_t)hd * c.nt * c.hv * c.nseq; const size_t out_size = (size_t)hd * c.nt * c.hv * c.nseq;
const size_t st_size = (size_t)hd * hd * c.hv * c.nseq; const size_t st_size = (size_t)hd * hd * c.hv * c.nseq;
// no-alloc context: ggml_backend_alloc_ctx_tensors assigns the buffers.
// (Pre-allocated tensors trip GGML_ASSERT(ggml_get_no_alloc(ctx)).)
const size_t mem = (out_size + st_size) * 4 const size_t mem = (out_size + st_size) * 4
+ (t.q.size() + t.k.size() + t.v.size() + t.g.size() + t.beta.size() + t.state.size()) * 4 + (t.q.size() + t.k.size() + t.v.size() + t.g.size() + t.beta.size() + t.state.size()) * 4
+ 64 * 1024 * 1024; + 64 * 1024 * 1024;
struct ggml_init_params ip = { mem, nullptr, false }; struct ggml_init_params ip = { mem, nullptr, true };
struct ggml_context * ctx = ggml_init(ip); struct ggml_context * ctx = ggml_init(ip);
if (!ctx) { err = "ggml_init failed"; ggml_backend_free(be); return false; } if (!ctx) { err = "ggml_init failed"; ggml_backend_free(be); return false; }
auto mkc = [&](const char * name, int64_t n0, int64_t n1, int64_t n2, int64_t n3, const void * data, size_t nbytes) { auto mkc = [&](const char * name, int64_t n0, int64_t n1, int64_t n2, int64_t n3) {
struct ggml_tensor * ten = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n0, n1, n2, n3); struct ggml_tensor * ten = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n0, n1, n2, n3);
ggml_set_name(ten, name); ggml_set_name(ten, name);
memcpy(ten->data, data, nbytes);
return ten; return ten;
}; };
struct ggml_tensor * q = mkc("q", hd, c.nt, c.hk, c.nseq, t.q.data(), t.q.size() * 4); struct ggml_tensor * q = mkc("q", hd, c.nt, c.hk, c.nseq);
struct ggml_tensor * k = mkc("k", hd, c.nt, c.hk, c.nseq, t.k.data(), t.k.size() * 4); struct ggml_tensor * k = mkc("k", hd, c.nt, c.hk, c.nseq);
struct ggml_tensor * v = mkc("v", hd, c.nt, c.hv, c.nseq, t.v.data(), t.v.size() * 4); struct ggml_tensor * v = mkc("v", hd, c.nt, c.hv, c.nseq);
// canonical [b,t,h] flat == contiguous base [hv,t,s] flat: permute to the // canonical [b,t,h] flat == contiguous base [hv,t,s] flat: permute to the
// exact view topology production uses ([nt,1,hv,s] / [1,nt,hv,s]). // exact view topology production uses ([nt,1,hv,s] / [1,nt,hv,s]).
struct ggml_tensor * gbase = mkc("gbase", c.hv, c.nt, c.nseq, 1, t.g.data(), t.g.size() * 4); struct ggml_tensor * gbase = mkc("gbase", c.hv, c.nt, c.nseq, 1);
struct ggml_tensor * g = ggml_permute(ctx, gbase, 2, 0, 3, 1); struct ggml_tensor * g = ggml_permute(ctx, gbase, 2, 0, 3, 1);
ggml_set_name(g, "g"); ggml_set_name(g, "g");
struct ggml_tensor * bbase = mkc("bbase", c.hv, 1, c.nt, c.nseq, t.beta.data(), t.beta.size() * 4); struct ggml_tensor * bbase = mkc("bbase", c.hv, 1, c.nt, c.nseq);
struct ggml_tensor * beta = ggml_permute(ctx, bbase, 2, 0, 1, 3); struct ggml_tensor * beta = ggml_permute(ctx, bbase, 2, 0, 1, 3);
ggml_set_name(beta, "beta"); ggml_set_name(beta, "beta");
struct ggml_tensor * st = mkc("st", hd, hd * c.hv, 1, c.nseq, t.state.data(), t.state.size() * 4); struct ggml_tensor * st = mkc("st", hd, hd * c.hv, 1, c.nseq);
struct ggml_tensor * saved = nullptr; struct ggml_tensor * saved = nullptr;
if (c.saved && c.nt > 1) { if (c.saved && c.nt > 1) {
saved = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (int64_t)(c.nt - 1) * st_size); saved = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (int64_t)(c.nt - 1) * st_size);
ggml_set_name(saved, "saved"); ggml_set_name(saved, "saved");
memset(saved->data, 0, ggml_nbytes(saved));
} }
struct ggml_tensor * res = ggml_delta_net(ctx, q, k, v, g, beta, st, saved); struct ggml_tensor * res = ggml_delta_net(ctx, q, k, v, g, beta, st, saved);
res->op_params[0] = c.repeat; res->op_params[0] = c.repeat;
@ -286,6 +286,12 @@ static bool run_ggml_op_cuda(const Case & c, const Tensors & t, Result & r, std:
{ {
ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, be); ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, be);
if (!buf) { err = "CUDA alloc failed"; goto fail; } if (!buf) { err = "CUDA alloc failed"; goto fail; }
ggml_backend_tensor_set(q, t.q.data(), 0, t.q.size() * 4);
ggml_backend_tensor_set(k, t.k.data(), 0, t.k.size() * 4);
ggml_backend_tensor_set(v, t.v.data(), 0, t.v.size() * 4);
ggml_backend_tensor_set(gbase, t.g.data(), 0, t.g.size() * 4);
ggml_backend_tensor_set(bbase, t.beta.data(), 0, t.beta.size() * 4);
ggml_backend_tensor_set(st, t.state.data(), 0, t.state.size() * 4);
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, 16, false); struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, 16, false);
ggml_build_forward_expand(gf, res); ggml_build_forward_expand(gf, res);
ggml_backend_graph_compute(be, gf); ggml_backend_graph_compute(be, gf);