diff --git a/tests/test-delta-chunk.cpp b/tests/test-delta-chunk.cpp index 9c18bc49..2c56ecea 100644 --- a/tests/test-delta-chunk.cpp +++ b/tests/test-delta-chunk.cpp @@ -248,36 +248,36 @@ static bool run_ggml_op_cuda(const Case & c, const Tensors & t, Result & r, std: const int hd = c.hd; const size_t out_size = (size_t)hd * c.nt * c.hv * c.nseq; const size_t st_size = (size_t)hd * hd * c.hv * c.nseq; + // no-alloc context: ggml_backend_alloc_ctx_tensors assigns the buffers. + // (Pre-allocated tensors trip GGML_ASSERT(ggml_get_no_alloc(ctx)).) const size_t mem = (out_size + st_size) * 4 + (t.q.size() + t.k.size() + t.v.size() + t.g.size() + t.beta.size() + t.state.size()) * 4 + 64 * 1024 * 1024; - struct ggml_init_params ip = { mem, nullptr, false }; + struct ggml_init_params ip = { mem, nullptr, true }; struct ggml_context * ctx = ggml_init(ip); if (!ctx) { err = "ggml_init failed"; ggml_backend_free(be); return false; } - auto mkc = [&](const char * name, int64_t n0, int64_t n1, int64_t n2, int64_t n3, const void * data, size_t nbytes) { + auto mkc = [&](const char * name, int64_t n0, int64_t n1, int64_t n2, int64_t n3) { struct ggml_tensor * ten = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n0, n1, n2, n3); ggml_set_name(ten, name); - memcpy(ten->data, data, nbytes); return ten; }; - struct ggml_tensor * q = mkc("q", hd, c.nt, c.hk, c.nseq, t.q.data(), t.q.size() * 4); - struct ggml_tensor * k = mkc("k", hd, c.nt, c.hk, c.nseq, t.k.data(), t.k.size() * 4); - struct ggml_tensor * v = mkc("v", hd, c.nt, c.hv, c.nseq, t.v.data(), t.v.size() * 4); + struct ggml_tensor * q = mkc("q", hd, c.nt, c.hk, c.nseq); + struct ggml_tensor * k = mkc("k", hd, c.nt, c.hk, c.nseq); + struct ggml_tensor * v = mkc("v", hd, c.nt, c.hv, c.nseq); // canonical [b,t,h] flat == contiguous base [hv,t,s] flat: permute to the // exact view topology production uses ([nt,1,hv,s] / [1,nt,hv,s]). - struct ggml_tensor * gbase = mkc("gbase", c.hv, c.nt, c.nseq, 1, t.g.data(), t.g.size() * 4); + struct ggml_tensor * gbase = mkc("gbase", c.hv, c.nt, c.nseq, 1); struct ggml_tensor * g = ggml_permute(ctx, gbase, 2, 0, 3, 1); ggml_set_name(g, "g"); - struct ggml_tensor * bbase = mkc("bbase", c.hv, 1, c.nt, c.nseq, t.beta.data(), t.beta.size() * 4); + struct ggml_tensor * bbase = mkc("bbase", c.hv, 1, c.nt, c.nseq); struct ggml_tensor * beta = ggml_permute(ctx, bbase, 2, 0, 1, 3); ggml_set_name(beta, "beta"); - struct ggml_tensor * st = mkc("st", hd, hd * c.hv, 1, c.nseq, t.state.data(), t.state.size() * 4); + struct ggml_tensor * st = mkc("st", hd, hd * c.hv, 1, c.nseq); struct ggml_tensor * saved = nullptr; if (c.saved && c.nt > 1) { saved = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (int64_t)(c.nt - 1) * st_size); ggml_set_name(saved, "saved"); - memset(saved->data, 0, ggml_nbytes(saved)); } struct ggml_tensor * res = ggml_delta_net(ctx, q, k, v, g, beta, st, saved); res->op_params[0] = c.repeat; @@ -286,6 +286,12 @@ static bool run_ggml_op_cuda(const Case & c, const Tensors & t, Result & r, std: { ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, be); if (!buf) { err = "CUDA alloc failed"; goto fail; } + ggml_backend_tensor_set(q, t.q.data(), 0, t.q.size() * 4); + ggml_backend_tensor_set(k, t.k.data(), 0, t.k.size() * 4); + ggml_backend_tensor_set(v, t.v.data(), 0, t.v.size() * 4); + ggml_backend_tensor_set(gbase, t.g.data(), 0, t.g.size() * 4); + ggml_backend_tensor_set(bbase, t.beta.data(), 0, t.beta.size() * 4); + ggml_backend_tensor_set(st, t.state.data(), 0, t.state.size() * 4); struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, 16, false); ggml_build_forward_expand(gf, res); ggml_backend_graph_compute(be, gf);