aditya/llama.cpp
1
0
Fork 0
mirror of https://git.adityakumar.xyz/llama.cpp.git synced 2024-11-14 00:59:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

llama : fix Metal KV cache sync (close #1695)

Browse source
This commit is contained in:
Georgi Gerganov 2023-06-05 10:19:03 +03:00
parent 827f5eda91
commit d1f563a743
No known key found for this signature in database
GPG key ID: 449E073F9DC10735
1 changed files with 8 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
llama.cpp
View file

@ -1455,6 +1455,14 @@ static bool llama_eval_internal(
// When we implement Matrix x Matrix Metal multiplication, we can avoid this branch. // When we implement Matrix x Matrix Metal multiplication, we can avoid this branch.
// But for now, we have focused only on Matrix x Vector Metal multiplication. // But for now, we have focused only on Matrix x Vector Metal multiplication.
// //
// TODO: avoid these syncs via shared memory (ref #1696)
//
if (lctx.ctx_metal) {
// We need to sync the GPU KV cache with the CPU KV cache
ggml_metal_get_tensor(lctx.ctx_metal, kv_self.k);
ggml_metal_get_tensor(lctx.ctx_metal, kv_self.v);
}
ggml_graph_compute(ctx0, &gf); ggml_graph_compute(ctx0, &gf);
if (lctx.ctx_metal) { if (lctx.ctx_metal) {