Do not call a GPU shard ready while it computes on the CPU
llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0c7934ba96
commit
057be16a92
13 files changed
+510
-2
No files matched your search
@@ -89,7 +89,11 @@ type fakeSystem struct {
|
||||
installed map[string]bool
|
||||
tools map[string]bool
|
||||
lddOut string
|
||||
calls []string
|
||||
// devices is what llama-server --list-devices prints, and backendLdd
|
||||
// what ldd says of its GPU backend.
|
||||
devices string
|
||||
backendLdd string
|
||||
calls []string
|
||||
// locked is how many more package commands find the lock held.
|
||||
locked int
|
||||
}
|
||||
@@ -127,8 +131,12 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
case name == "ldd" && strings.HasSuffix(args[0], "libggml-cuda.so"):
|
||||
return []byte(f.backendLdd), nil
|
||||
case name == "ldd":
|
||||
return []byte(f.lddOut), nil
|
||||
case strings.HasSuffix(name, "llama-server") && args[0] == "--list-devices":
|
||||
return []byte(f.devices), nil
|
||||
case strings.HasSuffix(name, "llama-server"):
|
||||
return []byte("version: 10991 (abc)\n"), nil
|
||||
}
|
||||
@@ -388,3 +396,32 @@ func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) {
|
||||
t.Errorf("link %q %v", target, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
|
||||
// Found in the field: libggml-cuda.so in the build, cuBLAS nowhere the
|
||||
// loader looks, and llama-server serving a GPU placement on the CPU.
|
||||
cuda := "bin-ubuntu-cuda-" + cudaForLlamaCpp + "-x64.tar.gz"
|
||||
rel := newRelease(map[string][]byte{
|
||||
"llama-b1-" + cuda: tarGz(t, map[string]string{
|
||||
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
|
||||
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
|
||||
})
|
||||
gpu := ubuntu
|
||||
gpu.NVIDIADriver = "570.86.15"
|
||||
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
|
||||
devices: "Available devices:\n (none)\n",
|
||||
backendLdd: "\tlibcublas.so.12 => not found\n\tlibcudart.so.12 => /x/libcudart.so.12\n"}
|
||||
in, _ := installer(t, rel.serve(t), sys, gpu)
|
||||
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
||||
if got[0].State != Failed || !strings.Contains(got[0].Detail, "sees no GPU") ||
|
||||
!strings.Contains(got[0].Detail, "libcublas.so.12 is not installed") {
|
||||
t.Fatalf("expected a failure naming the missing library: %+v", got[0])
|
||||
}
|
||||
|
||||
// The same build seeing its GPU is installed.
|
||||
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
|
||||
if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
|
||||
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user