Do not call a GPU shard ready while it computes on the CPU

llama.cpp whose CUDA backend cannot load runs on the CPU without an
error. The CUDA build is now asked for its devices when installed, a
shard on GPUs must hold memory on each before it is ready, and
engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID
so CUDA numbers GPUs as placement does.

Refs #2 (item 2).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 11:54:39 +02:00
1 parent 0c7934ba96
commit 057be16a92
13 files changed
+510 -2

No files matched your search

+46
View File
@@ -299,9 +299,55 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers
if err != nil {
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
}
if b.GPUBackend != "" {
if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil {
return "", "", err
}
}
return bin, firstVersionLine(out), nil
}
// verifyGPUBackend checks a GPU build can compute on this node's GPUs.
//
// The binary's own ldd says nothing about it: the GPU backend is a library
// the server opens at run time, and when it cannot — cuBLAS missing, a driver
// too old — the server carries on with its CPU backend. So the build is asked
// which devices it sees, and when it sees none, ldd on the backend says why.
func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error {
out, err := in.run(ctx, nil, bin, "--list-devices")
if err != nil {
return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out))
}
if listsDevice(string(out)) {
return nil
}
why := "its GPU backend did not load"
if lib, err := findFile(dir, b.GPUBackend); err != nil {
why = "the build has no " + b.GPUBackend
} else if out, err := in.run(ctx, nil, "ldd", lib); err == nil {
if missing := lddMissing(string(out)); len(missing) > 0 {
why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ")
}
}
return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why)
}
// listsDevice reads llama-server --list-devices: a device line after
// "Available devices:", which holds "(none)" when there is none.
func listsDevice(out string) bool {
listing := false
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "Available devices"):
listing = true
case listing && line != "" && line != "(none)":
return true
}
}
return false
}
// packages installs the operating system packages a build needs.
//
// Only those missing, and from the node's own package sources: a package the
+38 -1
View File
@@ -89,7 +89,11 @@ type fakeSystem struct {
installed map[string]bool
tools map[string]bool
lddOut string
calls []string
// devices is what llama-server --list-devices prints, and backendLdd
// what ldd says of its GPU backend.
devices string
backendLdd string
calls []string
// locked is how many more package commands find the lock held.
locked int
}
@@ -127,8 +131,12 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
}
}
return nil, nil
case name == "ldd" && strings.HasSuffix(args[0], "libggml-cuda.so"):
return []byte(f.backendLdd), nil
case name == "ldd":
return []byte(f.lddOut), nil
case strings.HasSuffix(name, "llama-server") && args[0] == "--list-devices":
return []byte(f.devices), nil
case strings.HasSuffix(name, "llama-server"):
return []byte("version: 10991 (abc)\n"), nil
}
@@ -388,3 +396,32 @@ func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) {
t.Errorf("link %q %v", target, err)
}
}
func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
// Found in the field: libggml-cuda.so in the build, cuBLAS nowhere the
// loader looks, and llama-server serving a GPU placement on the CPU.
cuda := "bin-ubuntu-cuda-" + cudaForLlamaCpp + "-x64.tar.gz"
rel := newRelease(map[string][]byte{
"llama-b1-" + cuda: tarGz(t, map[string]string{
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
})
gpu := ubuntu
gpu.NVIDIADriver = "570.86.15"
sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
devices: "Available devices:\n (none)\n",
backendLdd: "\tlibcublas.so.12 => not found\n\tlibcudart.so.12 => /x/libcudart.so.12\n"}
in, _ := installer(t, rel.serve(t), sys, gpu)
got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
if got[0].State != Failed || !strings.Contains(got[0].Detail, "sees no GPU") ||
!strings.Contains(got[0].Detail, "libcublas.so.12 is not installed") {
t.Fatalf("expected a failure naming the missing library: %+v", got[0])
}
// The same build seeing its GPU is installed.
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
}
}
+6
View File
@@ -43,6 +43,11 @@ type Build struct {
// VersionArgs is what Binary is run with to prove the build works, when
// --version is not it.
VersionArgs []string
// GPUBackend, when set, is the shared library the build computes on GPUs
// with, loaded at run time rather than linked: "libggml-cuda.so". The
// build must list a device with it, or it is not installed — a llama.cpp
// whose CUDA backend cannot load runs on the CPU without complaint.
GPUBackend string
// FreeGB is the disk space the installation needs before it starts: a
// Python engine is several gigabytes, and one that fills the disk halfway
// leaves the node worse off than one that never began.
@@ -234,6 +239,7 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
if h.NVIDIADriver != "" && arch == "x64" {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
b.Variant = "cuda-" + cudaForLlamaCpp
b.GPUBackend = "libggml-cuda.so"
b.Assets = []string{
fmt.Sprintf("llama-%s-%s", version, name),
fmt.Sprintf("cudart-llama-%s-%s", version, name),