Do not call a GPU shard ready while it computes on the CPU
llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0c7934ba96
commit
057be16a92
13 files changed
+510
-2
No files matched your search
@@ -299,9 +299,55 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
|
||||
}
|
||||
if b.GPUBackend != "" {
|
||||
if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
}
|
||||
return bin, firstVersionLine(out), nil
|
||||
}
|
||||
|
||||
// verifyGPUBackend checks a GPU build can compute on this node's GPUs.
|
||||
//
|
||||
// The binary's own ldd says nothing about it: the GPU backend is a library
|
||||
// the server opens at run time, and when it cannot — cuBLAS missing, a driver
|
||||
// too old — the server carries on with its CPU backend. So the build is asked
|
||||
// which devices it sees, and when it sees none, ldd on the backend says why.
|
||||
func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error {
|
||||
out, err := in.run(ctx, nil, bin, "--list-devices")
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out))
|
||||
}
|
||||
if listsDevice(string(out)) {
|
||||
return nil
|
||||
}
|
||||
why := "its GPU backend did not load"
|
||||
if lib, err := findFile(dir, b.GPUBackend); err != nil {
|
||||
why = "the build has no " + b.GPUBackend
|
||||
} else if out, err := in.run(ctx, nil, "ldd", lib); err == nil {
|
||||
if missing := lddMissing(string(out)); len(missing) > 0 {
|
||||
why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ")
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why)
|
||||
}
|
||||
|
||||
// listsDevice reads llama-server --list-devices: a device line after
|
||||
// "Available devices:", which holds "(none)" when there is none.
|
||||
func listsDevice(out string) bool {
|
||||
listing := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case strings.HasPrefix(line, "Available devices"):
|
||||
listing = true
|
||||
case listing && line != "" && line != "(none)":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// packages installs the operating system packages a build needs.
|
||||
//
|
||||
// Only those missing, and from the node's own package sources: a package the
|
||||
|
||||
Reference in new issue
Block a user