Do not call a GPU shard ready while it computes on the CPU
llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0c7934ba96
commit
057be16a92
13 files changed
+510
-2
No files matched your search
@@ -3,6 +3,7 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -126,6 +127,14 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
|
||||
// Flags accepted is not the same as running where it is placed: a
|
||||
// GPU build whose backend does not load accepts every flag and
|
||||
// computes on the CPU. Asked of the engine here, not of a help.
|
||||
var deviceErr error
|
||||
if fb, ok := eng.(engines.CPUFallback); ok && help == "" {
|
||||
deviceErr = checkDevices(cmd.Context(), name, fb, out)
|
||||
}
|
||||
|
||||
fmt.Fprintf(out, "%s: %d flags checked", name, checked)
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(out, ", %d commands skipped", skipped)
|
||||
@@ -137,7 +146,7 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
return fmt.Errorf("nothing was checked; install the engine here or pass --help-output")
|
||||
}
|
||||
fmt.Fprintln(out, "every flag llmgw builds is accepted")
|
||||
return nil
|
||||
return deviceErr
|
||||
}
|
||||
fmt.Fprintf(out, "\n%d flags not mentioned by the program they are passed to:\n", len(unknown))
|
||||
for _, f := range unknown {
|
||||
@@ -249,3 +258,22 @@ func isFlagChar(b byte) bool {
|
||||
return b == '-' || b == '_' || (b >= 'a' && b <= 'z') ||
|
||||
(b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')
|
||||
}
|
||||
|
||||
// checkDevices says which GPUs an engine that can fall back to the CPU sees on
|
||||
// this machine, and fails when the machine has NVIDIA GPUs and it sees none.
|
||||
func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out io.Writer) error {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
fmt.Fprintln(out, "no NVIDIA GPU here: which devices it computes on was not checked")
|
||||
return nil
|
||||
}
|
||||
devs, err := fb.Devices(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(devs) == 0 {
|
||||
return fmt.Errorf("this machine has NVIDIA GPUs and %s sees none: it would serve a GPU placement on the CPU. "+
|
||||
"Its GPU backend did not load; ldd on its libggml-cuda.so names the missing library", name)
|
||||
}
|
||||
fmt.Fprintf(out, "computes on: %s\n", strings.Join(devs, "; "))
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user