Do not call a GPU shard ready while it computes on the CPU
llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0c7934ba96
commit
057be16a92
13 files changed
+510
-2
No files matched your search
@@ -9,6 +9,7 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
@@ -146,6 +147,15 @@ type Engine interface {
|
||||
MetricsPath() string
|
||||
}
|
||||
|
||||
// CPUFallback is an engine that can come up on the CPU when it was placed on
|
||||
// GPUs, and keep serving there without an error. The agent checks such a shard
|
||||
// is on its GPUs before calling it ready.
|
||||
type CPUFallback interface {
|
||||
FallsBackToCPU() bool
|
||||
// Devices lists the GPUs the engine's build can see, for saying why.
|
||||
Devices(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// GPUInfo is the subset of GPU facts validation needs.
|
||||
type GPUInfo struct {
|
||||
Index int
|
||||
@@ -232,6 +242,11 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) }
|
||||
func baseEnv(s ShardSpec) map[string]string {
|
||||
env := map[string]string{
|
||||
"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
|
||||
// The indices above are nvidia-smi's, which counts by PCI bus. CUDA
|
||||
// counts fastest first unless told otherwise, and on a node with two
|
||||
// kinds of GPU the shard would get a different card than placement
|
||||
// gave it — one another shard may hold.
|
||||
"CUDA_DEVICE_ORDER": "PCI_BUS_ID",
|
||||
}
|
||||
|
||||
if len(s.HCAs) > 0 {
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
@@ -234,3 +236,38 @@ func (l *LlamaCpp) bin() string {
|
||||
}
|
||||
return "llama-server"
|
||||
}
|
||||
|
||||
// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does
|
||||
// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and
|
||||
// the server comes up and answers on the CPU, at a tenth of the speed, with no
|
||||
// error anywhere. Its readiness on a GPU placement needs checking separately.
|
||||
func (*LlamaCpp) FallsBackToCPU() bool { return true }
|
||||
|
||||
// Devices lists the GPUs this build can compute on, as llama-server's
|
||||
// --list-devices names them ("CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)").
|
||||
// Empty, on a machine with GPUs, is a build whose GPU backend did not load:
|
||||
// the CPU build, or a CUDA build missing a library its backend links.
|
||||
func (l *LlamaCpp) Devices(ctx context.Context) ([]string, error) {
|
||||
out, err := exec.CommandContext(ctx, l.bin(), "--list-devices").CombinedOutput() //nolint:gosec // the engine's own binary
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s --list-devices: %v: %s", l.bin(), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return parseDevices(string(out)), nil
|
||||
}
|
||||
|
||||
// parseDevices reads --list-devices: the lines after "Available devices:",
|
||||
// which says "(none)" when there are none. Log lines before it are skipped.
|
||||
func parseDevices(out string) []string {
|
||||
var devs []string
|
||||
listing := false
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case strings.HasPrefix(line, "Available devices"):
|
||||
listing = true
|
||||
case listing && line != "" && line != "(none)":
|
||||
devs = append(devs, line)
|
||||
}
|
||||
}
|
||||
return devs
|
||||
}
|
||||
@@ -323,3 +323,14 @@ func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
|
||||
t.Error("a model with no adapters was given lora flags")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDevicesAreReadFromListDevices(t *testing.T) {
|
||||
cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
|
||||
if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") {
|
||||
t.Errorf("CUDA build: %q", got)
|
||||
}
|
||||
// A CUDA build whose backend did not load says what the CPU build says.
|
||||
if got := parseDevices("Available devices:\n (none)\n"); len(got) != 0 {
|
||||
t.Errorf("no devices: %q", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user