diff --git a/CHANGELOG.md b/CHANGELOG.md index b00e563..4f5266b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,33 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. +## Unreleased + +From the field report in issue #2, after a deployment on GPU nodes. + +### A GPU placement no longer runs on the CPU unnoticed + +llama.cpp placed on a GPU, with its CUDA backend unable to load (cuBLAS not +where the loader looks), came up on the CPU, answered at CPU speed, and was +reported ready while its GPU sat reserved and idle. Three checks now stop it: + +- **Installing** the CUDA build asks it which devices it sees + (`llama-server --list-devices`). None, on a node with GPUs, is a failed + install, with what `ldd` says `libggml-cuda.so` is missing. The binary's + own `ldd` could not say it: the backend is opened at run time. +- **Starting** a shard on GPUs is ready only when its process holds memory on + each of them (`nvidia-smi --query-compute-apps`, its process group + included). One that does not is stopped and reported failed: "placed on GPU + 0 but computing on the CPU", with the devices the engine lists. This covers + a hand-installed binary (`engines.llamacpp.bin`), which nothing else checks. +- **`llmgw ctl engine-check llamacpp`**, run on a GPU node, now also fails + when the engine sees no GPU, and otherwise prints the devices it computes on. + +Shards are also given `CUDA_DEVICE_ORDER=PCI_BUS_ID`: placement numbers GPUs +as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a +node with two kinds of GPU, a shard could get a different card from the one it +was given. + ## v0.10.6 — 2026-10-01 Published first as release candidates v0.10.6-rc.1 to rc.8; rc.5 failed its diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 469e437..19f9d27 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -931,6 +931,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< } else { err = a.sup.WaitAlive(ctx, sh) } + if err == nil { + err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID) + if err != nil { + // Serving, but not where it was placed: stopped, so the GPU it + // holds in the inventory is not taken by a CPU process. + _ = a.sup.Stop(ctx, shardID) + } + } if err != nil { report(supervisor.Failed, err.Error(), sh.Logs(20)) return diff --git a/internal/agent/discovery/gpuprocs.go b/internal/agent/discovery/gpuprocs.go new file mode 100644 index 0000000..b99f2cf --- /dev/null +++ b/internal/agent/discovery/gpuprocs.go @@ -0,0 +1,65 @@ +package discovery + +import ( + "context" + "fmt" + "os/exec" + "strconv" + "strings" +) + +// GPUProcesses reports which processes hold memory on each GPU, by the GPU's +// nvidia-smi index: what a process actually computes on, as opposed to what it +// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose CUDA +// backend did not load runs on the CPU without a word, and this is where that +// shows — the process is on no GPU's list. +// +// A node with no nvidia-smi returns nil and no error, as discovery does. +func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) { + if binary == "" { + if _, err := exec.LookPath("nvidia-smi"); err != nil { + return nil, nil + } + binary = "nvidia-smi" + } + gpus, err := exec.CommandContext(ctx, binary, "--query-gpu=index,uuid", "--format=csv,noheader").Output() + if err != nil { + return nil, fmt.Errorf("running %s: %w", binary, err) + } + apps, err := exec.CommandContext(ctx, binary, "--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader").Output() + if err != nil { + return nil, fmt.Errorf("running %s: %w", binary, err) + } + return parseGPUProcesses(string(gpus), string(apps)), nil +} + +// parseGPUProcesses joins nvidia-smi's GPU list (index, uuid) to its compute +// apps (gpu_uuid, pid): the apps are listed by UUID, placement speaks index. +func parseGPUProcesses(gpus, apps string) map[int][]int { + index := map[string]int{} + for _, line := range strings.Split(gpus, "\n") { + i, uuid, ok := strings.Cut(line, ",") + if !ok { + continue + } + n, err := strconv.Atoi(strings.TrimSpace(i)) + if err != nil { + continue + } + index[strings.TrimSpace(uuid)] = n + } + out := map[int][]int{} + for _, line := range strings.Split(apps, "\n") { + uuid, p, ok := strings.Cut(line, ",") + if !ok { + continue + } + pid, err := strconv.Atoi(strings.TrimSpace(p)) + i, known := index[strings.TrimSpace(uuid)] + if err != nil || !known { + continue + } + out[i] = append(out[i], pid) + } + return out +} diff --git a/internal/agent/discovery/gpuprocs_test.go b/internal/agent/discovery/gpuprocs_test.go new file mode 100644 index 0000000..723d2ad --- /dev/null +++ b/internal/agent/discovery/gpuprocs_test.go @@ -0,0 +1,21 @@ +package discovery + +import ( + "reflect" + "testing" +) + +func TestGPUProcessesAreListedByIndexNotUUID(t *testing.T) { + gpus := "0, GPU-aaa\n1, GPU-bbb\n2, GPU-ccc\n" + // Two processes on GPU 1, one on GPU 2, and a GPU this list has never + // heard of, which is dropped rather than guessed at. + apps := "GPU-bbb, 4100\nGPU-ccc, 4200\nGPU-bbb, 4300\nGPU-zzz, 9999\n" + got := parseGPUProcesses(gpus, apps) + want := map[int][]int{1: {4100, 4300}, 2: {4200}} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %v, want %v", got, want) + } + if got := parseGPUProcesses(gpus, ""); len(got) != 0 { + t.Errorf("no compute apps should be no GPU in use: %v", got) + } +} diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 831d7f9..330d6d3 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -9,6 +9,7 @@ package engines import ( + "context" "fmt" "os" "slices" @@ -146,6 +147,15 @@ type Engine interface { MetricsPath() string } +// CPUFallback is an engine that can come up on the CPU when it was placed on +// GPUs, and keep serving there without an error. The agent checks such a shard +// is on its GPUs before calling it ready. +type CPUFallback interface { + FallsBackToCPU() bool + // Devices lists the GPUs the engine's build can see, for saying why. + Devices(ctx context.Context) ([]string, error) +} + // GPUInfo is the subset of GPU facts validation needs. type GPUInfo struct { Index int @@ -232,6 +242,11 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) } func baseEnv(s ShardSpec) map[string]string { env := map[string]string{ "CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices), + // The indices above are nvidia-smi's, which counts by PCI bus. CUDA + // counts fastest first unless told otherwise, and on a node with two + // kinds of GPU the shard would get a different card than placement + // gave it — one another shard may hold. + "CUDA_DEVICE_ORDER": "PCI_BUS_ID", } if len(s.HCAs) > 0 { diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go index 16fee3e..69a87d6 100644 --- a/internal/agent/engines/llamacpp.go +++ b/internal/agent/engines/llamacpp.go @@ -1,8 +1,10 @@ package engines import ( + "context" "fmt" "os" + "os/exec" "path/filepath" "runtime" "sort" @@ -234,3 +236,38 @@ func (l *LlamaCpp) bin() string { } return "llama-server" } + +// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does +// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and +// the server comes up and answers on the CPU, at a tenth of the speed, with no +// error anywhere. Its readiness on a GPU placement needs checking separately. +func (*LlamaCpp) FallsBackToCPU() bool { return true } + +// Devices lists the GPUs this build can compute on, as llama-server's +// --list-devices names them ("CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)"). +// Empty, on a machine with GPUs, is a build whose GPU backend did not load: +// the CPU build, or a CUDA build missing a library its backend links. +func (l *LlamaCpp) Devices(ctx context.Context) ([]string, error) { + out, err := exec.CommandContext(ctx, l.bin(), "--list-devices").CombinedOutput() //nolint:gosec // the engine's own binary + if err != nil { + return nil, fmt.Errorf("%s --list-devices: %v: %s", l.bin(), err, strings.TrimSpace(string(out))) + } + return parseDevices(string(out)), nil +} + +// parseDevices reads --list-devices: the lines after "Available devices:", +// which says "(none)" when there are none. Log lines before it are skipped. +func parseDevices(out string) []string { + var devs []string + listing := false + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "Available devices"): + listing = true + case listing && line != "" && line != "(none)": + devs = append(devs, line) + } + } + return devs +} diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go index 186aa41..4f967e3 100644 --- a/internal/agent/engines/llamacpp_test.go +++ b/internal/agent/engines/llamacpp_test.go @@ -323,3 +323,14 @@ func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) { t.Error("a model with no adapters was given lora flags") } } + +func TestDevicesAreReadFromListDevices(t *testing.T) { + cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n" + if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") { + t.Errorf("CUDA build: %q", got) + } + // A CUDA build whose backend did not load says what the CPU build says. + if got := parseDevices("Available devices:\n (none)\n"); len(got) != 0 { + t.Errorf("no devices: %q", got) + } +} diff --git a/internal/agent/gpucheck.go b/internal/agent/gpucheck.go new file mode 100644 index 0000000..0e8ca32 --- /dev/null +++ b/internal/agent/gpucheck.go @@ -0,0 +1,116 @@ +package agent + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" +) + +// gpuProcesses and processGroup are variables so tests can stand in for +// nvidia-smi and /proc. +var ( + gpuProcesses = discovery.GPUProcesses + processGroup = pgidOf + // gpuCheckWait is how long a shard has, once it answers, to show up on + // its GPUs: the weights are on them by then, but nvidia-smi can lag. + gpuCheckWait = 15 * time.Second +) + +// onItsGPUs checks that a shard placed on GPUs computes on them. +// +// For an engine that falls back to the CPU when its GPU backend does not load, +// "it answers" is not "it is ready": llama.cpp built for CUDA but missing +// cuBLAS comes up on the CPU, answers at a tenth of the speed, and the GPU it +// was given sits idle and reserved. What it computes on is what nvidia-smi +// lists it on, so a shard none of whose processes hold memory on one of its +// GPUs is refused, with what the engine itself can see. +func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.ShardSpec, pid int) error { + fb, ok := eng.(engines.CPUFallback) + if !ok || !fb.FallsBackToCPU() || len(spec.GPUIndices) == 0 || pid <= 0 { + return nil + } + group := processGroup(pid) + deadline := time.Now().Add(gpuCheckWait) + var missing []int + for { + procs, err := gpuProcesses(ctx, a.cfg.Discovery.NvidiaSMI) + if err != nil || procs == nil { + // No nvidia-smi to ask is no evidence either way, and a node that + // was given GPUs has one; say so and let the shard serve. + a.log.Warn("could not check that a GPU shard runs on its GPUs", "shard", spec.ShardID, "err", err) + return nil + } + missing = missing[:0] + for _, i := range spec.GPUIndices { + if !holds(procs[i], pid, group) { + missing = append(missing, i) + } + } + if len(missing) == 0 { + return nil + } + if time.Now().After(deadline) { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + seen := "it lists no GPU at all" + if devs, err := fb.Devices(ctx); err != nil { + seen = "listing its devices failed: " + err.Error() + } else if len(devs) > 0 { + seen = "it lists " + strings.Join(devs, "; ") + } + return fmt.Errorf("placed on GPU %s but computing on the CPU: the engine holds no memory there. "+ + "Its GPU backend did not load (%s). Check the build with `ldd` on its libggml-cuda.so, "+ + "or run `llmgw doctor` on this node", joinIndices(missing), seen) +} + +// holds reports whether any of pids is the shard's process or in its group: +// an engine that computes in a child process is still the shard. +func holds(pids []int, pid, group int) bool { + for _, p := range pids { + if p == pid || (group > 0 && processGroup(p) == group) { + return true + } + } + return false +} + +// pgidOf reads a process's group from /proc; zero when it cannot. +func pgidOf(pid int) int { + b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat") + if err != nil { + return 0 + } + // The command name, in parentheses, may hold spaces; the fields after its + // closing parenthesis are state, ppid, pgrp. + s := string(b) + i := strings.LastIndexByte(s, ')') + if i < 0 { + return 0 + } + f := strings.Fields(s[i+1:]) + if len(f) < 3 { + return 0 + } + n, _ := strconv.Atoi(f[2]) + return n +} + +func joinIndices(is []int) string { + s := make([]string, len(is)) + for k, i := range is { + s[k] = strconv.Itoa(i) + } + return strings.Join(s, ", ") +} diff --git a/internal/agent/gpucheck_test.go b/internal/agent/gpucheck_test.go new file mode 100644 index 0000000..6996cdb --- /dev/null +++ b/internal/agent/gpucheck_test.go @@ -0,0 +1,91 @@ +package agent + +import ( + "context" + "log/slog" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" +) + +// fallbackEngine is an engine that comes up on the CPU when its GPU backend +// does not load, and lists the devices it can see. +type fallbackEngine struct { + engines.Engine + devices []string +} + +func (fallbackEngine) FallsBackToCPU() bool { return true } +func (f fallbackEngine) Devices(context.Context) ([]string, error) { + return f.devices, nil +} + +func TestAGPUShardComputingOnTheCPUIsNotReady(t *testing.T) { + procs := map[int][]int{} + oldProcs, oldGroup, oldWait := gpuProcesses, processGroup, gpuCheckWait + gpuProcesses = func(context.Context, string) (map[int][]int, error) { return procs, nil } + processGroup = func(pid int) int { + if pid >= 500 && pid < 600 { + return 500 // the shard's group: 500 and its children + } + return pid + } + gpuCheckWait = 0 + t.Cleanup(func() { gpuProcesses, processGroup, gpuCheckWait = oldProcs, oldGroup, oldWait }) + + a := &Agent{log: slog.Default()} + spec := engines.ShardSpec{ShardID: "s", GPUIndices: []int{0, 1}} + eng := fallbackEngine{} + + // On no GPU at all: refused, saying what the engine sees. + err := a.onItsGPUs(context.Background(), eng, spec, 500) + if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || !strings.Contains(err.Error(), "lists no GPU") { + t.Fatalf("a shard on no GPU was let through: %v", err) + } + // On one of its two: refused, naming the one it is missing. + procs[0] = []int{500} + if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil || !strings.Contains(err.Error(), "GPU 1 ") { + t.Fatalf("a shard on half its GPUs: %v", err) + } + // Another process on GPU 1 is not the shard. + procs[1] = []int{900} + if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil { + t.Fatal("another process's memory was taken for the shard's") + } + // A child of the shard holding GPU 1 is the shard. + procs[1] = []int{900, 512} + if err := a.onItsGPUs(context.Background(), eng, spec, 500); err != nil { + t.Fatalf("a shard on both its GPUs: %v", err) + } + + // An engine that fails rather than falling back is not checked, nor is a + // shard with no GPUs. + procs = map[int][]int{} + if err := a.onItsGPUs(context.Background(), struct{ engines.Engine }{}, spec, 500); err != nil { + t.Errorf("checked an engine that does not fall back: %v", err) + } + if err := a.onItsGPUs(context.Background(), eng, engines.ShardSpec{}, 500); err != nil { + t.Errorf("checked a CPU shard: %v", err) + } +} + +func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) { + calls := 0 + oldProcs, oldWait := gpuProcesses, gpuCheckWait + gpuProcesses = func(context.Context, string) (map[int][]int, error) { + calls++ + if calls < 2 { + return map[int][]int{}, nil + } + return map[int][]int{0: {77}}, nil + } + gpuCheckWait = 5 * time.Second + t.Cleanup(func() { gpuProcesses, gpuCheckWait = oldProcs, oldWait }) + + a := &Agent{log: slog.Default()} + if err := a.onItsGPUs(context.Background(), fallbackEngine{}, engines.ShardSpec{GPUIndices: []int{0}}, 77); err != nil { + t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err) + } +} diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index c319118..c7f088d 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -299,9 +299,55 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers if err != nil { return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out)) } + if b.GPUBackend != "" { + if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil { + return "", "", err + } + } return bin, firstVersionLine(out), nil } +// verifyGPUBackend checks a GPU build can compute on this node's GPUs. +// +// The binary's own ldd says nothing about it: the GPU backend is a library +// the server opens at run time, and when it cannot — cuBLAS missing, a driver +// too old — the server carries on with its CPU backend. So the build is asked +// which devices it sees, and when it sees none, ldd on the backend says why. +func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error { + out, err := in.run(ctx, nil, bin, "--list-devices") + if err != nil { + return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out)) + } + if listsDevice(string(out)) { + return nil + } + why := "its GPU backend did not load" + if lib, err := findFile(dir, b.GPUBackend); err != nil { + why = "the build has no " + b.GPUBackend + } else if out, err := in.run(ctx, nil, "ldd", lib); err == nil { + if missing := lddMissing(string(out)); len(missing) > 0 { + why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ") + } + } + return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why) +} + +// listsDevice reads llama-server --list-devices: a device line after +// "Available devices:", which holds "(none)" when there is none. +func listsDevice(out string) bool { + listing := false + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "Available devices"): + listing = true + case listing && line != "" && line != "(none)": + return true + } + } + return false +} + // packages installs the operating system packages a build needs. // // Only those missing, and from the node's own package sources: a package the diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index 5aefcb4..4afbd57 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -89,7 +89,11 @@ type fakeSystem struct { installed map[string]bool tools map[string]bool lddOut string - calls []string + // devices is what llama-server --list-devices prints, and backendLdd + // what ldd says of its GPU backend. + devices string + backendLdd string + calls []string // locked is how many more package commands find the lock held. locked int } @@ -127,8 +131,12 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str } } return nil, nil + case name == "ldd" && strings.HasSuffix(args[0], "libggml-cuda.so"): + return []byte(f.backendLdd), nil case name == "ldd": return []byte(f.lddOut), nil + case strings.HasSuffix(name, "llama-server") && args[0] == "--list-devices": + return []byte(f.devices), nil case strings.HasSuffix(name, "llama-server"): return []byte("version: 10991 (abc)\n"), nil } @@ -388,3 +396,32 @@ func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) { t.Errorf("link %q %v", target, err) } } + +func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) { + // Found in the field: libggml-cuda.so in the build, cuBLAS nowhere the + // loader looks, and llama-server serving a GPU placement on the CPU. + cuda := "bin-ubuntu-cuda-" + cudaForLlamaCpp + "-x64.tar.gz" + rel := newRelease(map[string][]byte{ + "llama-b1-" + cuda: tarGz(t, map[string]string{ + "llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil), + "cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil), + }) + gpu := ubuntu + gpu.NVIDIADriver = "570.86.15" + sys := &fakeSystem{installed: map[string]bool{"libgomp1": true}, + devices: "Available devices:\n (none)\n", + backendLdd: "\tlibcublas.so.12 => not found\n\tlibcudart.so.12 => /x/libcudart.so.12\n"} + in, _ := installer(t, rel.serve(t), sys, gpu) + + got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Failed || !strings.Contains(got[0].Detail, "sees no GPU") || + !strings.Contains(got[0].Detail, "libcublas.so.12 is not installed") { + t.Fatalf("expected a failure naming the missing library: %+v", got[0]) + } + + // The same build seeing its GPU is installed. + sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n" + if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready { + t.Fatalf("a CUDA build that sees its GPU: %+v", got[0]) + } +} diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index e5d1fd8..e6355eb 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -43,6 +43,11 @@ type Build struct { // VersionArgs is what Binary is run with to prove the build works, when // --version is not it. VersionArgs []string + // GPUBackend, when set, is the shared library the build computes on GPUs + // with, loaded at run time rather than linked: "libggml-cuda.so". The + // build must list a device with it, or it is not installed — a llama.cpp + // whose CUDA backend cannot load runs on the CPU without complaint. + GPUBackend string // FreeGB is the disk space the installation needs before it starts: a // Python engine is several gigabytes, and one that fills the disk halfway // leaves the node worse off than one that never began. @@ -234,6 +239,7 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { if h.NVIDIADriver != "" && arch == "x64" { name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp) b.Variant = "cuda-" + cudaForLlamaCpp + b.GPUBackend = "libggml-cuda.so" b.Assets = []string{ fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name), diff --git a/internal/cli/enginecheck.go b/internal/cli/enginecheck.go index bad7d12..7726570 100644 --- a/internal/cli/enginecheck.go +++ b/internal/cli/enginecheck.go @@ -3,6 +3,7 @@ package cli import ( "context" "fmt" + "io" "os/exec" "path/filepath" "sort" @@ -126,6 +127,14 @@ func newEngineCheckCmd() *cobra.Command { } sort.Strings(unknown) + // Flags accepted is not the same as running where it is placed: a + // GPU build whose backend does not load accepts every flag and + // computes on the CPU. Asked of the engine here, not of a help. + var deviceErr error + if fb, ok := eng.(engines.CPUFallback); ok && help == "" { + deviceErr = checkDevices(cmd.Context(), name, fb, out) + } + fmt.Fprintf(out, "%s: %d flags checked", name, checked) if skipped > 0 { fmt.Fprintf(out, ", %d commands skipped", skipped) @@ -137,7 +146,7 @@ func newEngineCheckCmd() *cobra.Command { return fmt.Errorf("nothing was checked; install the engine here or pass --help-output") } fmt.Fprintln(out, "every flag llmgw builds is accepted") - return nil + return deviceErr } fmt.Fprintf(out, "\n%d flags not mentioned by the program they are passed to:\n", len(unknown)) for _, f := range unknown { @@ -249,3 +258,22 @@ func isFlagChar(b byte) bool { return b == '-' || b == '_' || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') } + +// checkDevices says which GPUs an engine that can fall back to the CPU sees on +// this machine, and fails when the machine has NVIDIA GPUs and it sees none. +func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out io.Writer) error { + if _, err := exec.LookPath("nvidia-smi"); err != nil { + fmt.Fprintln(out, "no NVIDIA GPU here: which devices it computes on was not checked") + return nil + } + devs, err := fb.Devices(ctx) + if err != nil { + return err + } + if len(devs) == 0 { + return fmt.Errorf("this machine has NVIDIA GPUs and %s sees none: it would serve a GPU placement on the CPU. "+ + "Its GPU backend did not load; ldd on its libggml-cuda.so names the missing library", name) + } + fmt.Fprintf(out, "computes on: %s\n", strings.Join(devs, "; ")) + return nil +}