Do not call a GPU shard ready while it computes on the CPU

llama.cpp whose CUDA backend cannot load runs on the CPU without an
error. The CUDA build is now asked for its devices when installed, a
shard on GPUs must hold memory on each before it is ready, and
engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID
so CUDA numbers GPUs as placement does.

Refs #2 (item 2).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -20,6 +20,33 @@ is only released once its **exit criterion** is met, not when its code lands.
 Until v1.0.0 the minor version is the breaking one: configuration formats and Go
 APIs change between minors without a deprecation period.
 
+## Unreleased
+
+From the field report in issue #2, after a deployment on GPU nodes.
+
+### A GPU placement no longer runs on the CPU unnoticed
+
+llama.cpp placed on a GPU, with its CUDA backend unable to load (cuBLAS not
+where the loader looks), came up on the CPU, answered at CPU speed, and was
+reported ready while its GPU sat reserved and idle. Three checks now stop it:
+
+- **Installing** the CUDA build asks it which devices it sees
+  (`llama-server --list-devices`). None, on a node with GPUs, is a failed
+  install, with what `ldd` says `libggml-cuda.so` is missing. The binary's
+  own `ldd` could not say it: the backend is opened at run time.
+- **Starting** a shard on GPUs is ready only when its process holds memory on
+  each of them (`nvidia-smi --query-compute-apps`, its process group
+  included). One that does not is stopped and reported failed: "placed on GPU
+  0 but computing on the CPU", with the devices the engine lists. This covers
+  a hand-installed binary (`engines.llamacpp.bin`), which nothing else checks.
+- **`llmgw ctl engine-check llamacpp`**, run on a GPU node, now also fails
+  when the engine sees no GPU, and otherwise prints the devices it computes on.
+
+Shards are also given `CUDA_DEVICE_ORDER=PCI_BUS_ID`: placement numbers GPUs
+as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a
+node with two kinds of GPU, a shard could get a different card from the one it
+was given.
+
 ## v0.10.6 — 2026-10-01
 
 Published first as release candidates v0.10.6-rc.1 to rc.8; rc.5 failed its

internal/agent/agent.go

@@ -931,6 +931,14 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
 	} else {
 		err = a.sup.WaitAlive(ctx, sh)
 	}
+	if err == nil {
+		err = a.onItsGPUs(ctx, eng, spec, sh.Status().PID)
+		if err != nil {
+			// Serving, but not where it was placed: stopped, so the GPU it
+			// holds in the inventory is not taken by a CPU process.
+			_ = a.sup.Stop(ctx, shardID)
+		}
+	}
 	if err != nil {
 		report(supervisor.Failed, err.Error(), sh.Logs(20))
 		return

internal/agent/discovery/gpuprocs.go

@@ -0,0 +1,65 @@
+package discovery
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
+// nvidia-smi index: what a process actually computes on, as opposed to what it
+// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose CUDA
+// backend did not load runs on the CPU without a word, and this is where that
+// shows — the process is on no GPU's list.
+//
+// A node with no nvidia-smi returns nil and no error, as discovery does.
+func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	gpus, err := exec.CommandContext(ctx, binary, "--query-gpu=index,uuid", "--format=csv,noheader").Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	apps, err := exec.CommandContext(ctx, binary, "--query-compute-apps=gpu_uuid,pid", "--format=csv,noheader").Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	return parseGPUProcesses(string(gpus), string(apps)), nil
+}
+
+// parseGPUProcesses joins nvidia-smi's GPU list (index, uuid) to its compute
+// apps (gpu_uuid, pid): the apps are listed by UUID, placement speaks index.
+func parseGPUProcesses(gpus, apps string) map[int][]int {
+	index := map[string]int{}
+	for _, line := range strings.Split(gpus, "\n") {
+		i, uuid, ok := strings.Cut(line, ",")
+		if !ok {
+			continue
+		}
+		n, err := strconv.Atoi(strings.TrimSpace(i))
+		if err != nil {
+			continue
+		}
+		index[strings.TrimSpace(uuid)] = n
+	}
+	out := map[int][]int{}
+	for _, line := range strings.Split(apps, "\n") {
+		uuid, p, ok := strings.Cut(line, ",")
+		if !ok {
+			continue
+		}
+		pid, err := strconv.Atoi(strings.TrimSpace(p))
+		i, known := index[strings.TrimSpace(uuid)]
+		if err != nil || !known {
+			continue
+		}
+		out[i] = append(out[i], pid)
+	}
+	return out
+}

internal/agent/discovery/gpuprocs_test.go

@@ -0,0 +1,21 @@
+package discovery
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGPUProcessesAreListedByIndexNotUUID(t *testing.T) {
+	gpus := "0, GPU-aaa\n1, GPU-bbb\n2, GPU-ccc\n"
+	// Two processes on GPU 1, one on GPU 2, and a GPU this list has never
+	// heard of, which is dropped rather than guessed at.
+	apps := "GPU-bbb, 4100\nGPU-ccc, 4200\nGPU-bbb, 4300\nGPU-zzz, 9999\n"
+	got := parseGPUProcesses(gpus, apps)
+	want := map[int][]int{1: {4100, 4300}, 2: {4200}}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if got := parseGPUProcesses(gpus, ""); len(got) != 0 {
+		t.Errorf("no compute apps should be no GPU in use: %v", got)
+	}
+}

internal/agent/engines/engine.go

@@ -9,6 +9,7 @@
 package engines
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"slices"
@@ -146,6 +147,15 @@ type Engine interface {
 	MetricsPath() string
 }
 
+// CPUFallback is an engine that can come up on the CPU when it was placed on
+// GPUs, and keep serving there without an error. The agent checks such a shard
+// is on its GPUs before calling it ready.
+type CPUFallback interface {
+	FallsBackToCPU() bool
+	// Devices lists the GPUs the engine's build can see, for saying why.
+	Devices(ctx context.Context) ([]string, error)
+}
+
 // GPUInfo is the subset of GPU facts validation needs.
 type GPUInfo struct {
 	Index             int
@@ -232,6 +242,11 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) }
 func baseEnv(s ShardSpec) map[string]string {
 	env := map[string]string{
 		"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
+		// The indices above are nvidia-smi's, which counts by PCI bus. CUDA
+		// counts fastest first unless told otherwise, and on a node with two
+		// kinds of GPU the shard would get a different card than placement
+		// gave it — one another shard may hold.
+		"CUDA_DEVICE_ORDER": "PCI_BUS_ID",
 	}
 
 	if len(s.HCAs) > 0 {

internal/agent/engines/llamacpp.go

@@ -1,8 +1,10 @@
 package engines
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"sort"
@@ -234,3 +236,38 @@ func (l *LlamaCpp) bin() string {
 	}
 	return "llama-server"
 }
+
+// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does
+// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and
+// the server comes up and answers on the CPU, at a tenth of the speed, with no
+// error anywhere. Its readiness on a GPU placement needs checking separately.
+func (*LlamaCpp) FallsBackToCPU() bool { return true }
+
+// Devices lists the GPUs this build can compute on, as llama-server's
+// --list-devices names them ("CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)").
+// Empty, on a machine with GPUs, is a build whose GPU backend did not load:
+// the CPU build, or a CUDA build missing a library its backend links.
+func (l *LlamaCpp) Devices(ctx context.Context) ([]string, error) {
+	out, err := exec.CommandContext(ctx, l.bin(), "--list-devices").CombinedOutput() //nolint:gosec // the engine's own binary
+	if err != nil {
+		return nil, fmt.Errorf("%s --list-devices: %v: %s", l.bin(), err, strings.TrimSpace(string(out)))
+	}
+	return parseDevices(string(out)), nil
+}
+
+// parseDevices reads --list-devices: the lines after "Available devices:",
+// which says "(none)" when there are none. Log lines before it are skipped.
+func parseDevices(out string) []string {
+	var devs []string
+	listing := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "Available devices"):
+			listing = true
+		case listing && line != "" && line != "(none)":
+			devs = append(devs, line)
+		}
+	}
+	return devs
+}

internal/agent/engines/llamacpp_test.go

@@ -323,3 +323,14 @@ func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
 		t.Error("a model with no adapters was given lora flags")
 	}
 }
+
+func TestDevicesAreReadFromListDevices(t *testing.T) {
+	cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n  CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n  CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
+	if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") {
+		t.Errorf("CUDA build: %q", got)
+	}
+	// A CUDA build whose backend did not load says what the CPU build says.
+	if got := parseDevices("Available devices:\n  (none)\n"); len(got) != 0 {
+		t.Errorf("no devices: %q", got)
+	}
+}

internal/agent/gpucheck.go

@@ -0,0 +1,116 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// gpuProcesses and processGroup are variables so tests can stand in for
+// nvidia-smi and /proc.
+var (
+	gpuProcesses = discovery.GPUProcesses
+	processGroup = pgidOf
+	// gpuCheckWait is how long a shard has, once it answers, to show up on
+	// its GPUs: the weights are on them by then, but nvidia-smi can lag.
+	gpuCheckWait = 15 * time.Second
+)
+
+// onItsGPUs checks that a shard placed on GPUs computes on them.
+//
+// For an engine that falls back to the CPU when its GPU backend does not load,
+// "it answers" is not "it is ready": llama.cpp built for CUDA but missing
+// cuBLAS comes up on the CPU, answers at a tenth of the speed, and the GPU it
+// was given sits idle and reserved. What it computes on is what nvidia-smi
+// lists it on, so a shard none of whose processes hold memory on one of its
+// GPUs is refused, with what the engine itself can see.
+func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.ShardSpec, pid int) error {
+	fb, ok := eng.(engines.CPUFallback)
+	if !ok || !fb.FallsBackToCPU() || len(spec.GPUIndices) == 0 || pid <= 0 {
+		return nil
+	}
+	group := processGroup(pid)
+	deadline := time.Now().Add(gpuCheckWait)
+	var missing []int
+	for {
+		procs, err := gpuProcesses(ctx, a.cfg.Discovery.NvidiaSMI)
+		if err != nil || procs == nil {
+			// No nvidia-smi to ask is no evidence either way, and a node that
+			// was given GPUs has one; say so and let the shard serve.
+			a.log.Warn("could not check that a GPU shard runs on its GPUs", "shard", spec.ShardID, "err", err)
+			return nil
+		}
+		missing = missing[:0]
+		for _, i := range spec.GPUIndices {
+			if !holds(procs[i], pid, group) {
+				missing = append(missing, i)
+			}
+		}
+		if len(missing) == 0 {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+	seen := "it lists no GPU at all"
+	if devs, err := fb.Devices(ctx); err != nil {
+		seen = "listing its devices failed: " + err.Error()
+	} else if len(devs) > 0 {
+		seen = "it lists " + strings.Join(devs, "; ")
+	}
+	return fmt.Errorf("placed on GPU %s but computing on the CPU: the engine holds no memory there. "+
+		"Its GPU backend did not load (%s). Check the build with `ldd` on its libggml-cuda.so, "+
+		"or run `llmgw doctor` on this node", joinIndices(missing), seen)
+}
+
+// holds reports whether any of pids is the shard's process or in its group:
+// an engine that computes in a child process is still the shard.
+func holds(pids []int, pid, group int) bool {
+	for _, p := range pids {
+		if p == pid || (group > 0 && processGroup(p) == group) {
+			return true
+		}
+	}
+	return false
+}
+
+// pgidOf reads a process's group from /proc; zero when it cannot.
+func pgidOf(pid int) int {
+	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
+	if err != nil {
+		return 0
+	}
+	// The command name, in parentheses, may hold spaces; the fields after its
+	// closing parenthesis are state, ppid, pgrp.
+	s := string(b)
+	i := strings.LastIndexByte(s, ')')
+	if i < 0 {
+		return 0
+	}
+	f := strings.Fields(s[i+1:])
+	if len(f) < 3 {
+		return 0
+	}
+	n, _ := strconv.Atoi(f[2])
+	return n
+}
+
+func joinIndices(is []int) string {
+	s := make([]string, len(is))
+	for k, i := range is {
+		s[k] = strconv.Itoa(i)
+	}
+	return strings.Join(s, ", ")
+}

internal/agent/gpucheck_test.go

@@ -0,0 +1,91 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// fallbackEngine is an engine that comes up on the CPU when its GPU backend
+// does not load, and lists the devices it can see.
+type fallbackEngine struct {
+	engines.Engine
+	devices []string
+}
+
+func (fallbackEngine) FallsBackToCPU() bool { return true }
+func (f fallbackEngine) Devices(context.Context) ([]string, error) {
+	return f.devices, nil
+}
+
+func TestAGPUShardComputingOnTheCPUIsNotReady(t *testing.T) {
+	procs := map[int][]int{}
+	oldProcs, oldGroup, oldWait := gpuProcesses, processGroup, gpuCheckWait
+	gpuProcesses = func(context.Context, string) (map[int][]int, error) { return procs, nil }
+	processGroup = func(pid int) int {
+		if pid >= 500 && pid < 600 {
+			return 500 // the shard's group: 500 and its children
+		}
+		return pid
+	}
+	gpuCheckWait = 0
+	t.Cleanup(func() { gpuProcesses, processGroup, gpuCheckWait = oldProcs, oldGroup, oldWait })
+
+	a := &Agent{log: slog.Default()}
+	spec := engines.ShardSpec{ShardID: "s", GPUIndices: []int{0, 1}}
+	eng := fallbackEngine{}
+
+	// On no GPU at all: refused, saying what the engine sees.
+	err := a.onItsGPUs(context.Background(), eng, spec, 500)
+	if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || !strings.Contains(err.Error(), "lists no GPU") {
+		t.Fatalf("a shard on no GPU was let through: %v", err)
+	}
+	// On one of its two: refused, naming the one it is missing.
+	procs[0] = []int{500}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil || !strings.Contains(err.Error(), "GPU 1 ") {
+		t.Fatalf("a shard on half its GPUs: %v", err)
+	}
+	// Another process on GPU 1 is not the shard.
+	procs[1] = []int{900}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil {
+		t.Fatal("another process's memory was taken for the shard's")
+	}
+	// A child of the shard holding GPU 1 is the shard.
+	procs[1] = []int{900, 512}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err != nil {
+		t.Fatalf("a shard on both its GPUs: %v", err)
+	}
+
+	// An engine that fails rather than falling back is not checked, nor is a
+	// shard with no GPUs.
+	procs = map[int][]int{}
+	if err := a.onItsGPUs(context.Background(), struct{ engines.Engine }{}, spec, 500); err != nil {
+		t.Errorf("checked an engine that does not fall back: %v", err)
+	}
+	if err := a.onItsGPUs(context.Background(), eng, engines.ShardSpec{}, 500); err != nil {
+		t.Errorf("checked a CPU shard: %v", err)
+	}
+}
+
+func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) {
+	calls := 0
+	oldProcs, oldWait := gpuProcesses, gpuCheckWait
+	gpuProcesses = func(context.Context, string) (map[int][]int, error) {
+		calls++
+		if calls < 2 {
+			return map[int][]int{}, nil
+		}
+		return map[int][]int{0: {77}}, nil
+	}
+	gpuCheckWait = 5 * time.Second
+	t.Cleanup(func() { gpuProcesses, gpuCheckWait = oldProcs, oldWait })
+
+	a := &Agent{log: slog.Default()}
+	if err := a.onItsGPUs(context.Background(), fallbackEngine{}, engines.ShardSpec{GPUIndices: []int{0}}, 77); err != nil {
+		t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err)
+	}
+}

internal/agent/provision/install.go

@@ -299,9 +299,55 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers
 	if err != nil {
 		return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
 	}
+	if b.GPUBackend != "" {
+		if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil {
+			return "", "", err
+		}
+	}
 	return bin, firstVersionLine(out), nil
 }
 
+// verifyGPUBackend checks a GPU build can compute on this node's GPUs.
+//
+// The binary's own ldd says nothing about it: the GPU backend is a library
+// the server opens at run time, and when it cannot — cuBLAS missing, a driver
+// too old — the server carries on with its CPU backend. So the build is asked
+// which devices it sees, and when it sees none, ldd on the backend says why.
+func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error {
+	out, err := in.run(ctx, nil, bin, "--list-devices")
+	if err != nil {
+		return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out))
+	}
+	if listsDevice(string(out)) {
+		return nil
+	}
+	why := "its GPU backend did not load"
+	if lib, err := findFile(dir, b.GPUBackend); err != nil {
+		why = "the build has no " + b.GPUBackend
+	} else if out, err := in.run(ctx, nil, "ldd", lib); err == nil {
+		if missing := lddMissing(string(out)); len(missing) > 0 {
+			why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ")
+		}
+	}
+	return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why)
+}
+
+// listsDevice reads llama-server --list-devices: a device line after
+// "Available devices:", which holds "(none)" when there is none.
+func listsDevice(out string) bool {
+	listing := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "Available devices"):
+			listing = true
+		case listing && line != "" && line != "(none)":
+			return true
+		}
+	}
+	return false
+}
+
 // packages installs the operating system packages a build needs.
 //
 // Only those missing, and from the node's own package sources: a package the

internal/agent/provision/install_test.go

@@ -89,7 +89,11 @@ type fakeSystem struct {
 	installed map[string]bool
 	tools     map[string]bool
 	lddOut    string
-	calls     []string
+	// devices is what llama-server --list-devices prints, and backendLdd
+	// what ldd says of its GPU backend.
+	devices    string
+	backendLdd string
+	calls      []string
 	// locked is how many more package commands find the lock held.
 	locked int
 }
@@ -127,8 +131,12 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
 			}
 		}
 		return nil, nil
+	case name == "ldd" && strings.HasSuffix(args[0], "libggml-cuda.so"):
+		return []byte(f.backendLdd), nil
 	case name == "ldd":
 		return []byte(f.lddOut), nil
+	case strings.HasSuffix(name, "llama-server") && args[0] == "--list-devices":
+		return []byte(f.devices), nil
 	case strings.HasSuffix(name, "llama-server"):
 		return []byte("version: 10991 (abc)\n"), nil
 	}
@@ -388,3 +396,32 @@ func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) {
 		t.Errorf("link %q %v", target, err)
 	}
 }
+
+func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
+	// Found in the field: libggml-cuda.so in the build, cuBLAS nowhere the
+	// loader looks, and llama-server serving a GPU placement on the CPU.
+	cuda := "bin-ubuntu-cuda-" + cudaForLlamaCpp + "-x64.tar.gz"
+	rel := newRelease(map[string][]byte{
+		"llama-b1-" + cuda: tarGz(t, map[string]string{
+			"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
+		"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
+	})
+	gpu := ubuntu
+	gpu.NVIDIADriver = "570.86.15"
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
+		devices:    "Available devices:\n  (none)\n",
+		backendLdd: "\tlibcublas.so.12 => not found\n\tlibcudart.so.12 => /x/libcudart.so.12\n"}
+	in, _ := installer(t, rel.serve(t), sys, gpu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "sees no GPU") ||
+		!strings.Contains(got[0].Detail, "libcublas.so.12 is not installed") {
+		t.Fatalf("expected a failure naming the missing library: %+v", got[0])
+	}
+
+	// The same build seeing its GPU is installed.
+	sys.devices = "Available devices:\n  CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
+	if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
+		t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
+	}
+}

internal/agent/provision/recipe.go

@@ -43,6 +43,11 @@ type Build struct {
 	// VersionArgs is what Binary is run with to prove the build works, when
 	// --version is not it.
 	VersionArgs []string
+	// GPUBackend, when set, is the shared library the build computes on GPUs
+	// with, loaded at run time rather than linked: "libggml-cuda.so". The
+	// build must list a device with it, or it is not installed — a llama.cpp
+	// whose CUDA backend cannot load runs on the CPU without complaint.
+	GPUBackend string
 	// FreeGB is the disk space the installation needs before it starts: a
 	// Python engine is several gigabytes, and one that fills the disk halfway
 	// leaves the node worse off than one that never began.
@@ -234,6 +239,7 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
 	if h.NVIDIADriver != "" && arch == "x64" {
 		name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
 		b.Variant = "cuda-" + cudaForLlamaCpp
+		b.GPUBackend = "libggml-cuda.so"
 		b.Assets = []string{
 			fmt.Sprintf("llama-%s-%s", version, name),
 			fmt.Sprintf("cudart-llama-%s-%s", version, name),

internal/cli/enginecheck.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"fmt"
+	"io"
 	"os/exec"
 	"path/filepath"
 	"sort"
@@ -126,6 +127,14 @@ func newEngineCheckCmd() *cobra.Command {
 			}
 			sort.Strings(unknown)
 
+			// Flags accepted is not the same as running where it is placed: a
+			// GPU build whose backend does not load accepts every flag and
+			// computes on the CPU. Asked of the engine here, not of a help.
+			var deviceErr error
+			if fb, ok := eng.(engines.CPUFallback); ok && help == "" {
+				deviceErr = checkDevices(cmd.Context(), name, fb, out)
+			}
+
 			fmt.Fprintf(out, "%s: %d flags checked", name, checked)
 			if skipped > 0 {
 				fmt.Fprintf(out, ", %d commands skipped", skipped)
@@ -137,7 +146,7 @@ func newEngineCheckCmd() *cobra.Command {
 					return fmt.Errorf("nothing was checked; install the engine here or pass --help-output")
 				}
 				fmt.Fprintln(out, "every flag llmgw builds is accepted")
-				return nil
+				return deviceErr
 			}
 			fmt.Fprintf(out, "\n%d flags not mentioned by the program they are passed to:\n", len(unknown))
 			for _, f := range unknown {
@@ -249,3 +258,22 @@ func isFlagChar(b byte) bool {
 	return b == '-' || b == '_' || (b >= 'a' && b <= 'z') ||
 		(b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')
 }
+
+// checkDevices says which GPUs an engine that can fall back to the CPU sees on
+// this machine, and fails when the machine has NVIDIA GPUs and it sees none.
+func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out io.Writer) error {
+	if _, err := exec.LookPath("nvidia-smi"); err != nil {
+		fmt.Fprintln(out, "no NVIDIA GPU here: which devices it computes on was not checked")
+		return nil
+	}
+	devs, err := fb.Devices(ctx)
+	if err != nil {
+		return err
+	}
+	if len(devs) == 0 {
+		return fmt.Errorf("this machine has NVIDIA GPUs and %s sees none: it would serve a GPU placement on the CPU. "+
+			"Its GPU backend did not load; ldd on its libggml-cuda.so names the missing library", name)
+	}
+	fmt.Fprintf(out, "computes on: %s\n", strings.Join(devs, "; "))
+	return nil
+}