From c2fedeb32602b849f2d5ee1b0351a784c44ac372 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 19 Nov 2026 12:29:02 +0100 Subject: [PATCH] An NVIDIA card with no driver is reported, not onboarded as a CPU node The agent found GPUs through nvidia-smi alone, so a machine whose driver was never installed looked exactly like one with no card and joined quietly as a CPU node. Only init looked at the PCI bus. The host discovery now reads the bus too (vendor 0x10de, class 0x03) and carries it to the control plane. The doctor fails the pre-install checks on a card without a driver, with the install and the reboot named, and tells a driver that is installed but not loaded (Secure Boot, nouveau) apart from a missing one. A node already in shows the warning in ctl nodes and the console. init uses the same check. Co-Authored-By: Claude Fable 5.1 --- api/gen/llmgw/v1/agent.pb.go | 20 +++++++++--- api/proto/llmgw/v1/agent.proto | 4 +++ docs/install.md | 5 ++- docs/node-onboarding.md | 6 ++++ internal/agent/agent.go | 2 +- internal/agent/discovery/host.go | 31 +++++++++++++++++++ internal/agent/discovery/host_driver_test.go | 32 ++++++++++++++++++++ internal/cli/serverinit_cluster.go | 20 +----------- internal/control/adminapi/api.go | 4 +++ internal/control/agentsvc/convert.go | 1 + internal/doctor/doctor.go | 15 ++++++++- internal/doctor/doctor_test.go | 24 +++++++++++++++ 12 files changed, 138 insertions(+), 26 deletions(-) diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 32ef177..be710ed 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1246,8 +1246,12 @@ type Host struct { // given a few threads of a large machine reads at a few times this, not at // the whole machine's rate. 0 unknown. MemBandwidthCoreGbs float64 `protobuf:"fixed64,8,opt,name=mem_bandwidth_core_gbs,json=memBandwidthCoreGbs,proto3" json:"mem_bandwidth_core_gbs,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // nvidia_device is whether an NVIDIA display controller sits on the PCI + // bus, driver or not. With nvidia_driver empty it is a card nothing can + // use yet, which must be said rather than counted as no card at all. + NvidiaDevice bool `protobuf:"varint,9,opt,name=nvidia_device,json=nvidiaDevice,proto3" json:"nvidia_device,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Host) Reset() { @@ -1336,6 +1340,13 @@ func (x *Host) GetMemBandwidthCoreGbs() float64 { return 0 } +func (x *Host) GetNvidiaDevice() bool { + if x != nil { + return x.NvidiaDevice + } + return false +} + // Storage is one filesystem a node keeps weights on. // // Per filesystem rather than per configured directory: free space belongs to @@ -6052,7 +6063,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb7\x02\n" + "\x04Host\x12\x13\n" + "\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" + "\n" + @@ -6063,7 +6074,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" + "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\x12*\n" + "\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\x123\n" + - "\x16mem_bandwidth_core_gbs\x18\b \x01(\x01R\x13memBandwidthCoreGbs\"u\n" + + "\x16mem_bandwidth_core_gbs\x18\b \x01(\x01R\x13memBandwidthCoreGbs\x12#\n" + + "\rnvidia_device\x18\t \x01(\bR\fnvidiaDevice\"u\n" + "\aStorage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index f092da6..16a80c3 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -247,6 +247,10 @@ message Host { // given a few threads of a large machine reads at a few times this, not at // the whole machine's rate. 0 unknown. double mem_bandwidth_core_gbs = 8; + // nvidia_device is whether an NVIDIA display controller sits on the PCI + // bus, driver or not. With nvidia_driver empty it is a card nothing can + // use yet, which must be said rather than counted as no card at all. + bool nvidia_device = 9; } // Storage is one filesystem a node keeps weights on. diff --git a/docs/install.md b/docs/install.md index 041c9c6..b23c1e9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -667,7 +667,10 @@ set `discovery.nvidia_smi` in the agent configuration rather than moving it. A node with no GPU at all is fine and is not an error. It reports zero cards and can host CPU profiles, which is a good way to try llmgw before committing -hardware to it. +hardware to it. A node with an NVIDIA card and no driver is an error: adding it +stops at the checks and says so, and a node already added shows the warning in +`llmgw ctl nodes` and the console until the driver is installed and its agent +restarted. ### NVLink, if your GPUs have it diff --git a/docs/node-onboarding.md b/docs/node-onboarding.md index 385aa62..5c1797c 100644 --- a/docs/node-onboarding.md +++ b/docs/node-onboarding.md @@ -181,6 +181,12 @@ Both default to the usual places. Containerized driver installs routinely put `nvidia-smi` somewhere else, and an agent that cannot find it registers with no GPUs — which looks exactly like a node that has none. +A node with no driver at all is told apart from one with no card: the PCI bus +is read as well, and an NVIDIA card there with no driver loaded is a failure in +the checks before the agent is installed (what to install, and that a reboot +follows), and a warning on the node in `llmgw ctl nodes` and the console +afterwards. It serves on CPU until the driver is in and its agent restarted. + ## Removing a node ``` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 5c351db..993c19c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -596,7 +596,7 @@ func (a *Agent) session(ctx context.Context, server string) error { a.forgetSent() reg.Host = &llmgwv1.Host{ OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename, - Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, + Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, NvidiaDevice: h.NVIDIADevice, // What is measured by now, not what the startup scan found before // the measurement ended: a reconnect would otherwise say 0. MemBandwidthGbs: a.memBandwidth().All, diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index a9a840e..6ac2451 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -3,6 +3,7 @@ package discovery import ( "bufio" "os" + "path/filepath" "regexp" "runtime" "strconv" @@ -28,6 +29,10 @@ type Host struct { CPUFlags []string `json:"cpu_flags,omitempty"` // NVIDIADriver is the loaded driver's version, empty when there is none. NVIDIADriver string `json:"nvidia_driver,omitempty"` + // NVIDIADevice is whether an NVIDIA display controller is on the PCI bus, + // driver or not. True with NVIDIADriver empty is a card without a driver: + // a machine that would otherwise serve on CPU with nothing saying why. + NVIDIADevice bool `json:"nvidia_device,omitempty"` // MinComputeCapability is the oldest GPU's, 0 when there are none or it // is not known. Not read here but from the GPU inventory, by the agent. MinComputeCapability float64 `json:"min_compute_capability,omitempty"` @@ -75,9 +80,35 @@ func DiscoverHost() Host { h.Arch = debianArch(runtime.GOARCH) h.CPUFlags = cpuFlagsFrom("/proc/cpuinfo") h.NVIDIADriver = nvidiaDriverFrom("/proc/driver/nvidia/version") + h.NVIDIADevice = HasNVIDIADevice() return h } +// CardWithoutDriver reports an NVIDIA card on the PCI bus with no driver +// loaded: a node that looks like a CPU node to nvidia-smi and is not one. +func (h Host) CardWithoutDriver() bool { return h.NVIDIADevice && h.NVIDIADriver == "" } + +// HasNVIDIADevice reports an NVIDIA display or 3D controller on the PCI bus, +// read from sysfs, which needs no driver. +func HasNVIDIADevice() bool { return hasNVIDIADeviceIn("/sys/bus/pci/devices") } + +func hasNVIDIADeviceIn(root string) bool { + vendors, _ := filepath.Glob(filepath.Join(root, "*", "vendor")) + for _, v := range vendors { + b, err := os.ReadFile(v) //nolint:gosec // sysfs + if err != nil || strings.TrimSpace(string(b)) != "0x10de" { + continue + } + class, _ := os.ReadFile(filepath.Join(filepath.Dir(v), "class")) //nolint:gosec // sysfs + // 0x03xxxx is a display or 3D controller; NVIDIA also makes bridges + // and audio devices. + if strings.HasPrefix(strings.TrimSpace(string(class)), "0x03") { + return true + } + } + return false +} + func hostFromOSRelease(path string) Host { var h Host f, err := os.Open(path) //nolint:gosec // a fixed system path diff --git a/internal/agent/discovery/host_driver_test.go b/internal/agent/discovery/host_driver_test.go index c05bdee..81d8d0b 100644 --- a/internal/agent/discovery/host_driver_test.go +++ b/internal/agent/discovery/host_driver_test.go @@ -23,3 +23,35 @@ func TestNVIDIADriverFrom(t *testing.T) { } } } + +func TestACardOnTheBusIsSeenWithoutADriver(t *testing.T) { + root := t.TempDir() + dev := func(name, vendor, class string) { + d := filepath.Join(root, name) + if err := os.MkdirAll(d, 0o750); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(d, "vendor"), []byte(vendor+"\n"), 0o600) + _ = os.WriteFile(filepath.Join(d, "class"), []byte(class+"\n"), 0o600) + } + dev("0000:00:1f.0", "0x8086", "0x060100") // the chipset + if hasNVIDIADeviceIn(root) { + t.Fatal("a machine with no NVIDIA device said it had one") + } + dev("0000:01:00.1", "0x10de", "0x040300") // the card's HDMI audio alone is not a GPU + if hasNVIDIADeviceIn(root) { + t.Fatal("NVIDIA's audio device was taken for a GPU") + } + dev("0000:01:00.0", "0x10de", "0x030000") + if !hasNVIDIADeviceIn(root) { + t.Fatal("a VGA controller of NVIDIA's was not seen") + } + h := Host{NVIDIADevice: true} + if !h.CardWithoutDriver() { + t.Error("a card with no driver version is a card without a driver") + } + h.NVIDIADriver = "580.82.07" + if h.CardWithoutDriver() { + t.Error("a card with a driver was reported as without one") + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 29fb783..fdfb37d 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -329,7 +329,7 @@ func (in *serverInit) ensureNode(ctx context.Context) (string, error) { if classes, n := gpuClasses(ctx); n > 0 { in.kept("%d NVIDIA GPU(s) here: %s — the node will use them", n, strings.Join(classes, ", ")) - } else if hasNVIDIADevice() { + } else if discovery.HasNVIDIADevice() { fmt.Fprintf(in.out, " !! an NVIDIA card is on the PCI bus but nvidia-smi does not answer: "+ "install the driver and run init again to serve on it; until then this node is CPU only\n") } else { @@ -434,24 +434,6 @@ labels: return nodeID, nil } -// hasNVIDIADevice reports an NVIDIA card on the PCI bus, driver or not, so a -// machine whose GPU is invisible for want of a driver says so instead of -// quietly serving on CPU. -func hasNVIDIADevice() bool { - vendors, _ := filepath.Glob("/sys/bus/pci/devices/*/vendor") - for _, v := range vendors { - b, err := os.ReadFile(v) - if err == nil && strings.TrimSpace(string(b)) == "0x10de" { - class, _ := os.ReadFile(filepath.Join(filepath.Dir(v), "class")) - // 0x03xxxx is a display or 3D controller; NVIDIA also makes bridges. - if strings.HasPrefix(strings.TrimSpace(string(class)), "0x03") { - return true - } - } - } - return false -} - // --- the starter model --- // starterModel is the catalog entry: a GPU profile first when there is a card, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 8ecb420..e8e783e 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -754,6 +754,10 @@ func (a *API) Nodes() []NodeView { } if h := n.Host; h.OSID != "" { v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch) + if h.CardWithoutDriver() { + v.Warnings = append(v.Warnings, "an NVIDIA card is on the PCI bus but no driver is loaded: "+ + "this node serves on CPU only (install the driver, reboot, check nvidia-smi, restart the agent)") + } if h.Arch == "amd64" && !h.HasCPUFlag("avx2") { v.Warnings = append(v.Warnings, "no AVX2: inference runs several times slower "+ "(a VM needs CPU type \"host\")") diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 0227be9..005384c 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -36,6 +36,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { n.Host = discovery.Host{ OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(), Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(), + NVIDIADevice: h.GetNvidiaDevice(), MemBandwidthGBs: h.GetMemBandwidthGbs(), MemBandwidthCoreGBs: h.GetMemBandwidthCoreGbs(), } } diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 53c9592..0f75d86 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -122,6 +122,10 @@ type Options struct { } // Collect reads the node. +// driverFix is what to do about a card with no working driver. +const driverFix = "install the NVIDIA driver the card needs (on Ubuntu: apt-get install nvidia-driver-580-open), reboot, " + + "and nvidia-smi must list it; then add the node again, or restart its agent if it was added already" + func Collect(ctx context.Context, o Options) Facts { f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}} f.Hostname, _ = os.Hostname() @@ -372,10 +376,19 @@ func Evaluate(f Facts) []Finding { // --- GPUs --- switch { + case !f.SMIFound && f.Host.NVIDIADevice: + add(Fail, "gpu", "an NVIDIA card is on the PCI bus but no driver is installed: this node would serve on CPU only", + driverFix) case !f.SMIFound: - add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "") + add(Info, "gpu", "no nvidia-smi: a CPU node", "") + case f.GPUError != "" && f.Host.CardWithoutDriver(): + add(Fail, "gpu", "an NVIDIA card is on the PCI bus but its driver is not loaded: "+f.GPUError, + "reboot if the driver was just installed; with Secure Boot on, the module must be signed (mokutil --sb-state); "+ + "check that nouveau is not holding the card (lsmod | grep nouveau)") case f.GPUError != "": add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand") + case len(f.GPUs) == 0 && f.Host.NVIDIADevice: + add(Fail, "gpu", "an NVIDIA card is on the PCI bus but nvidia-smi reports no GPUs", driverFix) case len(f.GPUs) == 0: add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "") default: diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 896fa2c..1ce70d4 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -276,3 +276,27 @@ func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) { t.Errorf("%+v", r) } } + +func TestACardWithoutADriverIsAFailureNotACPUNode(t *testing.T) { + // The mistake this guards: a GPU node whose driver was never installed + // onboarded quietly as a CPU node, with nothing anywhere saying why. + f := healthy() + f.SMIFound, f.GPUs, f.GPUError = false, nil, "" + f.Host.NVIDIADevice, f.Host.NVIDIADriver = true, "" + fs := Evaluate(f) + has(t, fs, Fail, "an NVIDIA card is on the PCI bus but no driver is installed") + + f.Host.NVIDIADevice = false + fs = Evaluate(f) + if bad := levels(fs, Fail); len(bad) > 0 { + t.Errorf("a plain CPU node was faulted: %v", bad) + } + has(t, fs, Info, "no nvidia-smi: a CPU node") + + // nvidia-smi installed, the module not loaded: the message points at the + // usual reasons rather than at a generic "run nvidia-smi". + f.SMIFound, f.GPUError = true, "running nvidia-smi: exit status 9" + f.Host.NVIDIADevice = true + fs = Evaluate(f) + has(t, fs, Fail, "its driver is not loaded") +}