diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 32ef177..be710ed 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1246,8 +1246,12 @@ type Host struct { // given a few threads of a large machine reads at a few times this, not at // the whole machine's rate. 0 unknown. MemBandwidthCoreGbs float64 `protobuf:"fixed64,8,opt,name=mem_bandwidth_core_gbs,json=memBandwidthCoreGbs,proto3" json:"mem_bandwidth_core_gbs,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // nvidia_device is whether an NVIDIA display controller sits on the PCI + // bus, driver or not. With nvidia_driver empty it is a card nothing can + // use yet, which must be said rather than counted as no card at all. + NvidiaDevice bool `protobuf:"varint,9,opt,name=nvidia_device,json=nvidiaDevice,proto3" json:"nvidia_device,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Host) Reset() { @@ -1336,6 +1340,13 @@ func (x *Host) GetMemBandwidthCoreGbs() float64 { return 0 } +func (x *Host) GetNvidiaDevice() bool { + if x != nil { + return x.NvidiaDevice + } + return false +} + // Storage is one filesystem a node keeps weights on. // // Per filesystem rather than per configured directory: free space belongs to @@ -6052,7 +6063,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x92\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb7\x02\n" + "\x04Host\x12\x13\n" + "\x05os_id\x18\x01 \x01(\tR\x04osId\x12\x1d\n" + "\n" + @@ -6063,7 +6074,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\tcpu_flags\x18\x05 \x03(\tR\bcpuFlags\x12#\n" + "\rnvidia_driver\x18\x06 \x01(\tR\fnvidiaDriver\x12*\n" + "\x11mem_bandwidth_gbs\x18\a \x01(\x01R\x0fmemBandwidthGbs\x123\n" + - "\x16mem_bandwidth_core_gbs\x18\b \x01(\x01R\x13memBandwidthCoreGbs\"u\n" + + "\x16mem_bandwidth_core_gbs\x18\b \x01(\x01R\x13memBandwidthCoreGbs\x12#\n" + + "\rnvidia_device\x18\t \x01(\bR\fnvidiaDevice\"u\n" + "\aStorage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + "\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index f092da6..16a80c3 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -247,6 +247,10 @@ message Host { // given a few threads of a large machine reads at a few times this, not at // the whole machine's rate. 0 unknown. double mem_bandwidth_core_gbs = 8; + // nvidia_device is whether an NVIDIA display controller sits on the PCI + // bus, driver or not. With nvidia_driver empty it is a card nothing can + // use yet, which must be said rather than counted as no card at all. + bool nvidia_device = 9; } // Storage is one filesystem a node keeps weights on. diff --git a/docs/install.md b/docs/install.md index 041c9c6..b23c1e9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -667,7 +667,10 @@ set `discovery.nvidia_smi` in the agent configuration rather than moving it. A node with no GPU at all is fine and is not an error. It reports zero cards and can host CPU profiles, which is a good way to try llmgw before committing -hardware to it. +hardware to it. A node with an NVIDIA card and no driver is an error: adding it +stops at the checks and says so, and a node already added shows the warning in +`llmgw ctl nodes` and the console until the driver is installed and its agent +restarted. ### NVLink, if your GPUs have it diff --git a/docs/node-onboarding.md b/docs/node-onboarding.md index 385aa62..5c1797c 100644 --- a/docs/node-onboarding.md +++ b/docs/node-onboarding.md @@ -181,6 +181,12 @@ Both default to the usual places. Containerized driver installs routinely put `nvidia-smi` somewhere else, and an agent that cannot find it registers with no GPUs — which looks exactly like a node that has none. +A node with no driver at all is told apart from one with no card: the PCI bus +is read as well, and an NVIDIA card there with no driver loaded is a failure in +the checks before the agent is installed (what to install, and that a reboot +follows), and a warning on the node in `llmgw ctl nodes` and the console +afterwards. It serves on CPU until the driver is in and its agent restarted. + ## Removing a node ``` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 5c351db..993c19c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -596,7 +596,7 @@ func (a *Agent) session(ctx context.Context, server string) error { a.forgetSent() reg.Host = &llmgwv1.Host{ OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename, - Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, + Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, NvidiaDevice: h.NVIDIADevice, // What is measured by now, not what the startup scan found before // the measurement ended: a reconnect would otherwise say 0. MemBandwidthGbs: a.memBandwidth().All, diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index a9a840e..6ac2451 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -3,6 +3,7 @@ package discovery import ( "bufio" "os" + "path/filepath" "regexp" "runtime" "strconv" @@ -28,6 +29,10 @@ type Host struct { CPUFlags []string `json:"cpu_flags,omitempty"` // NVIDIADriver is the loaded driver's version, empty when there is none. NVIDIADriver string `json:"nvidia_driver,omitempty"` + // NVIDIADevice is whether an NVIDIA display controller is on the PCI bus, + // driver or not. True with NVIDIADriver empty is a card without a driver: + // a machine that would otherwise serve on CPU with nothing saying why. + NVIDIADevice bool `json:"nvidia_device,omitempty"` // MinComputeCapability is the oldest GPU's, 0 when there are none or it // is not known. Not read here but from the GPU inventory, by the agent. MinComputeCapability float64 `json:"min_compute_capability,omitempty"` @@ -75,9 +80,35 @@ func DiscoverHost() Host { h.Arch = debianArch(runtime.GOARCH) h.CPUFlags = cpuFlagsFrom("/proc/cpuinfo") h.NVIDIADriver = nvidiaDriverFrom("/proc/driver/nvidia/version") + h.NVIDIADevice = HasNVIDIADevice() return h } +// CardWithoutDriver reports an NVIDIA card on the PCI bus with no driver +// loaded: a node that looks like a CPU node to nvidia-smi and is not one. +func (h Host) CardWithoutDriver() bool { return h.NVIDIADevice && h.NVIDIADriver == "" } + +// HasNVIDIADevice reports an NVIDIA display or 3D controller on the PCI bus, +// read from sysfs, which needs no driver. +func HasNVIDIADevice() bool { return hasNVIDIADeviceIn("/sys/bus/pci/devices") } + +func hasNVIDIADeviceIn(root string) bool { + vendors, _ := filepath.Glob(filepath.Join(root, "*", "vendor")) + for _, v := range vendors { + b, err := os.ReadFile(v) //nolint:gosec // sysfs + if err != nil || strings.TrimSpace(string(b)) != "0x10de" { + continue + } + class, _ := os.ReadFile(filepath.Join(filepath.Dir(v), "class")) //nolint:gosec // sysfs + // 0x03xxxx is a display or 3D controller; NVIDIA also makes bridges + // and audio devices. + if strings.HasPrefix(strings.TrimSpace(string(class)), "0x03") { + return true + } + } + return false +} + func hostFromOSRelease(path string) Host { var h Host f, err := os.Open(path) //nolint:gosec // a fixed system path diff --git a/internal/agent/discovery/host_driver_test.go b/internal/agent/discovery/host_driver_test.go index c05bdee..81d8d0b 100644 --- a/internal/agent/discovery/host_driver_test.go +++ b/internal/agent/discovery/host_driver_test.go @@ -23,3 +23,35 @@ func TestNVIDIADriverFrom(t *testing.T) { } } } + +func TestACardOnTheBusIsSeenWithoutADriver(t *testing.T) { + root := t.TempDir() + dev := func(name, vendor, class string) { + d := filepath.Join(root, name) + if err := os.MkdirAll(d, 0o750); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(d, "vendor"), []byte(vendor+"\n"), 0o600) + _ = os.WriteFile(filepath.Join(d, "class"), []byte(class+"\n"), 0o600) + } + dev("0000:00:1f.0", "0x8086", "0x060100") // the chipset + if hasNVIDIADeviceIn(root) { + t.Fatal("a machine with no NVIDIA device said it had one") + } + dev("0000:01:00.1", "0x10de", "0x040300") // the card's HDMI audio alone is not a GPU + if hasNVIDIADeviceIn(root) { + t.Fatal("NVIDIA's audio device was taken for a GPU") + } + dev("0000:01:00.0", "0x10de", "0x030000") + if !hasNVIDIADeviceIn(root) { + t.Fatal("a VGA controller of NVIDIA's was not seen") + } + h := Host{NVIDIADevice: true} + if !h.CardWithoutDriver() { + t.Error("a card with no driver version is a card without a driver") + } + h.NVIDIADriver = "580.82.07" + if h.CardWithoutDriver() { + t.Error("a card with a driver was reported as without one") + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 29fb783..fdfb37d 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -329,7 +329,7 @@ func (in *serverInit) ensureNode(ctx context.Context) (string, error) { if classes, n := gpuClasses(ctx); n > 0 { in.kept("%d NVIDIA GPU(s) here: %s — the node will use them", n, strings.Join(classes, ", ")) - } else if hasNVIDIADevice() { + } else if discovery.HasNVIDIADevice() { fmt.Fprintf(in.out, " !! an NVIDIA card is on the PCI bus but nvidia-smi does not answer: "+ "install the driver and run init again to serve on it; until then this node is CPU only\n") } else { @@ -434,24 +434,6 @@ labels: return nodeID, nil } -// hasNVIDIADevice reports an NVIDIA card on the PCI bus, driver or not, so a -// machine whose GPU is invisible for want of a driver says so instead of -// quietly serving on CPU. -func hasNVIDIADevice() bool { - vendors, _ := filepath.Glob("/sys/bus/pci/devices/*/vendor") - for _, v := range vendors { - b, err := os.ReadFile(v) - if err == nil && strings.TrimSpace(string(b)) == "0x10de" { - class, _ := os.ReadFile(filepath.Join(filepath.Dir(v), "class")) - // 0x03xxxx is a display or 3D controller; NVIDIA also makes bridges. - if strings.HasPrefix(strings.TrimSpace(string(class)), "0x03") { - return true - } - } - } - return false -} - // --- the starter model --- // starterModel is the catalog entry: a GPU profile first when there is a card, diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 8ecb420..e8e783e 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -754,6 +754,10 @@ func (a *API) Nodes() []NodeView { } if h := n.Host; h.OSID != "" { v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch) + if h.CardWithoutDriver() { + v.Warnings = append(v.Warnings, "an NVIDIA card is on the PCI bus but no driver is loaded: "+ + "this node serves on CPU only (install the driver, reboot, check nvidia-smi, restart the agent)") + } if h.Arch == "amd64" && !h.HasCPUFlag("avx2") { v.Warnings = append(v.Warnings, "no AVX2: inference runs several times slower "+ "(a VM needs CPU type \"host\")") diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 0227be9..005384c 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -36,6 +36,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { n.Host = discovery.Host{ OSID: h.GetOsId(), OSVersion: h.GetOsVersion(), OSCodename: h.GetOsCodename(), Arch: h.GetArch(), CPUFlags: h.GetCpuFlags(), NVIDIADriver: h.GetNvidiaDriver(), + NVIDIADevice: h.GetNvidiaDevice(), MemBandwidthGBs: h.GetMemBandwidthGbs(), MemBandwidthCoreGBs: h.GetMemBandwidthCoreGbs(), } } diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 53c9592..0f75d86 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -122,6 +122,10 @@ type Options struct { } // Collect reads the node. +// driverFix is what to do about a card with no working driver. +const driverFix = "install the NVIDIA driver the card needs (on Ubuntu: apt-get install nvidia-driver-580-open), reboot, " + + "and nvidia-smi must list it; then add the node again, or restart its agent if it was added already" + func Collect(ctx context.Context, o Options) Facts { f := Facts{At: time.Now().UTC(), Modules: map[string]bool{}, Engines: map[string]string{}} f.Hostname, _ = os.Hostname() @@ -372,10 +376,19 @@ func Evaluate(f Facts) []Finding { // --- GPUs --- switch { + case !f.SMIFound && f.Host.NVIDIADevice: + add(Fail, "gpu", "an NVIDIA card is on the PCI bus but no driver is installed: this node would serve on CPU only", + driverFix) case !f.SMIFound: - add(Info, "gpu", "no nvidia-smi: a CPU node, or the driver is not installed", "") + add(Info, "gpu", "no nvidia-smi: a CPU node", "") + case f.GPUError != "" && f.Host.CardWithoutDriver(): + add(Fail, "gpu", "an NVIDIA card is on the PCI bus but its driver is not loaded: "+f.GPUError, + "reboot if the driver was just installed; with Secure Boot on, the module must be signed (mokutil --sb-state); "+ + "check that nouveau is not holding the card (lsmod | grep nouveau)") case f.GPUError != "": add(Fail, "gpu", "nvidia-smi fails: "+f.GPUError, "check the driver: run nvidia-smi by hand") + case len(f.GPUs) == 0 && f.Host.NVIDIADevice: + add(Fail, "gpu", "an NVIDIA card is on the PCI bus but nvidia-smi reports no GPUs", driverFix) case len(f.GPUs) == 0: add(Warn, "gpu", "nvidia-smi is installed but reports no GPUs", "") default: diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 896fa2c..1ce70d4 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -276,3 +276,27 @@ func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) { t.Errorf("%+v", r) } } + +func TestACardWithoutADriverIsAFailureNotACPUNode(t *testing.T) { + // The mistake this guards: a GPU node whose driver was never installed + // onboarded quietly as a CPU node, with nothing anywhere saying why. + f := healthy() + f.SMIFound, f.GPUs, f.GPUError = false, nil, "" + f.Host.NVIDIADevice, f.Host.NVIDIADriver = true, "" + fs := Evaluate(f) + has(t, fs, Fail, "an NVIDIA card is on the PCI bus but no driver is installed") + + f.Host.NVIDIADevice = false + fs = Evaluate(f) + if bad := levels(fs, Fail); len(bad) > 0 { + t.Errorf("a plain CPU node was faulted: %v", bad) + } + has(t, fs, Info, "no nvidia-smi: a CPU node") + + // nvidia-smi installed, the module not loaded: the message points at the + // usual reasons rather than at a generic "run nvidia-smi". + f.SMIFound, f.GPUError = true, "running nvidia-smi: exit status 9" + f.Host.NVIDIADevice = true + fs = Evaluate(f) + has(t, fs, Fail, "its driver is not loaded") +}