An NVIDIA card with no driver is reported, not onboarded as a CPU node
CI / test (push) Failing after 9m12s
CI / test (push) Failing after 9m12s
The agent found GPUs through nvidia-smi alone, so a machine whose driver was never installed looked exactly like one with no card and joined quietly as a CPU node. Only init looked at the PCI bus. The host discovery now reads the bus too (vendor 0x10de, class 0x03) and carries it to the control plane. The doctor fails the pre-install checks on a card without a driver, with the install and the reboot named, and tells a driver that is installed but not loaded (Secure Boot, nouveau) apart from a missing one. A node already in shows the warning in ctl nodes and the console. init uses the same check. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
a53b2f4be9
commit
c2fedeb326
12 files changed
+138
-26
No files matched your search
@@ -596,7 +596,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
a.forgetSent()
|
||||
reg.Host = &llmgwv1.Host{
|
||||
OsId: h.OSID, OsVersion: h.OSVersion, OsCodename: h.OSCodename,
|
||||
Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver,
|
||||
Arch: h.Arch, CpuFlags: h.CPUFlags, NvidiaDriver: h.NVIDIADriver, NvidiaDevice: h.NVIDIADevice,
|
||||
// What is measured by now, not what the startup scan found before
|
||||
// the measurement ended: a reconnect would otherwise say 0.
|
||||
MemBandwidthGbs: a.memBandwidth().All,
|
||||
|
||||
@@ -3,6 +3,7 @@ package discovery
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
@@ -28,6 +29,10 @@ type Host struct {
|
||||
CPUFlags []string `json:"cpu_flags,omitempty"`
|
||||
// NVIDIADriver is the loaded driver's version, empty when there is none.
|
||||
NVIDIADriver string `json:"nvidia_driver,omitempty"`
|
||||
// NVIDIADevice is whether an NVIDIA display controller is on the PCI bus,
|
||||
// driver or not. True with NVIDIADriver empty is a card without a driver:
|
||||
// a machine that would otherwise serve on CPU with nothing saying why.
|
||||
NVIDIADevice bool `json:"nvidia_device,omitempty"`
|
||||
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
|
||||
// is not known. Not read here but from the GPU inventory, by the agent.
|
||||
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
|
||||
@@ -75,9 +80,35 @@ func DiscoverHost() Host {
|
||||
h.Arch = debianArch(runtime.GOARCH)
|
||||
h.CPUFlags = cpuFlagsFrom("/proc/cpuinfo")
|
||||
h.NVIDIADriver = nvidiaDriverFrom("/proc/driver/nvidia/version")
|
||||
h.NVIDIADevice = HasNVIDIADevice()
|
||||
return h
|
||||
}
|
||||
|
||||
// CardWithoutDriver reports an NVIDIA card on the PCI bus with no driver
|
||||
// loaded: a node that looks like a CPU node to nvidia-smi and is not one.
|
||||
func (h Host) CardWithoutDriver() bool { return h.NVIDIADevice && h.NVIDIADriver == "" }
|
||||
|
||||
// HasNVIDIADevice reports an NVIDIA display or 3D controller on the PCI bus,
|
||||
// read from sysfs, which needs no driver.
|
||||
func HasNVIDIADevice() bool { return hasNVIDIADeviceIn("/sys/bus/pci/devices") }
|
||||
|
||||
func hasNVIDIADeviceIn(root string) bool {
|
||||
vendors, _ := filepath.Glob(filepath.Join(root, "*", "vendor"))
|
||||
for _, v := range vendors {
|
||||
b, err := os.ReadFile(v) //nolint:gosec // sysfs
|
||||
if err != nil || strings.TrimSpace(string(b)) != "0x10de" {
|
||||
continue
|
||||
}
|
||||
class, _ := os.ReadFile(filepath.Join(filepath.Dir(v), "class")) //nolint:gosec // sysfs
|
||||
// 0x03xxxx is a display or 3D controller; NVIDIA also makes bridges
|
||||
// and audio devices.
|
||||
if strings.HasPrefix(strings.TrimSpace(string(class)), "0x03") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hostFromOSRelease(path string) Host {
|
||||
var h Host
|
||||
f, err := os.Open(path) //nolint:gosec // a fixed system path
|
||||
|
||||
@@ -23,3 +23,35 @@ func TestNVIDIADriverFrom(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACardOnTheBusIsSeenWithoutADriver(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dev := func(name, vendor, class string) {
|
||||
d := filepath.Join(root, name)
|
||||
if err := os.MkdirAll(d, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(d, "vendor"), []byte(vendor+"\n"), 0o600)
|
||||
_ = os.WriteFile(filepath.Join(d, "class"), []byte(class+"\n"), 0o600)
|
||||
}
|
||||
dev("0000:00:1f.0", "0x8086", "0x060100") // the chipset
|
||||
if hasNVIDIADeviceIn(root) {
|
||||
t.Fatal("a machine with no NVIDIA device said it had one")
|
||||
}
|
||||
dev("0000:01:00.1", "0x10de", "0x040300") // the card's HDMI audio alone is not a GPU
|
||||
if hasNVIDIADeviceIn(root) {
|
||||
t.Fatal("NVIDIA's audio device was taken for a GPU")
|
||||
}
|
||||
dev("0000:01:00.0", "0x10de", "0x030000")
|
||||
if !hasNVIDIADeviceIn(root) {
|
||||
t.Fatal("a VGA controller of NVIDIA's was not seen")
|
||||
}
|
||||
h := Host{NVIDIADevice: true}
|
||||
if !h.CardWithoutDriver() {
|
||||
t.Error("a card with no driver version is a card without a driver")
|
||||
}
|
||||
h.NVIDIADriver = "580.82.07"
|
||||
if h.CardWithoutDriver() {
|
||||
t.Error("a card with a driver was reported as without one")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user