diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 6ed8026..1225cf2 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -39,6 +39,12 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e log = slog.Default() } inv := Inventory{Source: "none"} + // First, because it has nothing to do with GPUs and every path below can + // return early. Read after them, a node with no nvidia-smi reported no + // memory at all: GPU discovery fails, Discover returns, and the fact that + // has nothing to do with cards goes with it. That is exactly the node whose + // memory matters most, since a CPU profile is what it will be asked to run. + inv.MemoryTotalMB = MemoryTotalMB() if !opts.SkipGPUs { switch { @@ -65,9 +71,6 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e return inv, fmt.Errorf("infiniband discovery: %w", err) } inv.HCAs = hcas - // Not fatal when it cannot be read: a node that does not report its memory - // constrains nothing, which is how every node behaved before this existed. - inv.MemoryTotalMB = MemoryTotalMB() return inv, nil } diff --git a/internal/agent/discovery/discover_test.go b/internal/agent/discovery/discover_test.go index 6791b5c..b79cc0c 100644 --- a/internal/agent/discovery/discover_test.go +++ b/internal/agent/discovery/discover_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "runtime" "testing" ) @@ -99,3 +100,20 @@ func TestNVMLSupportedMatchesBuild(t *testing.T) { } } } + +func TestMemoryIsReportedEvenWhenGPUDiscoveryFails(t *testing.T) { + // A node with no nvidia-smi is exactly the node whose memory matters, since + // a CPU profile is what it will be asked to run. Reading memory after GPU + // discovery meant that node reported none: the GPU error returns early and + // takes a fact that has nothing to do with cards with it. Seen on node05, + // which has 32 GB and was reporting "-". + inv, err := Discover(t.Context(), Options{NvidiaSMI: "/nonexistent/nvidia-smi"}, nil) + if err == nil { + t.Fatal("GPU discovery was expected to fail with no nvidia-smi") + } + // On Linux this is a real number; elsewhere there is no /proc/meminfo and 0 + // is the honest answer. Either way it must not be lost to the GPU error. + if runtime.GOOS == "linux" && inv.MemoryTotalMB <= 0 { + t.Errorf("memory was not reported alongside a GPU failure: %d", inv.MemoryTotalMB) + } +}