From d71d78559843ca49727aa2b6e8f1da727686a46d Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 23:50:53 +0200 Subject: [PATCH] Report node memory even when GPU discovery fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploying the memory-aware placement showed node01 to node04 reporting 7G and node05 reporting nothing, on a machine whose /proc/meminfo plainly says 32 GB. node05 has no nvidia-smi, so GPU discovery returns an error, Discover returns with it, and the memory read — which has nothing to do with cards — never happened. That is the node whose memory matters most, because a CPU profile is what a GPU-less node will be asked to run, and min_memory_gb would have excluded it from everything. Read first now, before any path that can return early. The test fails against the old order. Co-Authored-By: Claude Opus 5 (1M context) --- internal/agent/discovery/discover.go | 9 ++++++--- internal/agent/discovery/discover_test.go | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 6ed80269..1225cf2e 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -39,6 +39,12 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e log = slog.Default() } inv := Inventory{Source: "none"} + // First, because it has nothing to do with GPUs and every path below can + // return early. Read after them, a node with no nvidia-smi reported no + // memory at all: GPU discovery fails, Discover returns, and the fact that + // has nothing to do with cards goes with it. That is exactly the node whose + // memory matters most, since a CPU profile is what it will be asked to run. + inv.MemoryTotalMB = MemoryTotalMB() if !opts.SkipGPUs { switch { @@ -65,9 +71,6 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e return inv, fmt.Errorf("infiniband discovery: %w", err) } inv.HCAs = hcas - // Not fatal when it cannot be read: a node that does not report its memory - // constrains nothing, which is how every node behaved before this existed. - inv.MemoryTotalMB = MemoryTotalMB() return inv, nil } diff --git a/internal/agent/discovery/discover_test.go b/internal/agent/discovery/discover_test.go index 6791b5c5..b79cc0c0 100644 --- a/internal/agent/discovery/discover_test.go +++ b/internal/agent/discovery/discover_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "runtime" "testing" ) @@ -99,3 +100,20 @@ func TestNVMLSupportedMatchesBuild(t *testing.T) { } } } + +func TestMemoryIsReportedEvenWhenGPUDiscoveryFails(t *testing.T) { + // A node with no nvidia-smi is exactly the node whose memory matters, since + // a CPU profile is what it will be asked to run. Reading memory after GPU + // discovery meant that node reported none: the GPU error returns early and + // takes a fact that has nothing to do with cards with it. Seen on node05, + // which has 32 GB and was reporting "-". + inv, err := Discover(t.Context(), Options{NvidiaSMI: "/nonexistent/nvidia-smi"}, nil) + if err == nil { + t.Fatal("GPU discovery was expected to fail with no nvidia-smi") + } + // On Linux this is a real number; elsewhere there is no /proc/meminfo and 0 + // is the honest answer. Either way it must not be lost to the GPU error. + if runtime.GOOS == "linux" && inv.MemoryTotalMB <= 0 { + t.Errorf("memory was not reported alongside a GPU failure: %d", inv.MemoryTotalMB) + } +}