Deploying the memory-aware placement showed node01 to node04 reporting 7G and node05 reporting nothing, on a machine whose /proc/meminfo plainly says 32 GB. node05 has no nvidia-smi, so GPU discovery returns an error, Discover returns with it, and the memory read — which has nothing to do with cards — never happened. That is the node whose memory matters most, because a CPU profile is what a GPU-less node will be asked to run, and min_memory_gb would have excluded it from everything. Read first now, before any path that can return early. The test fails against the old order. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
aa2a1fe08f
commit
d71d785598
2 files changed
+24
-3
No files matched your search
@@ -39,6 +39,12 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
log = slog.Default()
|
||||
}
|
||||
inv := Inventory{Source: "none"}
|
||||
// First, because it has nothing to do with GPUs and every path below can
|
||||
// return early. Read after them, a node with no nvidia-smi reported no
|
||||
// memory at all: GPU discovery fails, Discover returns, and the fact that
|
||||
// has nothing to do with cards goes with it. That is exactly the node whose
|
||||
// memory matters most, since a CPU profile is what it will be asked to run.
|
||||
inv.MemoryTotalMB = MemoryTotalMB()
|
||||
|
||||
if !opts.SkipGPUs {
|
||||
switch {
|
||||
@@ -65,9 +71,6 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
|
||||
return inv, fmt.Errorf("infiniband discovery: %w", err)
|
||||
}
|
||||
inv.HCAs = hcas
|
||||
// Not fatal when it cannot be read: a node that does not report its memory
|
||||
// constrains nothing, which is how every node behaved before this existed.
|
||||
inv.MemoryTotalMB = MemoryTotalMB()
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -99,3 +100,20 @@ func TestNVMLSupportedMatchesBuild(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryIsReportedEvenWhenGPUDiscoveryFails(t *testing.T) {
|
||||
// A node with no nvidia-smi is exactly the node whose memory matters, since
|
||||
// a CPU profile is what it will be asked to run. Reading memory after GPU
|
||||
// discovery meant that node reported none: the GPU error returns early and
|
||||
// takes a fact that has nothing to do with cards with it. Seen on node05,
|
||||
// which has 32 GB and was reporting "-".
|
||||
inv, err := Discover(t.Context(), Options{NvidiaSMI: "/nonexistent/nvidia-smi"}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("GPU discovery was expected to fail with no nvidia-smi")
|
||||
}
|
||||
// On Linux this is a real number; elsewhere there is no /proc/meminfo and 0
|
||||
// is the honest answer. Either way it must not be lost to the GPU error.
|
||||
if runtime.GOOS == "linux" && inv.MemoryTotalMB <= 0 {
|
||||
t.Errorf("memory was not reported alongside a GPU failure: %d", inv.MemoryTotalMB)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user