CI / test (push) Successful in 1m37s
NVML is the only source of NVLink topology, and that topology is what keeps a tensor-parallel group off the slow path inside a node. It needs cgo, so it sits behind the nvml tag and the default build keeps using nvidia-smi. Discover() picks the backend and falls back to nvidia-smi when NVML fails, logging the fallback rather than performing it silently: on a node that is meant to have working NVML, a quiet downgrade hides a broken driver installation and costs the scheduler its topology. CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML was declared only in the untagged file, so the tests did not compile under -tags nvml. It is now declared where both builds can see it. Not verified: the NVML path has never run against a driver, because there is no GPU machine available. The NVLink domains it reports should be treated as unverified until someone runs `llmgw agent inventory` on a real node. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
102 lines
2.7 KiB
Go
102 lines
2.7 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func fakeSMI(t *testing.T) string {
|
|
t.Helper()
|
|
stub := filepath.Join(t.TempDir(), "nvidia-smi")
|
|
script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
|
|
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return stub
|
|
}
|
|
|
|
func TestDiscoverUsesSMIByDefault(t *testing.T) {
|
|
inv, err := Discover(t.Context(), Options{
|
|
NvidiaSMI: fakeSMI(t),
|
|
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
|
}, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if inv.Source != "nvidia-smi" {
|
|
t.Errorf("source=%q", inv.Source)
|
|
}
|
|
if len(inv.GPUs) != 2 {
|
|
t.Errorf("got %d GPUs", len(inv.GPUs))
|
|
}
|
|
}
|
|
|
|
func TestDiscoverFallsBackWhenNVMLUnavailable(t *testing.T) {
|
|
// Asking for NVML in a build without it must still produce an inventory:
|
|
// the fallback is the whole point of the build tag.
|
|
inv, err := Discover(t.Context(), Options{
|
|
PreferNVML: true,
|
|
NvidiaSMI: fakeSMI(t),
|
|
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
|
}, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(inv.GPUs) == 0 {
|
|
t.Fatal("no GPUs discovered through the fallback")
|
|
}
|
|
if NVMLSupported() {
|
|
// On an NVML build without a driver the NVML attempt fails and the
|
|
// fallback still has to work.
|
|
if inv.Source != "nvml" && inv.Source != "nvidia-smi" {
|
|
t.Errorf("source=%q", inv.Source)
|
|
}
|
|
} else if inv.Source != "nvidia-smi" {
|
|
t.Errorf("a build without NVML reported source=%q", inv.Source)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverSkipGPUs(t *testing.T) {
|
|
inv, err := Discover(t.Context(), Options{
|
|
SkipGPUs: true,
|
|
IBRoot: fakeSysfs(t),
|
|
}, slog.New(slog.DiscardHandler))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(inv.GPUs) != 0 {
|
|
t.Errorf("GPUs were discovered despite SkipGPUs")
|
|
}
|
|
if len(inv.HCAs) != 3 {
|
|
t.Errorf("got %d HCAs", len(inv.HCAs))
|
|
}
|
|
if inv.Source != "none" {
|
|
t.Errorf("source=%q", inv.Source)
|
|
}
|
|
}
|
|
|
|
func TestDiscoverReportsGPUFailure(t *testing.T) {
|
|
// A node meant to have GPUs whose discovery fails must say so rather than
|
|
// reporting an empty, healthy-looking inventory.
|
|
_, err := Discover(context.Background(), Options{
|
|
NvidiaSMI: filepath.Join(t.TempDir(), "absent"),
|
|
IBRoot: filepath.Join(t.TempDir(), "absent"),
|
|
}, slog.New(slog.DiscardHandler))
|
|
if err == nil {
|
|
t.Error("a failed GPU discovery was reported as success")
|
|
}
|
|
}
|
|
|
|
func TestNVMLSupportedMatchesBuild(t *testing.T) {
|
|
// Documents which backend this binary has, and keeps both files compiling.
|
|
t.Logf("NVML backend compiled in: %v", NVMLSupported())
|
|
if !NVMLSupported() {
|
|
if _, err := DiscoverGPUsNVML(t.Context()); err != ErrNoNVML {
|
|
t.Errorf("expected ErrNoNVML, got %v", err)
|
|
}
|
|
}
|
|
}
|