Files
llmgw/internal/agent/discovery/discover_test.go
T
ilkerandClaude Opus 5 55417546fc
CI / test (push) Successful in 1m37s
Phase 2 (2/2): NVML backend behind a build tag
NVML is the only source of NVLink topology, and that topology is what keeps a
tensor-parallel group off the slow path inside a node. It needs cgo, so it sits
behind the nvml tag and the default build keeps using nvidia-smi.

Discover() picks the backend and falls back to nvidia-smi when NVML fails,
logging the fallback rather than performing it silently: on a node that is meant
to have working NVML, a quiet downgrade hides a broken driver installation and
costs the scheduler its topology.

CI now compiles the tagged build. That immediately paid for itself: ErrNoNVML
was declared only in the untagged file, so the tests did not compile under
-tags nvml. It is now declared where both builds can see it.

Not verified: the NVML path has never run against a driver, because there is no
GPU machine available. The NVLink domains it reports should be treated as
unverified until someone runs `llmgw agent inventory` on a real node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 02:05:25 +02:00

102 lines
2.7 KiB
Go

package discovery
import (
"context"
"log/slog"
"os"
"path/filepath"
"testing"
)
func fakeSMI(t *testing.T) string {
t.Helper()
stub := filepath.Join(t.TempDir(), "nvidia-smi")
script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
return stub
}
func TestDiscoverUsesSMIByDefault(t *testing.T) {
inv, err := Discover(t.Context(), Options{
NvidiaSMI: fakeSMI(t),
IBRoot: filepath.Join(t.TempDir(), "absent"),
}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
if inv.Source != "nvidia-smi" {
t.Errorf("source=%q", inv.Source)
}
if len(inv.GPUs) != 2 {
t.Errorf("got %d GPUs", len(inv.GPUs))
}
}
func TestDiscoverFallsBackWhenNVMLUnavailable(t *testing.T) {
// Asking for NVML in a build without it must still produce an inventory:
// the fallback is the whole point of the build tag.
inv, err := Discover(t.Context(), Options{
PreferNVML: true,
NvidiaSMI: fakeSMI(t),
IBRoot: filepath.Join(t.TempDir(), "absent"),
}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
if len(inv.GPUs) == 0 {
t.Fatal("no GPUs discovered through the fallback")
}
if NVMLSupported() {
// On an NVML build without a driver the NVML attempt fails and the
// fallback still has to work.
if inv.Source != "nvml" && inv.Source != "nvidia-smi" {
t.Errorf("source=%q", inv.Source)
}
} else if inv.Source != "nvidia-smi" {
t.Errorf("a build without NVML reported source=%q", inv.Source)
}
}
func TestDiscoverSkipGPUs(t *testing.T) {
inv, err := Discover(t.Context(), Options{
SkipGPUs: true,
IBRoot: fakeSysfs(t),
}, slog.New(slog.DiscardHandler))
if err != nil {
t.Fatal(err)
}
if len(inv.GPUs) != 0 {
t.Errorf("GPUs were discovered despite SkipGPUs")
}
if len(inv.HCAs) != 3 {
t.Errorf("got %d HCAs", len(inv.HCAs))
}
if inv.Source != "none" {
t.Errorf("source=%q", inv.Source)
}
}
func TestDiscoverReportsGPUFailure(t *testing.T) {
// A node meant to have GPUs whose discovery fails must say so rather than
// reporting an empty, healthy-looking inventory.
_, err := Discover(context.Background(), Options{
NvidiaSMI: filepath.Join(t.TempDir(), "absent"),
IBRoot: filepath.Join(t.TempDir(), "absent"),
}, slog.New(slog.DiscardHandler))
if err == nil {
t.Error("a failed GPU discovery was reported as success")
}
}
func TestNVMLSupportedMatchesBuild(t *testing.T) {
// Documents which backend this binary has, and keeps both files compiling.
t.Logf("NVML backend compiled in: %v", NVMLSupported())
if !NVMLSupported() {
if _, err := DiscoverGPUsNVML(t.Context()); err != ErrNoNVML {
t.Errorf("expected ErrNoNVML, got %v", err)
}
}
}