CI / test (push) Failing after 10m13s
Only the NVML build filled it, and the release's portable binary, the one nodes run, never did: the CUDA 13.x check that came before this saw nothing on cuda01 and installed b12606's CUDA 13.5 build again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
195 lines
6.3 KiB
Go
195 lines
6.3 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
const smiSample = `0, GPU-11111111-2222-3333-4444-555555555555, NVIDIA H100 80GB HBM3, 81559, 4, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
|
|
1, GPU-66666666-7777-8888-9999-000000000000, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1C:00.0, 550.54.15, 36, 0
|
|
`
|
|
|
|
func TestParseSMI(t *testing.T) {
|
|
gpus, err := parseSMI(smiSample)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(gpus) != 2 {
|
|
t.Fatalf("expected 2 GPUs, got %d", len(gpus))
|
|
}
|
|
|
|
g := gpus[0]
|
|
if g.Index != 0 || g.UUID != "GPU-11111111-2222-3333-4444-555555555555" {
|
|
t.Errorf("identity wrong: %+v", g)
|
|
}
|
|
if g.MemoryTotalMB != 81559 {
|
|
t.Errorf("MemoryTotalMB=%d", g.MemoryTotalMB)
|
|
}
|
|
if g.ComputeCapability != "9.0" {
|
|
t.Errorf("ComputeCapability=%q; FP8 placement depends on it", g.ComputeCapability)
|
|
}
|
|
if g.DriverVersion != "550.54.15" {
|
|
t.Errorf("DriverVersion=%q", g.DriverVersion)
|
|
}
|
|
if !g.Health.OK() {
|
|
t.Errorf("a healthy GPU was flagged: %v", g.Health.Problems)
|
|
}
|
|
}
|
|
|
|
func TestParseSMIUnsupportedFields(t *testing.T) {
|
|
// Consumer cards report [N/A] for ECC; that must not fail the whole parse.
|
|
const row = `0, GPU-abc, NVIDIA GeForce RTX 4090, 24564, 0, 8.9, 00000000:01:00.0, 550.54.15, 45, [N/A]
|
|
`
|
|
gpus, err := parseSMI(row)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(gpus) != 1 {
|
|
t.Fatalf("expected 1 GPU, got %d", len(gpus))
|
|
}
|
|
if gpus[0].Health.ECCErrors != 0 || !gpus[0].Health.OK() {
|
|
t.Errorf("[N/A] ECC was misread: %+v", gpus[0].Health)
|
|
}
|
|
}
|
|
|
|
func TestHealthFlagsForeignMemory(t *testing.T) {
|
|
// Memory in use on an otherwise idle card means another process is there.
|
|
const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 40000, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
|
|
`
|
|
gpus, _ := parseSMI(row)
|
|
if gpus[0].Health.OK() {
|
|
t.Error("40 GiB allocated by another process was not flagged")
|
|
}
|
|
}
|
|
|
|
func TestHealthFlagsECC(t *testing.T) {
|
|
const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1B:00.0, 550.54.15, 34, 3
|
|
`
|
|
gpus, _ := parseSMI(row)
|
|
if gpus[0].Health.OK() {
|
|
t.Error("uncorrected ECC errors were not flagged")
|
|
}
|
|
}
|
|
|
|
func TestGPUClass(t *testing.T) {
|
|
// The same card must derive the same class however the driver names it,
|
|
// because a tensor-parallel group may not mix classes.
|
|
cases := []struct {
|
|
name string
|
|
memMB int
|
|
expect string
|
|
}{
|
|
{"NVIDIA H100 80GB HBM3", 81559, "h100-80g"},
|
|
{"NVIDIA H100-SXM5-80GB", 81559, "h100-80g"},
|
|
{"Tesla A100-SXM4-40GB", 40960, "a100-40g"},
|
|
{"NVIDIA A100 80GB PCIe", 81920, "a100-80g"},
|
|
// Sizes come from what the driver reports, in GiB. That does not always
|
|
// match the marketing name: an L40S is sold as 48GB but reports 44.99
|
|
// GiB, because the marketing figure is decimal GB. Being consistent
|
|
// matters more than matching the box, since the number is what decides
|
|
// whether a model fits.
|
|
{"NVIDIA L40S", 46068, "l40s-45g"},
|
|
// The model token is what distinguishes a 4090 from a 4080, so "rtx"
|
|
// is not enough on its own.
|
|
{"NVIDIA GeForce RTX 4090", 24564, "4090-24g"},
|
|
}
|
|
for _, c := range cases {
|
|
got := GPU{Name: c.name, MemoryTotalMB: c.memMB}.Class()
|
|
if got != c.expect {
|
|
t.Errorf("%q (%d MiB) -> %q, want %q", c.name, c.memMB, got, c.expect)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInventoryClasses(t *testing.T) {
|
|
inv := Inventory{GPUs: []GPU{
|
|
{Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559},
|
|
{Name: "NVIDIA H100-SXM5-80GB", MemoryTotalMB: 81559},
|
|
{Name: "Tesla A100-SXM4-40GB", MemoryTotalMB: 40960},
|
|
}}
|
|
got := inv.Classes()
|
|
if len(got) != 2 || got[0] != "a100-40g" || got[1] != "h100-80g" {
|
|
t.Errorf("classes=%v", got)
|
|
}
|
|
}
|
|
|
|
// TestDiscoverGPUsSMI runs the real code path against a stub binary, so the
|
|
// command construction is covered too, not just the parser.
|
|
func TestDiscoverGPUsSMI(t *testing.T) {
|
|
dir := t.TempDir()
|
|
stub := filepath.Join(dir, "nvidia-smi")
|
|
script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
|
|
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
gpus, err := DiscoverGPUsSMI(t.Context(), stub)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(gpus) != 2 {
|
|
t.Fatalf("expected 2 GPUs, got %d", len(gpus))
|
|
}
|
|
}
|
|
|
|
func TestDiscoverGPUsSMIMissingBinary(t *testing.T) {
|
|
// A node with no NVIDIA driver must produce a clear error rather than a
|
|
// panic or an empty success.
|
|
_, err := DiscoverGPUsSMI(context.Background(), filepath.Join(t.TempDir(), "absent"))
|
|
if err == nil {
|
|
t.Error("expected an error when nvidia-smi is missing")
|
|
}
|
|
}
|
|
|
|
func TestAnOlderDriverWithoutUtilisationStillDiscovers(t *testing.T) {
|
|
// An nvidia-smi too old for utilization.gpu returns a shorter row. Rejecting
|
|
// those would make an llmgw upgrade silently stop discovering GPUs on an
|
|
// older node, which is far worse than not knowing how busy a card is — and
|
|
// is what the first version of this change did.
|
|
old := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0"
|
|
gpus, err := parseSMI(old)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(gpus) != 1 {
|
|
t.Fatalf("a row without utilisation produced %d GPUs", len(gpus))
|
|
}
|
|
if gpus[0].MemoryTotalMB != 81559 || gpus[0].Health.TemperatureC != 41 {
|
|
t.Errorf("columns shifted: %+v", gpus[0])
|
|
}
|
|
if gpus[0].UtilizationPercent != 0 {
|
|
t.Errorf("utilisation = %d, want 0 when unreported", gpus[0].UtilizationPercent)
|
|
}
|
|
}
|
|
|
|
func TestUtilisationIsReadWhenPresent(t *testing.T) {
|
|
row := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0, 87"
|
|
gpus, err := parseSMI(row)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(gpus) != 1 {
|
|
t.Fatalf("got %d GPUs", len(gpus))
|
|
}
|
|
if gpus[0].UtilizationPercent != 87 {
|
|
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
|
|
}
|
|
}
|
|
|
|
func TestTheDriversCUDAIsReadFromNvidiaSMIVersion(t *testing.T) {
|
|
for out, want := range map[string]string{
|
|
// cuda01, driver 615.
|
|
"NVIDIA-SMI version : 615.71.09\nNVML version : 615.71\nDRIVER version : Deprecated, see \"KMD version\" instead\n" +
|
|
"CUDA version : Deprecated, see \"CUDA UMD version\" instead\nKMD version : 615.71.09\nCUDA UMD version : 13.4\n": "13.4",
|
|
// An older driver.
|
|
"NVIDIA-SMI version : 550.54.15\nNVML version : 550.54\nDRIVER version : 550.54.15\nCUDA Version : 12.4\n": "12.4",
|
|
"nothing useful": "",
|
|
} {
|
|
if got := parseSMIDriverCUDA(out); got != want {
|
|
t.Errorf("%q: %q, want %q", out, got, want)
|
|
}
|
|
}
|
|
}
|