llmgw/internal/agent/discovery/gpu_test.go

package discovery

import (
	"context"
	"os"
	"path/filepath"
	"testing"
)

const smiSample = `0, GPU-11111111-2222-3333-4444-555555555555, NVIDIA H100 80GB HBM3, 81559, 4, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
1, GPU-66666666-7777-8888-9999-000000000000, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1C:00.0, 550.54.15, 36, 0
`

func TestParseSMI(t *testing.T) {
	gpus, err := parseSMI(smiSample)
	if err != nil {
		t.Fatal(err)
	}
	if len(gpus) != 2 {
		t.Fatalf("expected 2 GPUs, got %d", len(gpus))
	}

	g := gpus[0]
	if g.Index != 0 || g.UUID != "GPU-11111111-2222-3333-4444-555555555555" {
		t.Errorf("identity wrong: %+v", g)
	}
	if g.MemoryTotalMB != 81559 {
		t.Errorf("MemoryTotalMB=%d", g.MemoryTotalMB)
	}
	if g.ComputeCapability != "9.0" {
		t.Errorf("ComputeCapability=%q; FP8 placement depends on it", g.ComputeCapability)
	}
	if g.DriverVersion != "550.54.15" {
		t.Errorf("DriverVersion=%q", g.DriverVersion)
	}
	if !g.Health.OK() {
		t.Errorf("a healthy GPU was flagged: %v", g.Health.Problems)
	}
}

func TestParseSMIUnsupportedFields(t *testing.T) {
	// Consumer cards report [N/A] for ECC; that must not fail the whole parse.
	const row = `0, GPU-abc, NVIDIA GeForce RTX 4090, 24564, 0, 8.9, 00000000:01:00.0, 550.54.15, 45, [N/A]
`
	gpus, err := parseSMI(row)
	if err != nil {
		t.Fatal(err)
	}
	if len(gpus) != 1 {
		t.Fatalf("expected 1 GPU, got %d", len(gpus))
	}
	if gpus[0].Health.ECCErrors != 0 || !gpus[0].Health.OK() {
		t.Errorf("[N/A] ECC was misread: %+v", gpus[0].Health)
	}
}

func TestHealthFlagsForeignMemory(t *testing.T) {
	// Memory in use on an otherwise idle card means another process is there.
	const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 40000, 9.0, 00000000:1B:00.0, 550.54.15, 34, 0
`
	gpus, _ := parseSMI(row)
	if gpus[0].Health.OK() {
		t.Error("40 GiB allocated by another process was not flagged")
	}
}

func TestHealthFlagsECC(t *testing.T) {
	const row = `0, GPU-abc, NVIDIA H100 80GB HBM3, 81559, 0, 9.0, 00000000:1B:00.0, 550.54.15, 34, 3
`
	gpus, _ := parseSMI(row)
	if gpus[0].Health.OK() {
		t.Error("uncorrected ECC errors were not flagged")
	}
}

func TestGPUClass(t *testing.T) {
	// The same card must derive the same class however the driver names it,
	// because a tensor-parallel group may not mix classes.
	cases := []struct {
		name   string
		memMB  int
		expect string
	}{
		{"NVIDIA H100 80GB HBM3", 81559, "h100-80g"},
		{"NVIDIA H100-SXM5-80GB", 81559, "h100-80g"},
		{"Tesla A100-SXM4-40GB", 40960, "a100-40g"},
		{"NVIDIA A100 80GB PCIe", 81920, "a100-80g"},
		// Sizes come from what the driver reports, in GiB. That does not always
		// match the marketing name: an L40S is sold as 48GB but reports 44.99
		// GiB, because the marketing figure is decimal GB. Being consistent
		// matters more than matching the box, since the number is what decides
		// whether a model fits.
		{"NVIDIA L40S", 46068, "l40s-45g"},
		// The model token is what distinguishes a 4090 from a 4080, so "rtx"
		// is not enough on its own.
		{"NVIDIA GeForce RTX 4090", 24564, "4090-24g"},
	}
	for _, c := range cases {
		got := GPU{Name: c.name, MemoryTotalMB: c.memMB}.Class()
		if got != c.expect {
			t.Errorf("%q (%d MiB) -> %q, want %q", c.name, c.memMB, got, c.expect)
		}
	}
}

func TestInventoryClasses(t *testing.T) {
	inv := Inventory{GPUs: []GPU{
		{Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559},
		{Name: "NVIDIA H100-SXM5-80GB", MemoryTotalMB: 81559},
		{Name: "Tesla A100-SXM4-40GB", MemoryTotalMB: 40960},
	}}
	got := inv.Classes()
	if len(got) != 2 || got[0] != "a100-40g" || got[1] != "h100-80g" {
		t.Errorf("classes=%v", got)
	}
}

// TestDiscoverGPUsSMI runs the real code path against a stub binary, so the
// command construction is covered too, not just the parser.
func TestDiscoverGPUsSMI(t *testing.T) {
	dir := t.TempDir()
	stub := filepath.Join(dir, "nvidia-smi")
	script := "#!/bin/sh\ncat <<'EOF'\n" + smiSample + "EOF\n"
	if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
		t.Fatal(err)
	}

	gpus, err := DiscoverGPUsSMI(t.Context(), stub)
	if err != nil {
		t.Fatal(err)
	}
	if len(gpus) != 2 {
		t.Fatalf("expected 2 GPUs, got %d", len(gpus))
	}
}

func TestDiscoverGPUsSMIMissingBinary(t *testing.T) {
	// A node with no NVIDIA driver must produce a clear error rather than a
	// panic or an empty success.
	_, err := DiscoverGPUsSMI(context.Background(), filepath.Join(t.TempDir(), "absent"))
	if err == nil {
		t.Error("expected an error when nvidia-smi is missing")
	}
}

func TestAnOlderDriverWithoutUtilisationStillDiscovers(t *testing.T) {
	// An nvidia-smi too old for utilization.gpu returns a shorter row. Rejecting
	// those would make an llmgw upgrade silently stop discovering GPUs on an
	// older node, which is far worse than not knowing how busy a card is — and
	// is what the first version of this change did.
	old := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0"
	gpus, err := parseSMI(old)
	if err != nil {
		t.Fatal(err)
	}
	if len(gpus) != 1 {
		t.Fatalf("a row without utilisation produced %d GPUs", len(gpus))
	}
	if gpus[0].MemoryTotalMB != 81559 || gpus[0].Health.TemperatureC != 41 {
		t.Errorf("columns shifted: %+v", gpus[0])
	}
	if gpus[0].UtilizationPercent != 0 {
		t.Errorf("utilisation = %d, want 0 when unreported", gpus[0].UtilizationPercent)
	}
}

func TestUtilisationIsReadWhenPresent(t *testing.T) {
	row := "0, GPU-aaa, NVIDIA H100 80GB HBM3, 81559, 1024, 9.0, 0000:01:00.0, 550.54, 41, 0, 87"
	gpus, err := parseSMI(row)
	if err != nil {
		t.Fatal(err)
	}
	if len(gpus) != 1 {
		t.Fatalf("got %d GPUs", len(gpus))
	}
	if gpus[0].UtilizationPercent != 87 {
		t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
	}
}