package discovery
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
// Two A100 80GB cards: card 0 in MIG mode, cut into two 3g.40gb instances,
// card 1 whole. What nvidia-smi prints for that machine, by command. None of
// it is from real MIG hardware, which this was written without; the shapes
// follow NVIDIA's documentation and the output of drivers R470 to R550.
const (
migQuery = `0, GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:07:00.0, 550.54.15, 31, 0, [N/A]
1, GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718, NVIDIA A100-SXM4-80GB, 81920, 0, 8.0, 00000000:0F:00.0, 550.54.15, 33, 0, 0
`
migList = `GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)
MIG 3g.40gb Device 0: (UUID: MIG-c6d4f1ef-42e4-5de3-91c7-45d71c87eb3f)
MIG 3g.40gb Device 1: (UUID: MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d)
GPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-1f2e3d4c-5b6a-7980-a1b2-c3d4e5f60718)
`
migReport = `
550.54.15
NVIDIA A100-SXM4-80GB
EnabledEnabled
1
1
0
42
40192 MiB0 MiB21000 MiB19192 MiB
0
2
0
40192 MiB0 MiB37 MiB40155 MiB
81920 MiB21037 MiB
1
0
4100Cvllm
20900 MiB
NVIDIA A100-SXM4-80GB
DisabledDisabled
None
81920 MiB300 MiB
N/A
N/A
5200C250 MiB
`
migLGI = `+-------------------------------------------------------+
| GPU instances: |
| GPU Name Profile Instance Placement |
| ID ID Start:Size |
|=======================================================|
| 0 MIG 3g.40gb 9 1 4:4 |
+-------------------------------------------------------+
| 0 MIG 3g.40gb 9 2 0:4 |
+-------------------------------------------------------+
`
migLCI = `+--------------------------------------------------------------------+
| Compute instances: |
| GPU GPU Name Profile Instance Placement |
| Instance ID ID Start:Size |
| ID |
|====================================================================|
| 0 1 MIG 3g.40gb 2 0 0:3 |
+--------------------------------------------------------------------+
| 0 2 MIG 3g.40gb 2 0 0:3 |
+--------------------------------------------------------------------+
`
)
func migMachine(t *testing.T) []GPU {
t.Helper()
cards, err := parseSMI(migQuery)
if err != nil {
t.Fatal(err)
}
report, err := parseMIGReport([]byte(migReport))
if err != nil {
t.Fatal(err)
}
return expandMIG(cards, parseSMIList(migList), report)
}
func TestACardInMIGModeIsReportedAsItsInstancesInsteadOfItself(t *testing.T) {
gpus := migMachine(t)
var got []string
for _, g := range gpus {
got = append(got, DeviceLabel(g.Index)+" "+g.Class())
}
want := []string{"0/mig0 a100-80g-mig-3g.40gb", "0/mig1 a100-80g-mig-3g.40gb", "1 a100-80g"}
if !slices.Equal(got, want) {
t.Fatalf("devices = %q, want %q", got, want)
}
mig := gpus[1]
if mig.Index != 1001 || mig.UUID != "MIG-0f7c3a2e-9d1b-5a8e-b3c4-1e2f3a4b5c6d" {
t.Errorf("instance 1 is %d %s; an engine is given it by that UUID", mig.Index, mig.UUID)
}
if mig.MemoryTotalMB != 40192 || mig.MemoryUsedMB != 21000 {
t.Errorf("instance 1 memory %d/%d, want its own 21000/40192 rather than the card's",
mig.MemoryUsedMB, mig.MemoryTotalMB)
}
if m := mig.MIG; m == nil || m.ParentIndex != 0 || m.Device != 1 || m.Profile != "3g.40gb" ||
m.GPUInstance != 1 || m.ComputeInstance != 0 || m.ParentUUID != "GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77" {
t.Errorf("instance 1's parentage: %+v", mig.MIG)
}
if mig.ComputeCapability != "8.0" || mig.DriverVersion != "550.54.15" || mig.Health.TemperatureC != 31 {
t.Errorf("what the card and its instances share was not carried: %+v", mig)
}
// Twenty gigabytes in use on an instance is a foreign tenant there, and
// only there: instance 0 next to it is free.
if mig.Health.OK() || !gpus[0].Health.OK() {
t.Errorf("foreign memory judged per instance: 0 %v, 1 %v", gpus[0].Health.Problems, mig.Health.Problems)
}
// No driver reports utilisation per instance, and none is made up.
if mig.UtilizationPercent != 0 {
t.Errorf("an instance reported utilisation %d", mig.UtilizationPercent)
}
if gpus[2].IsMIG() || gpus[2].Index != 1 || gpus[2].CardIndex() != 1 || gpus[0].CardIndex() != 0 {
t.Errorf("the whole card beside it changed: %+v", gpus[2])
}
}
func TestAMIGInstanceSurvivesTheTripToTheControlPlaneWithItsClass(t *testing.T) {
g := migMachine(t)[0]
b, err := json.Marshal(g)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(b), `"class":"a100-80g-mig-3g.40gb"`) {
t.Errorf("the JSON carries no class for the instance: %s", b)
}
var back GPU
if err := json.Unmarshal(b, &back); err != nil {
t.Fatal(err)
}
if back.Class() != g.Class() || !reflect.DeepEqual(back.MIG, g.MIG) {
t.Errorf("came back as %s %+v", back.Class(), back.MIG)
}
}
func TestMIGClassesAreNamedFromTheCardAndTheProfile(t *testing.T) {
for _, c := range []struct {
name string
cardMB int
profile string
want string
}{
{"NVIDIA A100-SXM4-80GB MIG 3g.40gb", 81920, "3g.40gb", "a100-80g-mig-3g.40gb"},
// The same profile name on a 40 GB card is another class: its
// instance is the same size, but the card is not, and neither is
// the set of profiles that card can be cut to.
{"NVIDIA A100-SXM4-40GB MIG 3g.20gb", 40960, "3g.20gb", "a100-40g-mig-3g.20gb"},
{"NVIDIA H100 80GB HBM3 MIG 1g.10gb+me", 81559, "1g.10gb+me", "h100-80g-mig-1g.10gb-me"},
{"NVIDIA H100 80GB HBM3 MIG 7g.80gb", 81559, "7g.80gb", "h100-80g-mig-7g.80gb"},
} {
g := GPU{Name: c.name, MemoryTotalMB: 10000, MIG: &MIGDevice{ParentMemoryTotalMB: c.cardMB, Profile: c.profile}}
if got := g.Class(); got != c.want {
t.Errorf("%s -> %s, want %s", c.name, got, c.want)
}
if !IsMIGClass(g.Class()) {
t.Errorf("%s is not recognised as a MIG class", g.Class())
}
}
if IsMIGClass("a100-80g") || IsMIGClass("6000-24g") {
t.Error("a whole card's class was taken for a MIG one")
}
if MIGProfileMemoryGB("a100-80g-mig-3g.40gb") != 40 || MIGProfileMemoryGB("1g.10gb+me") != 10 || MIGProfileMemoryGB("a100-80g") != 0 {
t.Error("the instance's memory is misread from its profile")
}
}
func TestDeviceNumbersAreShownAsCardAndInstance(t *testing.T) {
for i, want := range map[int]string{0: "0", 7: "7", MIGIndex(0, 2): "0/mig2", MIGIndex(3, 0): "3/mig0", 1302: "3/mig2"} {
if got := DeviceLabel(i); got != want {
t.Errorf("DeviceLabel(%d) = %s, want %s", i, got, want)
}
}
if IsMIGIndex(15) || !IsMIGIndex(MIGIndex(0, 0)) {
t.Error("whole cards and instances are told apart wrongly")
}
}
func TestACardInMIGModeWithNoInstancesIsKeptOutOfPlacement(t *testing.T) {
cards, _ := parseSMI(migQuery)
report, _ := parseMIGReport([]byte(strings.Replace(migReport,
migReport[strings.Index(migReport, ""):strings.Index(migReport, "")+len("")],
"None", 1)))
gpus := expandMIG(cards, parseSMIList("GPU 0: NVIDIA A100-SXM4-80GB (UUID: GPU-x)\nGPU 1: NVIDIA A100-SXM4-80GB (UUID: GPU-y)\n"), report)
if len(gpus) != 2 || gpus[0].IsMIG() {
t.Fatalf("got %+v", gpus)
}
// CUDA will not open the whole card while MIG is on, and there is no
// instance to open instead: placing anything here fails at start.
if gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "no instances") {
t.Errorf("card 0's problems: %v", gpus[0].Health.Problems)
}
if !gpus[1].Health.OK() {
t.Errorf("card 1, not in MIG mode, was flagged: %v", gpus[1].Health.Problems)
}
// Instances the report sees but -L could not name cannot be given to an
// engine either; the card says why.
report, _ = parseMIGReport([]byte(migReport))
gpus = expandMIG(cards, nil, report)
if len(gpus) != 2 || gpus[0].Health.OK() || !strings.Contains(gpus[0].Health.Problems[0], "did not name") {
t.Errorf("with -L failing: %+v", gpus)
}
}
func TestInstancesAreFoundFromTheListAloneWhenTheReportCannotBeRead(t *testing.T) {
cards, _ := parseSMI(migQuery)
gpus := expandMIG(cards, parseSMIList(migList), nil)
if len(gpus) != 3 || !gpus[0].IsMIG() || gpus[0].MemoryTotalMB != 40*1024 {
t.Errorf("without the report: %+v", gpus)
}
}
func TestAnOlderDriversInstanceUUIDIsTakenAsItIs(t *testing.T) {
// R450 and R460 name an instance by its card and ids, and CUDA of that
// era takes exactly that in CUDA_VISIBLE_DEVICES.
l := parseSMIList("GPU 0: A100-SXM4-40GB (UUID: GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77)\n" +
" MIG 1g.5gb Device 0: (UUID: MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0)\n")
if len(l[0]) != 1 || l[0][0].UUID != "MIG-GPU-5d5ba0d6-d33d-2b2c-524d-9e3d8d2b8a77/7/0" || l[0][0].Profile != "1g.5gb" {
t.Errorf("parsed %+v", l)
}
}
func TestProcessesOnAMIGCardAreChargedToTheirOwnInstance(t *testing.T) {
mem, err := parseGPUMemory([]byte(migReport))
if err != nil {
t.Fatal(err)
}
want := []GPUMemory{
{Index: 1000, UsedMB: 37},
{Index: 1001, UsedMB: 21000, Procs: []GPUProcess{{PID: 4100, UsedMB: 20900}}},
{Index: 1, UsedMB: 300, Procs: []GPUProcess{{PID: 5200, UsedMB: 250}}},
}
if !reflect.DeepEqual(mem, want) {
t.Errorf("got %+v\nwant %+v", mem, want)
}
// In the order discovery reports the devices, so a heartbeat's readings
// line up with the inventory one for one.
gpus := migMachine(t)
for i := range gpus {
if mem[i].Index != gpus[i].Index {
t.Errorf("reading %d is for device %d, inventory has %d", i, mem[i].Index, gpus[i].Index)
}
}
procs, _ := parseGPUProcesses([]byte(migReport))
if !reflect.DeepEqual(procs, map[int][]int{1001: {4100}, 1: {5200}}) {
t.Errorf("processes by device: %v", procs)
}
}
func TestAMIGInstanceTakesItsCardsPlaceInTheTopologyButNoNVLink(t *testing.T) {
gpus := migMachine(t)
topo := Topology{
NVLink: map[int]map[int]int{0: {1: 12}, 1: {0: 12}},
NUMA: map[int]int{0: 0, 1: 1},
NICPath: map[int]map[string]string{0: {"mlx5_0": "PIX"}},
}
topo.Apply(gpus)
if len(gpus[0].NVLinkPeers) != 0 || len(gpus[1].NVLinkPeers) != 0 {
t.Errorf("MIG instances were given NVLink peers: %v %v", gpus[0].NVLinkPeers, gpus[1].NVLinkPeers)
}
if gpus[1].NUMANode != 0 || gpus[1].NICPaths["mlx5_0"] != "PIX" {
t.Errorf("instance 1 did not take its card's NUMA node and adapters: %+v", gpus[1])
}
if gpus[2].NUMANode != 1 || !slices.Equal(gpus[2].NVLinkPeers, []int{0}) {
t.Errorf("the whole card's topology: %+v", gpus[2])
}
}
func TestMIGInstancesAreDiscoveredThroughNvidiaSMI(t *testing.T) {
// The real code path against a stub binary that answers each command as
// the machine above would.
dir := t.TempDir()
stub := filepath.Join(dir, "nvidia-smi")
script := "#!/bin/sh\ncase \"$1\" in\n" +
"-L) cat <<'EOF'\n" + migList + "EOF\n;;\n" +
"-q) cat <<'EOF'\n" + migReport + "\nEOF\n;;\n" +
"*) cat <<'EOF'\n" + migQuery + "EOF\n;;\nesac\n"
if err := os.WriteFile(stub, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
gpus, err := DiscoverGPUsSMI(t.Context(), stub)
if err != nil {
t.Fatal(err)
}
inv := Inventory{GPUs: gpus}
if !slices.Equal(inv.Classes(), []string{"a100-80g", "a100-80g-mig-3g.40gb"}) || len(gpus) != 3 {
t.Errorf("classes %v from %d devices", inv.Classes(), len(gpus))
}
}
// fakeMIGCard is NVML's view of one card, as nvmlDeviceGetMigMode,
// nvmlDeviceGetMaxMigDeviceCount and nvmlDeviceGetMigDeviceHandleByIndex
// would give it.
type fakeMIGCard struct {
mode, pending, supported bool
max int
devs map[int]fakeMIGInstance
}
func (c fakeMIGCard) MIGMode() (bool, bool, bool) { return c.mode, c.pending, c.supported }
func (c fakeMIGCard) MaxMIGDevices() int { return c.max }
func (c fakeMIGCard) MIGDevice(i int) (nvmlMIGInstance, bool) {
d, ok := c.devs[i]
return d, ok
}
type fakeMIGInstance struct {
uuid, name string
total, used int
gi, ci int
slices, attrMem int
}
func (m fakeMIGInstance) UUID() string { return m.uuid }
func (m fakeMIGInstance) Name() string { return m.name }
func (m fakeMIGInstance) Memory() (int, int, bool) {
return m.total, m.used, m.total > 0
}
func (m fakeMIGInstance) Instances() (int, int) { return m.gi, m.ci }
func (m fakeMIGInstance) Attributes() (int, int, bool) {
return m.slices, m.attrMem, m.slices > 0
}
func TestNVMLReportsACardInMIGModeAsItsInstances(t *testing.T) {
card := GPU{Index: 2, UUID: "GPU-card", Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559,
ComputeCapability: "9.0", NUMANode: 1, Health: GPUHealth{TemperatureC: 40}}
// Seven slots, three filled, one of them on a driver that does not put
// the profile in the device's name.
dev := fakeMIGCard{mode: true, pending: true, supported: true, max: 7, devs: map[int]fakeMIGInstance{
0: {uuid: "MIG-aaa", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 2, ci: 0},
1: {uuid: "MIG-bbb", name: "NVIDIA H100 80GB HBM3 MIG 3g.40gb", total: 40448, used: 10, gi: 1, ci: 0},
4: {uuid: "MIG-ccc", name: "NVIDIA H100 80GB HBM3", total: 9984, gi: 9, ci: 0, slices: 1, attrMem: 9984},
}}
gpus, isMIG := migFromNVML(card, dev)
if !isMIG || len(gpus) != 3 {
t.Fatalf("got %d devices, MIG %v", len(gpus), isMIG)
}
var classes []string
for _, g := range gpus {
classes = append(classes, DeviceLabel(g.Index)+" "+g.UUID+" "+g.Class())
}
want := []string{"2/mig0 MIG-aaa h100-80g-mig-3g.40gb", "2/mig1 MIG-bbb h100-80g-mig-3g.40gb",
"2/mig4 MIG-ccc h100-80g-mig-1g.10gb"}
if !slices.Equal(classes, want) {
t.Errorf("devices %q, want %q", classes, want)
}
if g := gpus[1]; g.MemoryTotalMB != 40448 || g.MIG.GPUInstance != 1 || g.NUMANode != 1 ||
g.Health.TemperatureC != 40 || len(g.NVLinkPeers) != 0 {
t.Errorf("instance 1: %+v", g)
}
// MIG on and nothing carved: the card, kept out of placement.
gpus, isMIG = migFromNVML(card, fakeMIGCard{mode: true, supported: true, max: 7})
if !isMIG || len(gpus) != 1 || gpus[0].IsMIG() || gpus[0].Health.OK() {
t.Errorf("an empty MIG card: %+v", gpus)
}
// MIG off, or a card without it: the card, as it always was.
for _, d := range []fakeMIGCard{{supported: true, max: 7}, {}} {
gpus, isMIG = migFromNVML(card, d)
if isMIG || len(gpus) != 1 || !reflect.DeepEqual(gpus[0], card) {
t.Errorf("a card not in MIG mode came back as %+v", gpus)
}
}
}
func TestMIGInstanceTablesAreRead(t *testing.T) {
gis := ParseMIGInstances(migLGI)
if len(gis) != 2 || gis[0] != (MIGInstance{GPU: 0, Profile: "3g.40gb", ProfileID: 9, GPUInstance: 1,
ComputeInstance: -1, Placement: "4:4"}) {
t.Errorf("-lgi: %+v", gis)
}
cis := ParseMIGInstances(migLCI)
if len(cis) != 2 || cis[1] != (MIGInstance{GPU: 0, GPUInstance: 2, Profile: "3g.40gb", ProfileID: 2,
ComputeInstance: 0, Placement: "0:3"}) {
t.Errorf("-lci: %+v", cis)
}
if got := ParseMIGInstances("No GPU instances found: Not Found\n"); len(got) != 0 {
t.Errorf("nothing to list read as %+v", got)
}
cards, err := buildMIGCards([]byte(migReport), migList, migLGI, migLCI)
if err != nil {
t.Fatal(err)
}
if len(cards) != 2 || !cards[0].Enabled || len(cards[0].Instances) != 2 || len(cards[0].GPUInstances) != 2 ||
len(cards[0].ComputeInstances) != 2 || cards[1].Enabled {
t.Errorf("cards: %+v", cards)
}
}