package doctor import ( "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) // migFacts is the healthy node with card 0 in MIG mode, as given. func migFacts(card discovery.MIGCard) Facts { f := healthy() f.MIG = []discovery.MIGCard{card} return f } func TestTheDoctorListsACardsMIGInstances(t *testing.T) { fs := Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "NVIDIA A100-SXM4-80GB", Enabled: true, PendingEnabled: true, Instances: []discovery.MIGListed{{Device: 0, Profile: "3g.40gb", UUID: "MIG-aaa"}, {Device: 1, Profile: "3g.40gb", UUID: "MIG-bbb"}}, GPUInstances: []discovery.MIGInstance{{GPU: 0, Profile: "3g.40gb", GPUInstance: 1, ComputeInstance: -1}, {GPU: 0, Profile: "3g.40gb", GPUInstance: 2, ComputeInstance: -1}}, ComputeInstances: []discovery.MIGInstance{{GPU: 0, Profile: "3g.40gb", GPUInstance: 1}, {GPU: 0, Profile: "3g.40gb", GPUInstance: 2}}, })) has(t, fs, OK, "GPU 0 (NVIDIA A100-SXM4-80GB) is in MIG mode with 2 instance(s)") has(t, fs, OK, "1: 3g.40gb MIG-bbb") for _, f := range fs { if f.Area == "mig" && f.Level != OK { t.Errorf("a card with its instances made was warned of: %+v", f) } } } func TestTheDoctorWarnsOfMIGEnabledWithNoInstances(t *testing.T) { // The state a card is left in by nvidia-smi -mig 1 alone: in MIG mode, // so CUDA will not open it whole, and nothing carved to open instead. fs := Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "NVIDIA A100-SXM4-80GB", Enabled: true, PendingEnabled: true})) has(t, fs, Warn, "is in MIG mode with no instances created") for _, f := range fs { if f.Area == "mig" && f.Level == Warn && !strings.Contains(f.Fix, "nvidia-smi mig -i 0 -cgi") { t.Errorf("the fix does not say how to create them: %q", f.Fix) } } // GPU instances with no compute instance in them are not devices either. fs = Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "A100", Enabled: true, PendingEnabled: true, GPUInstances: []discovery.MIGInstance{{GPU: 0, Profile: "7g.80gb", GPUInstance: 0, ComputeInstance: -1}}})) has(t, fs, Warn, "GPU instances 0 (7g.80gb) but no compute instances") // A mode change waiting for a reset. fs = Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "A100", Enabled: false, PendingEnabled: true})) has(t, fs, Warn, "MIG turning on at its next reset") } func TestACardWithMIGOffIsNotMentioned(t *testing.T) { fs := Evaluate(migFacts(discovery.MIGCard{Index: 0, Name: "A100"})) for _, f := range fs { if f.Area == "mig" { t.Errorf("a card with MIG off was reported: %+v", f) } } } func TestNVLinkIsJudgedBetweenWholeCardsOnly(t *testing.T) { // Every card of the node in MIG mode: the instances have no NVLink, and // that is not a node "without NVLink" to warn about. f := healthy() f.GPUs = nil for card := range 2 { for dev := range 2 { f.GPUs = append(f.GPUs, discovery.GPU{Index: discovery.MIGIndex(card, dev), Name: "A100 MIG 3g.40gb", MIG: &discovery.MIGDevice{ParentIndex: card, Device: dev, Profile: "3g.40gb"}}) } } for _, x := range Evaluate(f) { if x.Area == "nvlink" && x.Level != OK { t.Errorf("MIG instances were judged for NVLink: %+v", x) } } }