package discovery import ( "context" "errors" "testing" "time" ) // dcgmBody is dcgm-exporter's output for two cards and a MIG instance: // card 0 profiled, card 1 a model DCGM cannot profile, which has the device // fields and none of the PROF ones. const dcgmBody = `# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %). # TYPE DCGM_FI_DEV_GPU_UTIL gauge DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 100 DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-b",device="nvidia1",modelName="Quadro RTX 6000",Hostname="n1"} 92 # HELP DCGM_FI_PROF_SM_ACTIVE The ratio of cycles an SM has at least 1 warp assigned. # TYPE DCGM_FI_PROF_SM_ACTIVE gauge DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.007042 DCGM_FI_PROF_SM_ACTIVE{gpu="2",UUID="GPU-c",device="nvidia2",modelName="NVIDIA A100",GPU_I_PROFILE="3g.40gb",GPU_I_ID="1",Hostname="n1"} 0.5 DCGM_FI_PROF_SM_OCCUPANCY{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.25 DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0 ` func TestDCGMExporterSaysHowMuchOfEachChipIsInUse(t *testing.T) { got := parseDCGM([]byte(dcgmBody)) if len(got) != 1 { t.Fatalf("want card 0 alone (card 1 unprofiled, card 2 a MIG instance): %+v", got) } p := got[0] // The LinkedIn L40S: 100% busy, one SM of 142 at work. if p.SMActive < 0.70 || p.SMActive > 0.71 || p.SMOccupancy != 25 || p.TensorActive != 0 || p.Source != "dcgm" { t.Errorf("card 0 %+v", p) } // A field the exporter does not send is unknown, not zero. if p.DRAMActive != -1 { t.Errorf("DRAM active not sent, read as %v", p.DRAMActive) } } type fakeGPM map[int]GPUProfile func (f fakeGPM) read() map[int]GPUProfile { return f } func TestGPMIsPreferredAndDCGMFillsTheCardsItDoesNotCover(t *testing.T) { calls := 0 p := &ProfSampler{ gpm: fakeGPM{1: {SMActive: 40, Source: "gpm"}}, dcgm: DefaultDCGMExporter, get: func(context.Context, string) ([]byte, error) { calls++ return []byte(`DCGM_FI_PROF_SM_ACTIVE{gpu="0"} 0.5` + "\n" + `DCGM_FI_PROF_SM_ACTIVE{gpu="1"} 0.9` + "\n"), nil }, now: time.Now, } got := p.Sample(t.Context()) if got[0].Source != "dcgm" || got[0].SMActive != 50 { t.Errorf("card 0 %+v", got[0]) } if got[1].Source != "gpm" || got[1].SMActive != 40 { t.Errorf("GPM's card was taken from DCGM: %+v", got[1]) } if calls != 1 { t.Errorf("%d calls", calls) } } func TestANodeWithoutDCGMIsNotAskedEverySample(t *testing.T) { now := time.Now() calls := 0 p := &ProfSampler{dcgm: DefaultDCGMExporter, now: func() time.Time { return now }, get: func(context.Context, string) ([]byte, error) { calls++; return nil, errors.New("connection refused") }} for range 5 { if got := p.Sample(t.Context()); len(got) != 0 { t.Fatalf("%+v", got) } now = now.Add(10 * time.Second) } if calls != 1 { t.Errorf("asked %d times in 50 s", calls) } now = now.Add(dcgmRetry) p.Sample(t.Context()) if calls != 2 { t.Errorf("not asked again after %v", dcgmRetry) } if off := NewProfSampler("off"); off.dcgm != "" { t.Errorf("off still asks %q", off.dcgm) } } func TestEachSampleCarriesTheProfileOfTheCardsThatHaveOne(t *testing.T) { smi := func(context.Context, string) ([]byte, error) { return []byte("0, 71, 239, 260, 0x0, 92, 30, 23000, 24576, 1878, 0, 0\n" + "1, 30, 20, 260, 0x0, 0, 0, 512, 24576, 300, 0, 0\n"), nil } now := time.Now() s := sampler(t.TempDir(), smi, &now).WithProfile(&ProfSampler{ gpm: fakeGPM{0: {SMActive: 69, SMOccupancy: 45, TensorActive: 1, DRAMActive: 33, Source: "gpm"}}, now: time.Now}) got := s.Sample(t.Context()).GPUs if got[0].Profile == nil || got[0].Profile.SMActive != 69 || got[0].UtilPercent != 92 { t.Errorf("card 0 %+v", got[0]) } if got[1].Profile != nil { t.Errorf("card 1 has a profile nobody read: %+v", got[1].Profile) } }