Ilker ManapandClaude Opus 5.5
c866b3a8ee
CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
108 lines
3.9 KiB
Go
108 lines
3.9 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// dcgmBody is dcgm-exporter's output for two cards and a MIG instance:
|
|
// card 0 profiled, card 1 a model DCGM cannot profile, which has the device
|
|
// fields and none of the PROF ones.
|
|
const dcgmBody = `# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
|
|
# TYPE DCGM_FI_DEV_GPU_UTIL gauge
|
|
DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 100
|
|
DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-b",device="nvidia1",modelName="Quadro RTX 6000",Hostname="n1"} 92
|
|
# HELP DCGM_FI_PROF_SM_ACTIVE The ratio of cycles an SM has at least 1 warp assigned.
|
|
# TYPE DCGM_FI_PROF_SM_ACTIVE gauge
|
|
DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.007042
|
|
DCGM_FI_PROF_SM_ACTIVE{gpu="2",UUID="GPU-c",device="nvidia2",modelName="NVIDIA A100",GPU_I_PROFILE="3g.40gb",GPU_I_ID="1",Hostname="n1"} 0.5
|
|
DCGM_FI_PROF_SM_OCCUPANCY{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.25
|
|
DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0
|
|
`
|
|
|
|
func TestDCGMExporterSaysHowMuchOfEachChipIsInUse(t *testing.T) {
|
|
got := parseDCGM([]byte(dcgmBody))
|
|
if len(got) != 1 {
|
|
t.Fatalf("want card 0 alone (card 1 unprofiled, card 2 a MIG instance): %+v", got)
|
|
}
|
|
p := got[0]
|
|
// The LinkedIn L40S: 100% busy, one SM of 142 at work.
|
|
if p.SMActive < 0.70 || p.SMActive > 0.71 || p.SMOccupancy != 25 || p.TensorActive != 0 || p.Source != "dcgm" {
|
|
t.Errorf("card 0 %+v", p)
|
|
}
|
|
// A field the exporter does not send is unknown, not zero.
|
|
if p.DRAMActive != -1 {
|
|
t.Errorf("DRAM active not sent, read as %v", p.DRAMActive)
|
|
}
|
|
}
|
|
|
|
type fakeGPM map[int]GPUProfile
|
|
|
|
func (f fakeGPM) read() map[int]GPUProfile { return f }
|
|
|
|
func TestGPMIsPreferredAndDCGMFillsTheCardsItDoesNotCover(t *testing.T) {
|
|
calls := 0
|
|
p := &ProfSampler{
|
|
gpm: fakeGPM{1: {SMActive: 40, Source: "gpm"}},
|
|
dcgm: DefaultDCGMExporter,
|
|
get: func(context.Context, string) ([]byte, error) {
|
|
calls++
|
|
return []byte(`DCGM_FI_PROF_SM_ACTIVE{gpu="0"} 0.5` + "\n" + `DCGM_FI_PROF_SM_ACTIVE{gpu="1"} 0.9` + "\n"), nil
|
|
},
|
|
now: time.Now,
|
|
}
|
|
got := p.Sample(t.Context())
|
|
if got[0].Source != "dcgm" || got[0].SMActive != 50 {
|
|
t.Errorf("card 0 %+v", got[0])
|
|
}
|
|
if got[1].Source != "gpm" || got[1].SMActive != 40 {
|
|
t.Errorf("GPM's card was taken from DCGM: %+v", got[1])
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("%d calls", calls)
|
|
}
|
|
}
|
|
|
|
func TestANodeWithoutDCGMIsNotAskedEverySample(t *testing.T) {
|
|
now := time.Now()
|
|
calls := 0
|
|
p := &ProfSampler{dcgm: DefaultDCGMExporter, now: func() time.Time { return now },
|
|
get: func(context.Context, string) ([]byte, error) { calls++; return nil, errors.New("connection refused") }}
|
|
for range 5 {
|
|
if got := p.Sample(t.Context()); len(got) != 0 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
now = now.Add(10 * time.Second)
|
|
}
|
|
if calls != 1 {
|
|
t.Errorf("asked %d times in 50 s", calls)
|
|
}
|
|
now = now.Add(dcgmRetry)
|
|
p.Sample(t.Context())
|
|
if calls != 2 {
|
|
t.Errorf("not asked again after %v", dcgmRetry)
|
|
}
|
|
if off := NewProfSampler("off"); off.dcgm != "" {
|
|
t.Errorf("off still asks %q", off.dcgm)
|
|
}
|
|
}
|
|
|
|
func TestEachSampleCarriesTheProfileOfTheCardsThatHaveOne(t *testing.T) {
|
|
smi := func(context.Context, string) ([]byte, error) {
|
|
return []byte("0, 71, 239, 260, 0x0, 92, 30, 23000, 24576, 1878, 0, 0\n" +
|
|
"1, 30, 20, 260, 0x0, 0, 0, 512, 24576, 300, 0, 0\n"), nil
|
|
}
|
|
now := time.Now()
|
|
s := sampler(t.TempDir(), smi, &now).WithProfile(&ProfSampler{
|
|
gpm: fakeGPM{0: {SMActive: 69, SMOccupancy: 45, TensorActive: 1, DRAMActive: 33, Source: "gpm"}}, now: time.Now})
|
|
got := s.Sample(t.Context()).GPUs
|
|
if got[0].Profile == nil || got[0].Profile.SMActive != 69 || got[0].UtilPercent != 92 {
|
|
t.Errorf("card 0 %+v", got[0])
|
|
}
|
|
if got[1].Profile != nil {
|
|
t.Errorf("card 1 has a profile nobody read: %+v", got[1].Profile)
|
|
}
|
|
}
|