package discovery import ( "context" "errors" "strings" "testing" "time" ) const healthXML = ` 0 N/A N/A N/A 2 1 Yes No 4021 3 1 Yes N/A ` const nvlinkOut = `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-aaa) Link 0: Replay Errors: 1 Link 0: Recovery Errors: 0 Link 0: CRC Errors: 5 Link 1: Data CRC Errors: 2 GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-bbb) Link 0: Replay Errors: 0 ` func TestAGPUsWearAndFailuresAreReadAndCountedFromWhenTheAgentStarted(t *testing.T) { now := time.Unix(1_800_000_000, 0) journal := "" h := &HealthSampler{now: func() time.Time { return now }, xids: map[int]*GPUWear{}, run: func(_ context.Context, name string, args ...string) ([]byte, error) { switch { case name == "nvidia-smi" && args[0] == "-q": return []byte(healthXML), nil case name == "nvidia-smi" && args[0] == "nvlink": return []byte(nvlinkOut), nil case name == "journalctl": if journal == "" { return nil, errors.New("exit status 1") } return []byte(journal), nil } return nil, errors.New("unexpected " + name) }} got := h.Sample(t.Context()) if len(got) != 2 { t.Fatalf("%+v", got) } a, b := got[0], got[1] if a.RemappedCorrectable != 2 || a.RemappedUncorrectable != 1 || !a.RemapPending || a.RemapFailure || a.RetiredSBE != -1 || a.PCIeReplays != 0 || a.NVLinkCRC != 7 || a.NVLinkReplay != 1 { t.Errorf("card 0 %+v", a) } if b.PCIeReplays != 4021 || b.RetiredSBE != 3 || b.RetiredDBE != 1 || !b.RetirePending || b.RemappedCorrectable != -1 { t.Errorf("card 1 %+v", b) } // An XID on card 1, logged after the agent started; read on the next // sample a minute on. journal = "1800000030.123456 gpu07 kernel: NVRM: Xid (PCI:0000:5e:00): 79, pid=1234, name=llama-server, GPU has fallen off the bus.\n" now = now.Add(61 * time.Second) got = h.Sample(t.Context()) if got[1].XIDs != 1 || got[1].LastXID != 79 || !strings.Contains(got[1].LastXIDText, "fallen off the bus") || got[1].LastXIDAt.Unix() != 1_800_000_030 || got[0].XIDs != 0 { t.Errorf("after an XID: %+v / %+v", got[0], got[1]) } // Within the minute, the last reading. journal = "" now = now.Add(10 * time.Second) if again := h.Sample(t.Context()); again[1].XIDs != 1 { t.Errorf("%+v", again[1]) } } func TestBusIDsAreReadAsTheKernelWritesThem(t *testing.T) { for in, want := range map[string]string{"00000000:3B:00.0": "3b:00", "0000:3b:00": "3b:00", "0000:5e:00.0": "5e:00"} { if got := busOf(in); got != want { t.Errorf("%s: %s, want %s", in, got, want) } } }