Ilker ManapandClaude Opus 5.5
82a63cfc6e
Every minute the agent reads nvidia-smi's full report, its NVLink error counters and the kernel's XID lines. They are published per card, shown on the node page and the GPU map, charted in Grafana, and five alerts watch them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
102 lines
3.5 KiB
Go
102 lines
3.5 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
const healthXML = `<?xml version="1.0" ?>
|
|
<nvidia_smi_log>
|
|
<gpu id="00000000:3B:00.0">
|
|
<pci><replay_counter>0</replay_counter></pci>
|
|
<retired_pages>
|
|
<multiple_single_bit_retirement><retired_count>N/A</retired_count></multiple_single_bit_retirement>
|
|
<double_bit_retirement><retired_count>N/A</retired_count></double_bit_retirement>
|
|
<pending_retirement>N/A</pending_retirement>
|
|
</retired_pages>
|
|
<remapped_rows>
|
|
<remapped_row_corr>2</remapped_row_corr>
|
|
<remapped_row_unc>1</remapped_row_unc>
|
|
<remapped_row_pending>Yes</remapped_row_pending>
|
|
<remapped_row_failure>No</remapped_row_failure>
|
|
</remapped_rows>
|
|
</gpu>
|
|
<gpu id="00000000:5E:00.0">
|
|
<pci><replay_counter>4021</replay_counter></pci>
|
|
<retired_pages>
|
|
<multiple_single_bit_retirement><retired_count>3</retired_count></multiple_single_bit_retirement>
|
|
<double_bit_retirement><retired_count>1</retired_count></double_bit_retirement>
|
|
<pending_blacklist>Yes</pending_blacklist>
|
|
</retired_pages>
|
|
<remapped_rows>N/A</remapped_rows>
|
|
</gpu>
|
|
</nvidia_smi_log>`
|
|
|
|
const nvlinkOut = `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-aaa)
|
|
Link 0: Replay Errors: 1
|
|
Link 0: Recovery Errors: 0
|
|
Link 0: CRC Errors: 5
|
|
Link 1: Data CRC Errors: 2
|
|
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-bbb)
|
|
Link 0: Replay Errors: 0
|
|
`
|
|
|
|
func TestAGPUsWearAndFailuresAreReadAndCountedFromWhenTheAgentStarted(t *testing.T) {
|
|
now := time.Unix(1_800_000_000, 0)
|
|
journal := ""
|
|
h := &HealthSampler{now: func() time.Time { return now }, xids: map[int]*GPUWear{},
|
|
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
|
switch {
|
|
case name == "nvidia-smi" && args[0] == "-q":
|
|
return []byte(healthXML), nil
|
|
case name == "nvidia-smi" && args[0] == "nvlink":
|
|
return []byte(nvlinkOut), nil
|
|
case name == "journalctl":
|
|
if journal == "" {
|
|
return nil, errors.New("exit status 1")
|
|
}
|
|
return []byte(journal), nil
|
|
}
|
|
return nil, errors.New("unexpected " + name)
|
|
}}
|
|
got := h.Sample(t.Context())
|
|
if len(got) != 2 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
a, b := got[0], got[1]
|
|
if a.RemappedCorrectable != 2 || a.RemappedUncorrectable != 1 || !a.RemapPending || a.RemapFailure ||
|
|
a.RetiredSBE != -1 || a.PCIeReplays != 0 || a.NVLinkCRC != 7 || a.NVLinkReplay != 1 {
|
|
t.Errorf("card 0 %+v", a)
|
|
}
|
|
if b.PCIeReplays != 4021 || b.RetiredSBE != 3 || b.RetiredDBE != 1 || !b.RetirePending || b.RemappedCorrectable != -1 {
|
|
t.Errorf("card 1 %+v", b)
|
|
}
|
|
|
|
// An XID on card 1, logged after the agent started; read on the next
|
|
// sample a minute on.
|
|
journal = "1800000030.123456 gpu07 kernel: NVRM: Xid (PCI:0000:5e:00): 79, pid=1234, name=llama-server, GPU has fallen off the bus.\n"
|
|
now = now.Add(61 * time.Second)
|
|
got = h.Sample(t.Context())
|
|
if got[1].XIDs != 1 || got[1].LastXID != 79 || !strings.Contains(got[1].LastXIDText, "fallen off the bus") ||
|
|
got[1].LastXIDAt.Unix() != 1_800_000_030 || got[0].XIDs != 0 {
|
|
t.Errorf("after an XID: %+v / %+v", got[0], got[1])
|
|
}
|
|
// Within the minute, the last reading.
|
|
journal = ""
|
|
now = now.Add(10 * time.Second)
|
|
if again := h.Sample(t.Context()); again[1].XIDs != 1 {
|
|
t.Errorf("%+v", again[1])
|
|
}
|
|
}
|
|
|
|
func TestBusIDsAreReadAsTheKernelWritesThem(t *testing.T) {
|
|
for in, want := range map[string]string{"00000000:3B:00.0": "3b:00", "0000:3b:00": "3b:00", "0000:5e:00.0": "5e:00"} {
|
|
if got := busOf(in); got != want {
|
|
t.Errorf("%s: %s, want %s", in, got, want)
|
|
}
|
|
}
|
|
}
|