Read what wears a GPU out: XIDs, remapped rows, PCIe and NVLink errors

Every minute the agent reads nvidia-smi's full report, its NVLink error
counters and the kernel's XID lines. They are published per card,
shown on the node page and the GPU map, charted in Grafana, and five
alerts watch them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 16:02:07 +02:00
1 parent ff554c7062
commit 82a63cfc6e
16 files changed
+885 -11

No files matched your search

+152 -4
View File
@@ -1561,8 +1561,27 @@ type GPUSensor struct {
SmClockMhz float64 `protobuf:"fixed64,11,opt,name=sm_clock_mhz,json=smClockMhz,proto3" json:"sm_clock_mhz,omitempty"`
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
EccUncorrected float64 `protobuf:"fixed64,12,opt,name=ecc_uncorrected,json=eccUncorrected,proto3" json:"ecc_uncorrected,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// What wears the card out, read every minute; has_wear says these are
// present. Counts of -1 are not reported for this card.
HasWear bool `protobuf:"varint,13,opt,name=has_wear,json=hasWear,proto3" json:"has_wear,omitempty"`
PcieReplays int64 `protobuf:"varint,14,opt,name=pcie_replays,json=pcieReplays,proto3" json:"pcie_replays,omitempty"`
RetiredPagesSbe int64 `protobuf:"varint,15,opt,name=retired_pages_sbe,json=retiredPagesSbe,proto3" json:"retired_pages_sbe,omitempty"`
RetiredPagesDbe int64 `protobuf:"varint,16,opt,name=retired_pages_dbe,json=retiredPagesDbe,proto3" json:"retired_pages_dbe,omitempty"`
RetirementPending bool `protobuf:"varint,17,opt,name=retirement_pending,json=retirementPending,proto3" json:"retirement_pending,omitempty"`
RemappedRowsCorrectable int64 `protobuf:"varint,18,opt,name=remapped_rows_correctable,json=remappedRowsCorrectable,proto3" json:"remapped_rows_correctable,omitempty"`
RemappedRowsUncorrectable int64 `protobuf:"varint,19,opt,name=remapped_rows_uncorrectable,json=remappedRowsUncorrectable,proto3" json:"remapped_rows_uncorrectable,omitempty"`
RemapPending bool `protobuf:"varint,20,opt,name=remap_pending,json=remapPending,proto3" json:"remap_pending,omitempty"`
RemapFailure bool `protobuf:"varint,21,opt,name=remap_failure,json=remapFailure,proto3" json:"remap_failure,omitempty"`
NvlinkCrcErrors int64 `protobuf:"varint,22,opt,name=nvlink_crc_errors,json=nvlinkCrcErrors,proto3" json:"nvlink_crc_errors,omitempty"`
NvlinkReplayErrors int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"`
NvlinkRecoveryErrors int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"`
// xid_errors are the driver's error events since the agent started.
XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *GPUSensor) Reset() {
@@ -1679,6 +1698,118 @@ func (x *GPUSensor) GetEccUncorrected() float64 {
return 0
}
func (x *GPUSensor) GetHasWear() bool {
if x != nil {
return x.HasWear
}
return false
}
func (x *GPUSensor) GetPcieReplays() int64 {
if x != nil {
return x.PcieReplays
}
return 0
}
func (x *GPUSensor) GetRetiredPagesSbe() int64 {
if x != nil {
return x.RetiredPagesSbe
}
return 0
}
func (x *GPUSensor) GetRetiredPagesDbe() int64 {
if x != nil {
return x.RetiredPagesDbe
}
return 0
}
func (x *GPUSensor) GetRetirementPending() bool {
if x != nil {
return x.RetirementPending
}
return false
}
func (x *GPUSensor) GetRemappedRowsCorrectable() int64 {
if x != nil {
return x.RemappedRowsCorrectable
}
return 0
}
func (x *GPUSensor) GetRemappedRowsUncorrectable() int64 {
if x != nil {
return x.RemappedRowsUncorrectable
}
return 0
}
func (x *GPUSensor) GetRemapPending() bool {
if x != nil {
return x.RemapPending
}
return false
}
func (x *GPUSensor) GetRemapFailure() bool {
if x != nil {
return x.RemapFailure
}
return false
}
func (x *GPUSensor) GetNvlinkCrcErrors() int64 {
if x != nil {
return x.NvlinkCrcErrors
}
return 0
}
func (x *GPUSensor) GetNvlinkReplayErrors() int64 {
if x != nil {
return x.NvlinkReplayErrors
}
return 0
}
func (x *GPUSensor) GetNvlinkRecoveryErrors() int64 {
if x != nil {
return x.NvlinkRecoveryErrors
}
return 0
}
func (x *GPUSensor) GetXidErrors() int64 {
if x != nil {
return x.XidErrors
}
return 0
}
func (x *GPUSensor) GetLastXid() int32 {
if x != nil {
return x.LastXid
}
return 0
}
func (x *GPUSensor) GetLastXidText() string {
if x != nil {
return x.LastXidText
}
return ""
}
func (x *GPUSensor) GetLastXidUnix() int64 {
if x != nil {
return x.LastXidUnix
}
return 0
}
// ShardLoad is one engine process group's use.
type ShardLoad struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -4051,7 +4182,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
" \x01(\x01R\tcpuPowerW\x12 \n" +
"\finlet_temp_c\x18\v \x01(\x01R\n" +
"inletTempC\x12'\n" +
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xbd\x03\n" +
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" +
"\tGPUSensor\x12\x14\n" +
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" +
"\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" +
@@ -4066,7 +4197,24 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
" \x01(\x01R\rmemoryTotalMb\x12 \n" +
"\fsm_clock_mhz\x18\v \x01(\x01R\n" +
"smClockMhz\x12'\n" +
"\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\"m\n" +
"\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\x12\x19\n" +
"\bhas_wear\x18\r \x01(\bR\ahasWear\x12!\n" +
"\fpcie_replays\x18\x0e \x01(\x03R\vpcieReplays\x12*\n" +
"\x11retired_pages_sbe\x18\x0f \x01(\x03R\x0fretiredPagesSbe\x12*\n" +
"\x11retired_pages_dbe\x18\x10 \x01(\x03R\x0fretiredPagesDbe\x12-\n" +
"\x12retirement_pending\x18\x11 \x01(\bR\x11retirementPending\x12:\n" +
"\x19remapped_rows_correctable\x18\x12 \x01(\x03R\x17remappedRowsCorrectable\x12>\n" +
"\x1bremapped_rows_uncorrectable\x18\x13 \x01(\x03R\x19remappedRowsUncorrectable\x12#\n" +
"\rremap_pending\x18\x14 \x01(\bR\fremapPending\x12#\n" +
"\rremap_failure\x18\x15 \x01(\bR\fremapFailure\x12*\n" +
"\x11nvlink_crc_errors\x18\x16 \x01(\x03R\x0fnvlinkCrcErrors\x120\n" +
"\x14nvlink_replay_errors\x18\x17 \x01(\x03R\x12nvlinkReplayErrors\x124\n" +
"\x16nvlink_recovery_errors\x18\x18 \x01(\x03R\x14nvlinkRecoveryErrors\x12\x1d\n" +
"\n" +
"xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" +
"\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" +
"\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" +
"\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" +
"\tShardLoad\x12\x19\n" +
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
+19
View File
@@ -314,6 +314,25 @@ message GPUSensor {
double sm_clock_mhz = 11;
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
double ecc_uncorrected = 12;
// What wears the card out, read every minute; has_wear says these are
// present. Counts of -1 are not reported for this card.
bool has_wear = 13;
int64 pcie_replays = 14;
int64 retired_pages_sbe = 15;
int64 retired_pages_dbe = 16;
bool retirement_pending = 17;
int64 remapped_rows_correctable = 18;
int64 remapped_rows_uncorrectable = 19;
bool remap_pending = 20;
bool remap_failure = 21;
int64 nvlink_crc_errors = 22;
int64 nvlink_replay_errors = 23;
int64 nvlink_recovery_errors = 24;
// xid_errors are the driver's error events since the agent started.
int64 xid_errors = 25;
int32 last_xid = 26;
string last_xid_text = 27;
int64 last_xid_unix = 28;
}
// ShardLoad is one engine process group's use.