diff --git a/CHANGELOG.md b/CHANGELOG.md
index 905ccca..9bee72f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,17 @@ and a figure the driver does not give is left out rather than shown as zero.
- **What each model gets from its cards**, on the cluster page and in
Grafana: output tokens a second per GPU and per watt-hour, with the cards'
draw and average utilisation, over the last quarter hour.
+- **What wears a card out**, read every minute: XID errors from the kernel's
+ log (counted from when the agent started), memory rows remapped or pages
+ retired and whether either waits for a reset, a card out of spare rows,
+ PCIe link retries and NVLink errors. Published as `llmgw_gpu_xid_errors`,
+ `llmgw_gpu_last_xid`, `llmgw_gpu_remapped_rows`, `llmgw_gpu_remap_pending`,
+ `llmgw_gpu_remap_failure`, `llmgw_gpu_retired_pages`,
+ `llmgw_gpu_retirement_pending`, `llmgw_gpu_pcie_replays`,
+ `llmgw_gpu_nvlink_errors`; said in words on the node page; a red outline
+ on the GPU map; a Grafana panel; and alerts — `GPUXidError`,
+ `GPUOutOfSpareRows`, `GPUNeedsReset`, `GPUPCIeRetriesClimbing`,
+ `GPUNVLinkErrorsClimbing`.
- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds
each card and whose quota it counts against, for joining the GPU series to
models and tenants.
diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go
index d4f4efb..5a21fd7 100644
--- a/api/gen/llmgw/v1/agent.pb.go
+++ b/api/gen/llmgw/v1/agent.pb.go
@@ -1561,8 +1561,27 @@ type GPUSensor struct {
SmClockMhz float64 `protobuf:"fixed64,11,opt,name=sm_clock_mhz,json=smClockMhz,proto3" json:"sm_clock_mhz,omitempty"`
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
EccUncorrected float64 `protobuf:"fixed64,12,opt,name=ecc_uncorrected,json=eccUncorrected,proto3" json:"ecc_uncorrected,omitempty"`
- unknownFields protoimpl.UnknownFields
- sizeCache protoimpl.SizeCache
+ // What wears the card out, read every minute; has_wear says these are
+ // present. Counts of -1 are not reported for this card.
+ HasWear bool `protobuf:"varint,13,opt,name=has_wear,json=hasWear,proto3" json:"has_wear,omitempty"`
+ PcieReplays int64 `protobuf:"varint,14,opt,name=pcie_replays,json=pcieReplays,proto3" json:"pcie_replays,omitempty"`
+ RetiredPagesSbe int64 `protobuf:"varint,15,opt,name=retired_pages_sbe,json=retiredPagesSbe,proto3" json:"retired_pages_sbe,omitempty"`
+ RetiredPagesDbe int64 `protobuf:"varint,16,opt,name=retired_pages_dbe,json=retiredPagesDbe,proto3" json:"retired_pages_dbe,omitempty"`
+ RetirementPending bool `protobuf:"varint,17,opt,name=retirement_pending,json=retirementPending,proto3" json:"retirement_pending,omitempty"`
+ RemappedRowsCorrectable int64 `protobuf:"varint,18,opt,name=remapped_rows_correctable,json=remappedRowsCorrectable,proto3" json:"remapped_rows_correctable,omitempty"`
+ RemappedRowsUncorrectable int64 `protobuf:"varint,19,opt,name=remapped_rows_uncorrectable,json=remappedRowsUncorrectable,proto3" json:"remapped_rows_uncorrectable,omitempty"`
+ RemapPending bool `protobuf:"varint,20,opt,name=remap_pending,json=remapPending,proto3" json:"remap_pending,omitempty"`
+ RemapFailure bool `protobuf:"varint,21,opt,name=remap_failure,json=remapFailure,proto3" json:"remap_failure,omitempty"`
+ NvlinkCrcErrors int64 `protobuf:"varint,22,opt,name=nvlink_crc_errors,json=nvlinkCrcErrors,proto3" json:"nvlink_crc_errors,omitempty"`
+ NvlinkReplayErrors int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"`
+ NvlinkRecoveryErrors int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"`
+ // xid_errors are the driver's error events since the agent started.
+ XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
+ LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
+ LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
+ LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
}
func (x *GPUSensor) Reset() {
@@ -1679,6 +1698,118 @@ func (x *GPUSensor) GetEccUncorrected() float64 {
return 0
}
+func (x *GPUSensor) GetHasWear() bool {
+ if x != nil {
+ return x.HasWear
+ }
+ return false
+}
+
+func (x *GPUSensor) GetPcieReplays() int64 {
+ if x != nil {
+ return x.PcieReplays
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetRetiredPagesSbe() int64 {
+ if x != nil {
+ return x.RetiredPagesSbe
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetRetiredPagesDbe() int64 {
+ if x != nil {
+ return x.RetiredPagesDbe
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetRetirementPending() bool {
+ if x != nil {
+ return x.RetirementPending
+ }
+ return false
+}
+
+func (x *GPUSensor) GetRemappedRowsCorrectable() int64 {
+ if x != nil {
+ return x.RemappedRowsCorrectable
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetRemappedRowsUncorrectable() int64 {
+ if x != nil {
+ return x.RemappedRowsUncorrectable
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetRemapPending() bool {
+ if x != nil {
+ return x.RemapPending
+ }
+ return false
+}
+
+func (x *GPUSensor) GetRemapFailure() bool {
+ if x != nil {
+ return x.RemapFailure
+ }
+ return false
+}
+
+func (x *GPUSensor) GetNvlinkCrcErrors() int64 {
+ if x != nil {
+ return x.NvlinkCrcErrors
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetNvlinkReplayErrors() int64 {
+ if x != nil {
+ return x.NvlinkReplayErrors
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetNvlinkRecoveryErrors() int64 {
+ if x != nil {
+ return x.NvlinkRecoveryErrors
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetXidErrors() int64 {
+ if x != nil {
+ return x.XidErrors
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetLastXid() int32 {
+ if x != nil {
+ return x.LastXid
+ }
+ return 0
+}
+
+func (x *GPUSensor) GetLastXidText() string {
+ if x != nil {
+ return x.LastXidText
+ }
+ return ""
+}
+
+func (x *GPUSensor) GetLastXidUnix() int64 {
+ if x != nil {
+ return x.LastXidUnix
+ }
+ return 0
+}
+
// ShardLoad is one engine process group's use.
type ShardLoad struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -4051,7 +4182,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
" \x01(\x01R\tcpuPowerW\x12 \n" +
"\finlet_temp_c\x18\v \x01(\x01R\n" +
"inletTempC\x12'\n" +
- "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xbd\x03\n" +
+ "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" +
"\tGPUSensor\x12\x14\n" +
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" +
"\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" +
@@ -4066,7 +4197,24 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
" \x01(\x01R\rmemoryTotalMb\x12 \n" +
"\fsm_clock_mhz\x18\v \x01(\x01R\n" +
"smClockMhz\x12'\n" +
- "\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\"m\n" +
+ "\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\x12\x19\n" +
+ "\bhas_wear\x18\r \x01(\bR\ahasWear\x12!\n" +
+ "\fpcie_replays\x18\x0e \x01(\x03R\vpcieReplays\x12*\n" +
+ "\x11retired_pages_sbe\x18\x0f \x01(\x03R\x0fretiredPagesSbe\x12*\n" +
+ "\x11retired_pages_dbe\x18\x10 \x01(\x03R\x0fretiredPagesDbe\x12-\n" +
+ "\x12retirement_pending\x18\x11 \x01(\bR\x11retirementPending\x12:\n" +
+ "\x19remapped_rows_correctable\x18\x12 \x01(\x03R\x17remappedRowsCorrectable\x12>\n" +
+ "\x1bremapped_rows_uncorrectable\x18\x13 \x01(\x03R\x19remappedRowsUncorrectable\x12#\n" +
+ "\rremap_pending\x18\x14 \x01(\bR\fremapPending\x12#\n" +
+ "\rremap_failure\x18\x15 \x01(\bR\fremapFailure\x12*\n" +
+ "\x11nvlink_crc_errors\x18\x16 \x01(\x03R\x0fnvlinkCrcErrors\x120\n" +
+ "\x14nvlink_replay_errors\x18\x17 \x01(\x03R\x12nvlinkReplayErrors\x124\n" +
+ "\x16nvlink_recovery_errors\x18\x18 \x01(\x03R\x14nvlinkRecoveryErrors\x12\x1d\n" +
+ "\n" +
+ "xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" +
+ "\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" +
+ "\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" +
+ "\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" +
"\tShardLoad\x12\x19\n" +
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto
index 6a3a03b..3ee090c 100644
--- a/api/proto/llmgw/v1/agent.proto
+++ b/api/proto/llmgw/v1/agent.proto
@@ -314,6 +314,25 @@ message GPUSensor {
double sm_clock_mhz = 11;
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
double ecc_uncorrected = 12;
+ // What wears the card out, read every minute; has_wear says these are
+ // present. Counts of -1 are not reported for this card.
+ bool has_wear = 13;
+ int64 pcie_replays = 14;
+ int64 retired_pages_sbe = 15;
+ int64 retired_pages_dbe = 16;
+ bool retirement_pending = 17;
+ int64 remapped_rows_correctable = 18;
+ int64 remapped_rows_uncorrectable = 19;
+ bool remap_pending = 20;
+ bool remap_failure = 21;
+ int64 nvlink_crc_errors = 22;
+ int64 nvlink_replay_errors = 23;
+ int64 nvlink_recovery_errors = 24;
+ // xid_errors are the driver's error events since the agent started.
+ int64 xid_errors = 25;
+ int32 last_xid = 26;
+ string last_xid_text = 27;
+ int64 last_xid_unix = 28;
}
// ShardLoad is one engine process group's use.
diff --git a/deploy/monitoring/grafana/llmgw-overview.json b/deploy/monitoring/grafana/llmgw-overview.json
index ea533cc..cfe385b 100644
--- a/deploy/monitoring/grafana/llmgw-overview.json
+++ b/deploy/monitoring/grafana/llmgw-overview.json
@@ -855,6 +855,59 @@
"mode": "multi"
}
}
+ },
+ {
+ "type": "timeseries",
+ "title": "GPU wear",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "llmgw-prom"
+ },
+ "id": 24,
+ "description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.",
+ "gridPos": {
+ "x": 0,
+ "y": 66,
+ "w": 24,
+ "h": 7
+ },
+ "targets": [
+ {
+ "expr": "increase(llmgw_gpu_xid_errors[15m]) > 0",
+ "legendFormat": "XID {{node}} gpu {{gpu}}",
+ "refId": "A"
+ },
+ {
+ "expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0",
+ "legendFormat": "rows remapped {{node}} gpu {{gpu}}",
+ "refId": "B"
+ },
+ {
+ "expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0",
+ "legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}",
+ "refId": "C"
+ },
+ {
+ "expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0",
+ "legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}",
+ "refId": "D"
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom"
+ },
+ "tooltip": {
+ "mode": "multi"
+ }
+ }
}
]
}
\ No newline at end of file
diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml
index 16bf5af..e877019 100644
--- a/deploy/monitoring/llmgw-rules.yml
+++ b/deploy/monitoring/llmgw-rules.yml
@@ -224,6 +224,60 @@ groups:
card may be computing on corrupted memory. Drain the node
(llmgw ctl drain) and have the card looked at.
+ - alert: GPUXidError
+ expr: increase(llmgw_gpu_xid_errors[15m]) > 0
+ for: 0m
+ labels: { severity: warning }
+ annotations:
+ summary: "{{ $labels.node }} GPU {{ $labels.gpu }} logged an XID error"
+ description: >-
+ The driver reported an error on the card (its code is
+ llmgw_gpu_last_xid, and the node page says it in words). 79 is a
+ card fallen off the bus, 48 and 63/64 memory errors, 74 NVLink,
+ 94/95 contained and uncontained ECC errors: those need the card
+ looked at. Others are often an engine's own fault, and pass.
+
+ - alert: GPUOutOfSpareRows
+ expr: llmgw_gpu_remap_failure == 1
+ for: 1m
+ labels: { severity: critical }
+ annotations:
+ summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has no spare memory rows left"
+ description: >-
+ The card cannot remap another failing row. Drain the node and
+ have the card replaced.
+
+ - alert: GPUNeedsReset
+ expr: llmgw_gpu_remap_pending == 1 or llmgw_gpu_retirement_pending == 1
+ for: 5m
+ labels: { severity: warning }
+ annotations:
+ summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has memory repairs waiting for a reset"
+ description: >-
+ Rows remapped or pages retired take effect only after the card is
+ reset. Drain the node (llmgw ctl drain), reset or reboot, and bring
+ it back.
+
+ - alert: GPUPCIeRetriesClimbing
+ expr: increase(llmgw_gpu_pcie_replays[1h]) > 100
+ for: 0m
+ labels: { severity: warning }
+ annotations:
+ summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: {{ $value }} PCIe retries in an hour"
+ description: >-
+ The card's link to the host is retrying. A riser, the slot or a
+ cable is the usual cause, and transfers to the card slow down.
+
+ - alert: GPUNVLinkErrorsClimbing
+ expr: increase(llmgw_gpu_nvlink_errors{kind="crc"}[1h]) > 10
+ for: 0m
+ labels: { severity: warning }
+ annotations:
+ summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: NVLink CRC errors climbing"
+ description: >-
+ Errors on the card's NVLink links: tensor-parallel replicas on it
+ slow down, and a link that keeps failing takes a replica with it.
+
- alert: GPUHot
expr: llmgw_gpu_temperature_celsius > 87
for: 10m
diff --git a/internal/agent/agent.go b/internal/agent/agent.go
index 95f39fc..3a87ca1 100644
--- a/internal/agent/agent.go
+++ b/internal/agent/agent.go
@@ -104,6 +104,8 @@ type Agent struct {
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
// from nvidia-smi.
sensors *discovery.SensorSampler
+ // wear reads what wears a GPU out: remapped rows, PCIe retries, XIDs.
+ wear *discovery.HealthSampler
// fetchWatchers are the shards waiting on each source, so progress on one
// download reaches every shard that asked for it. A fetch is single-flight
@@ -150,6 +152,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
provKick: make(chan struct{}, 1),
load: discovery.NewLoadSampler(),
sensors: discovery.NewSensorSampler(),
+ wear: discovery.NewHealthSampler(),
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -656,14 +659,34 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
if a.sensors != nil {
se := a.sensors.Sample(context.Background())
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
+ var wear map[int]discovery.GPUWear
+ if a.wear != nil && len(se.GPUs) > 0 {
+ wear = map[int]discovery.GPUWear{}
+ for _, w := range a.wear.Sample(context.Background()) {
+ wear[w.Index] = w
+ }
+ }
for _, g := range se.GPUs {
- out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
+ gs := &llmgwv1.GPUSensor{
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
UtilizationPercent: g.UtilPercent, MemoryUtilizationPercent: g.MemUtilPercent,
MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB,
SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected,
- })
+ }
+ if w, ok := wear[g.Index]; ok {
+ gs.HasWear = true
+ gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending =
+ w.PCIeReplays, w.RetiredSBE, w.RetiredDBE, w.RetirePending
+ gs.RemappedRowsCorrectable, gs.RemappedRowsUncorrectable, gs.RemapPending, gs.RemapFailure =
+ w.RemappedCorrectable, w.RemappedUncorrectable, w.RemapPending, w.RemapFailure
+ gs.NvlinkCrcErrors, gs.NvlinkReplayErrors, gs.NvlinkRecoveryErrors = w.NVLinkCRC, w.NVLinkReplay, w.NVLinkRecovery
+ gs.XidErrors, gs.LastXid, gs.LastXidText = w.XIDs, int32(w.LastXID), w.LastXIDText
+ if !w.LastXIDAt.IsZero() {
+ gs.LastXidUnix = w.LastXIDAt.Unix()
+ }
+ }
+ out.Gpus = append(out.Gpus, gs)
}
}
return out
diff --git a/internal/agent/discovery/gpuhealth.go b/internal/agent/discovery/gpuhealth.go
new file mode 100644
index 0000000..9552df4
--- /dev/null
+++ b/internal/agent/discovery/gpuhealth.go
@@ -0,0 +1,280 @@
+package discovery
+
+import (
+ "context"
+ "encoding/xml"
+ "os/exec"
+ "regexp"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+)
+
+// GPUWear is what wears a card out or takes it down, beyond temperature:
+// read every minute, from nvidia-smi and the kernel's log. The early signs of
+// a card failing are here — a growing count of rows its memory had to remap,
+// PCIe link retries, XID errors — long before a replica on it fails.
+//
+// A count of -1 is not reported: a card without row remapping (before
+// Ampere) has no such count, which is not the same as none.
+type GPUWear struct {
+ Index int
+ // PCIeReplays are link-level retries on the card's PCIe link: a few are
+ // noise, a count that keeps climbing is a riser, slot or cable.
+ PCIeReplays int64
+ // RetiredSBE and RetiredDBE are memory pages retired for single and
+ // double bit errors (cards before Ampere); RetirePending is a page
+ // waiting for a reset to be retired.
+ RetiredSBE, RetiredDBE int64
+ RetirePending bool
+ // RemappedCorrectable and RemappedUncorrectable are memory rows remapped
+ // (Ampere and later). RemapPending waits for a reset; RemapFailure is a
+ // card out of spare rows — it needs replacing.
+ RemappedCorrectable, RemappedUncorrectable int64
+ RemapPending, RemapFailure bool
+ // NVLink errors summed over the card's links, by kind.
+ NVLinkCRC, NVLinkReplay, NVLinkRecovery int64
+ // XIDs are the driver's error events for this card since the agent
+ // started, and the last of them.
+ XIDs int64
+ LastXID int
+ LastXIDText string
+ LastXIDAt time.Time
+}
+
+// healthEvery is how often health is read: it changes over hours, and the
+// kernel log and nvidia-smi's full report are the heavier reads.
+const healthEvery = time.Minute
+
+// HealthSampler reads GPUWear, keeping the XID counts between reads.
+type HealthSampler struct {
+ run func(ctx context.Context, name string, args ...string) ([]byte, error)
+ now func() time.Time
+
+ mu sync.Mutex
+ at time.Time
+ last []GPUWear
+ since time.Time
+ xids map[int]*GPUWear
+ started bool
+}
+
+// NewHealthSampler reads the machine's own nvidia-smi and kernel log.
+func NewHealthSampler() *HealthSampler {
+ return &HealthSampler{now: time.Now, xids: map[int]*GPUWear{},
+ run: func(ctx context.Context, name string, args ...string) ([]byte, error) {
+ return exec.CommandContext(ctx, name, args...).Output() //nolint:gosec // fixed tools
+ }}
+}
+
+// Sample is the cards' health, read again once healthEvery has passed. Nil
+// on a machine without nvidia-smi.
+func (h *HealthSampler) Sample(ctx context.Context) []GPUWear {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ now := h.now()
+ if !h.at.IsZero() && now.Sub(h.at) < healthEvery {
+ return h.last
+ }
+ h.at = now
+ ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+ defer cancel()
+ report, err := h.run(ctx, "nvidia-smi", "-q", "-x")
+ if err != nil {
+ h.last = nil
+ return nil
+ }
+ cards, bus := parseHealthReport(report)
+ if out, err := h.run(ctx, "nvidia-smi", "nvlink", "-e"); err == nil {
+ for i, e := range parseNVLinkErrors(string(out)) {
+ if i < len(cards) {
+ cards[i].NVLinkCRC, cards[i].NVLinkReplay, cards[i].NVLinkRecovery = e[0], e[1], e[2]
+ }
+ }
+ }
+ h.readXIDs(ctx, now, bus)
+ for i := range cards {
+ if x := h.xids[i]; x != nil {
+ cards[i].XIDs, cards[i].LastXID, cards[i].LastXIDText, cards[i].LastXIDAt = x.XIDs, x.LastXID, x.LastXIDText, x.LastXIDAt
+ }
+ }
+ h.last = cards
+ return cards
+}
+
+// readXIDs adds the XID events logged since the last read. The first read
+// starts from now: what happened before the agent started was before, and
+// counting the whole log would raise every alert on every restart.
+func (h *HealthSampler) readXIDs(ctx context.Context, now time.Time, bus map[string]int) {
+ if !h.started {
+ h.started, h.since = true, now
+ return
+ }
+ since := h.since
+ h.since = now
+ out, err := h.run(ctx, "journalctl", "-k", "--no-pager", "-o", "short-unix",
+ "--since", "@"+strconv.FormatInt(since.Unix(), 10), "-g", "NVRM: Xid")
+ if err != nil {
+ // No journal, or nothing matched (journalctl exits 1 for that).
+ return
+ }
+ for _, e := range parseXIDs(string(out)) {
+ i, ok := bus[e.bus]
+ if !ok {
+ continue
+ }
+ x := h.xids[i]
+ if x == nil {
+ x = &GPUWear{Index: i}
+ h.xids[i] = x
+ }
+ x.XIDs++
+ x.LastXID, x.LastXIDText = e.code, e.text
+ x.LastXIDAt = now
+ if e.at > 0 {
+ x.LastXIDAt = time.Unix(int64(e.at), 0)
+ }
+ }
+}
+
+// healthLog is the part of nvidia-smi's XML report health is read from.
+type healthLog struct {
+ GPUs []struct {
+ ID string `xml:"id,attr"`
+ PCI struct {
+ Replay string `xml:"replay_counter"`
+ } `xml:"pci"`
+ Retired struct {
+ SBE string `xml:"multiple_single_bit_retirement>retired_count"`
+ DBE string `xml:"double_bit_retirement>retired_count"`
+ Pending string `xml:"pending_retirement"`
+ // Older drivers name it pending_blacklist.
+ Blacklist string `xml:"pending_blacklist"`
+ } `xml:"retired_pages"`
+ Remapped struct {
+ Corr string `xml:"remapped_row_corr"`
+ Unc string `xml:"remapped_row_unc"`
+ Pending string `xml:"remapped_row_pending"`
+ Failure string `xml:"remapped_row_failure"`
+ } `xml:"remapped_rows"`
+ } `xml:"gpu"`
+}
+
+// parseHealthReport reads each card's health, in index order, and maps each
+// card's PCI bus ("3b:00", as the kernel names it) to its index.
+func parseHealthReport(report []byte) ([]GPUWear, map[string]int) {
+ var log healthLog
+ if err := xml.Unmarshal(report, &log); err != nil {
+ return nil, nil
+ }
+ bus := map[string]int{}
+ out := make([]GPUWear, 0, len(log.GPUs))
+ for i, g := range log.GPUs {
+ bus[busOf(g.ID)] = i
+ out = append(out, GPUWear{
+ Index: i,
+ PCIeReplays: count(g.PCI.Replay),
+ RetiredSBE: count(g.Retired.SBE),
+ RetiredDBE: count(g.Retired.DBE),
+ RetirePending: yes(g.Retired.Pending) || yes(g.Retired.Blacklist),
+ RemappedCorrectable: count(g.Remapped.Corr),
+ RemappedUncorrectable: count(g.Remapped.Unc),
+ RemapPending: yes(g.Remapped.Pending),
+ RemapFailure: yes(g.Remapped.Failure),
+ })
+ }
+ return out, bus
+}
+
+// busOf is a PCI address as the kernel's XID lines carry it: nvidia-smi
+// says "00000000:3B:00.0", the kernel "0000:3b:00".
+func busOf(id string) string {
+ id = strings.ToLower(strings.TrimSpace(id))
+ if i := strings.LastIndex(id, "."); i >= 0 {
+ id = id[:i]
+ }
+ parts := strings.Split(id, ":")
+ if len(parts) >= 2 {
+ return parts[len(parts)-2] + ":" + parts[len(parts)-1]
+ }
+ return id
+}
+
+func count(s string) int64 {
+ n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+ if err != nil {
+ return -1
+ }
+ return n
+}
+
+func yes(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "yes") }
+
+// nvlinkError matches a line of `nvidia-smi nvlink -e`, whose wording varies
+// by driver: "Link 0: Replay Errors: 0", "Link 0: CRC Errors: 0",
+// "Link 3: Data CRC Errors: 2".
+var (
+ nvlinkGPU = regexp.MustCompile(`^GPU (\d+):`)
+ nvlinkError = regexp.MustCompile(`Link \d+:\s*(.*?)\s*Errors?\s*:\s*(\d+)`)
+)
+
+// parseNVLinkErrors sums each card's link errors by kind: CRC, replay,
+// recovery.
+func parseNVLinkErrors(out string) map[int][3]int64 {
+ res := map[int][3]int64{}
+ gpu := -1
+ for _, line := range strings.Split(out, "\n") {
+ if m := nvlinkGPU.FindStringSubmatch(strings.TrimSpace(line)); m != nil {
+ gpu, _ = strconv.Atoi(m[1])
+ continue
+ }
+ m := nvlinkError.FindStringSubmatch(line)
+ if m == nil || gpu < 0 {
+ continue
+ }
+ n, _ := strconv.ParseInt(m[2], 10, 64)
+ e := res[gpu]
+ kind := strings.ToLower(m[1])
+ switch {
+ case strings.Contains(kind, "crc"):
+ e[0] += n
+ case strings.Contains(kind, "replay"):
+ e[1] += n
+ case strings.Contains(kind, "recovery"):
+ e[2] += n
+ }
+ res[gpu] = e
+ }
+ return res
+}
+
+type xidEvent struct {
+ bus string
+ code int
+ text string
+ at float64
+}
+
+// xidLine matches the driver's report of an XID:
+// "NVRM: Xid (PCI:0000:3b:00): 79, pid=..., GPU has fallen off the bus."
+var xidLine = regexp.MustCompile(`NVRM: Xid \(PCI:([0-9a-fA-F:.]+)\): (\d+),\s*(.*)$`)
+
+// parseXIDs reads journalctl's short-unix lines: a timestamp, then the
+// message.
+func parseXIDs(out string) []xidEvent {
+ var evs []xidEvent
+ for _, line := range strings.Split(out, "\n") {
+ m := xidLine.FindStringSubmatch(line)
+ if m == nil {
+ continue
+ }
+ code, _ := strconv.Atoi(m[2])
+ e := xidEvent{bus: busOf(m[1]), code: code, text: strings.TrimSpace(m[3])}
+ if f := strings.Fields(line); len(f) > 0 {
+ e.at, _ = strconv.ParseFloat(f[0], 64)
+ }
+ evs = append(evs, e)
+ }
+ return evs
+}
diff --git a/internal/agent/discovery/gpuhealth_test.go b/internal/agent/discovery/gpuhealth_test.go
new file mode 100644
index 0000000..efe5c9e
--- /dev/null
+++ b/internal/agent/discovery/gpuhealth_test.go
@@ -0,0 +1,101 @@
+package discovery
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+)
+
+const healthXML = `
+chat-1
gpu07", "30m0s ago",
diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html
index 0d4a34a..37b2f28 100644
--- a/internal/web/templates/dashboard.html
+++ b/internal/web/templates/dashboard.html
@@ -55,7 +55,7 @@
{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.
+{{.Legend}} A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.
{{end}} {{with .Efficiency}}{{join .Unhealthy}}
{{end}} {{with .Load}}{{if .GPUs}}| GPU | Busy | Memory used | Temperature | Memory temp. | Power | Clock | Held back by |
|---|
| GPU | Busy | Memory used | Temperature | Memory temp. | Power | Clock | Held back by, and wear |
|---|---|---|---|---|---|---|---|
| {{.Index}} | {{/* From the last sample; a dash where the driver or an older agent did @@ -46,7 +46,7 @@{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}} | {{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}{{printf "%.0f" .PowerW}} W of {{printf "%.0f" .PowerLimitW}}{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}} | {{if .SMClockMHz}}{{printf "%.0f" .SMClockMHz}} MHz{{else}}—{{end}} | -{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if gt .ECCUncorrected 0.0}} {{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors{{end}} | +{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if gt .ECCUncorrected 0.0}} {{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors{{end}}{{with .Wear}}{{with .Problems}} {{join .}}{{end}}{{if gt .PCIeReplays 0}} {{.PCIeReplays}} PCIe retries{{end}}{{end}} |