Read what wears a GPU out: XIDs, remapped rows, PCIe and NVLink errors

Every minute the agent reads nvidia-smi's full report, its NVLink error
counters and the kernel's XID lines. They are published per card,
shown on the node page and the GPU map, charted in Grafana, and five
alerts watch them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 16:02:07 +02:00
1 parent ff554c7062
commit 82a63cfc6e
16 files changed
+885 -11

No files matched your search

+12 -1
View File
@@ -44,6 +44,8 @@ type gpuCell struct {
Title string
// Label is the figure written in the cell.
Label string
// Faulty is a card whose wear needs seeing to.
Faulty bool
}
// idleHolding is a replica holding GPUs that are doing nothing.
@@ -108,12 +110,19 @@ func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, pre
if len(s.Throttle) > 0 {
title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
}
title = append(title, s.Problems...)
}
c.Title = strings.Join(title, " · ")
style, label := cellColour(m.By, s, sampled)
if c.Model == "" {
style += ";border-style:dashed"
}
// A card wearing out is outlined whatever the map is coloured
// by: it is the one thing on the map that needs doing.
if len(s.Problems) > 0 {
style += ";border:2px solid #c0392b"
c.Faulty = true
}
c.Style, c.Label = template.CSS(style), label
row.Cells = append(row.Cells, c)
}
@@ -216,6 +225,7 @@ func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu
type sensorView = struct {
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
Throttle []string
Problems []string
}
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
@@ -225,7 +235,8 @@ func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
for _, s := range n.Load.GPUs {
if s.Index == gpu {
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle,
Problems: s.Wear.Problems()}, true
}
}
return sensorView{}, false
+8 -1
View File
@@ -20,7 +20,12 @@ func (mapFleet) Nodes() []adminapi.NodeView {
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
}
return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}}
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
}
func worn(s inventory.GPUSensor) inventory.GPUSensor {
s.Wear = &inventory.GPUWear{RemapPending: true, XIDs: 1, LastXID: 79, LastXIDText: "GPU has fallen off the bus."}
return s
}
func (mapFleet) Replicas(string) []adminapi.ReplicaView {
@@ -38,6 +43,8 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
for _, want := range []string{
"gpu07 GPU 2 · code (code-1) · 93% busy",
"gpu07 GPU 3 · free",
"memory rows remapped, waiting for a reset · XID 79 (GPU has fallen off the bus.)",
"border:2px solid #c0392b",
"border-style:dashed",
"Cards held and doing nothing",
"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
+1 -1
View File
@@ -55,7 +55,7 @@
<div class="gpu-map">
{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
{{end}}</div>
<p class="note">{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.</p>
<p class="note">{{.Legend}} A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.</p>
{{end}}
{{with .Efficiency}}
<h3>What each model gets from its cards <span class="opt">last 15 minutes</span></h3>
+2 -2
View File
@@ -35,7 +35,7 @@
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
{{with .Load}}{{if .GPUs}}
<h2>GPUs</h2>
<table class="rows"><tr><th class="num">GPU</th><th>Busy</th><th>Memory used</th><th class="num">Temperature</th><th class="num">Memory temp.</th><th>Power</th><th class="num">Clock</th><th>Held back by</th></tr>
<table class="rows"><tr><th class="num">GPU</th><th>Busy</th><th>Memory used</th><th class="num">Temperature</th><th class="num">Memory temp.</th><th>Power</th><th class="num">Clock</th><th>Held back by, and wear</th></tr>
{{range .GPUs}}<tr>
<td class="num">{{.Index}}</td>
{{/* From the last sample; a dash where the driver or an older agent did
@@ -46,7 +46,7 @@
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
<td class="num">{{if .SMClockMHz}}{{printf "%.0f" .SMClockMHz}} MHz{{else}}—{{end}}</td>
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors</span>{{end}}</td>
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors</span>{{end}}{{with .Wear}}{{with .Problems}}<br><span class="bad">{{join .}}</span>{{end}}{{if gt .PCIeReplays 0}} <span class="note">{{.PCIeReplays}} PCIe retries</span>{{end}}{{end}}</td>
</tr>{{end}}</table>
{{end}}{{end}}
{{end}}