diff --git a/CHANGELOG.md b/CHANGELOG.md index 905ccca..9bee72f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,17 @@ and a figure the driver does not give is left out rather than shown as zero. - **What each model gets from its cards**, on the cluster page and in Grafana: output tokens a second per GPU and per watt-hour, with the cards' draw and average utilisation, over the last quarter hour. +- **What wears a card out**, read every minute: XID errors from the kernel's + log (counted from when the agent started), memory rows remapped or pages + retired and whether either waits for a reset, a card out of spare rows, + PCIe link retries and NVLink errors. Published as `llmgw_gpu_xid_errors`, + `llmgw_gpu_last_xid`, `llmgw_gpu_remapped_rows`, `llmgw_gpu_remap_pending`, + `llmgw_gpu_remap_failure`, `llmgw_gpu_retired_pages`, + `llmgw_gpu_retirement_pending`, `llmgw_gpu_pcie_replays`, + `llmgw_gpu_nvlink_errors`; said in words on the node page; a red outline + on the GPU map; a Grafana panel; and alerts — `GPUXidError`, + `GPUOutOfSpareRows`, `GPUNeedsReset`, `GPUPCIeRetriesClimbing`, + `GPUNVLinkErrorsClimbing`. - **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds each card and whose quota it counts against, for joining the GPU series to models and tenants. diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index d4f4efb..5a21fd7 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1561,8 +1561,27 @@ type GPUSensor struct { SmClockMhz float64 `protobuf:"fixed64,11,opt,name=sm_clock_mhz,json=smClockMhz,proto3" json:"sm_clock_mhz,omitempty"` // ecc_uncorrected counts uncorrected ECC errors since the driver loaded. EccUncorrected float64 `protobuf:"fixed64,12,opt,name=ecc_uncorrected,json=eccUncorrected,proto3" json:"ecc_uncorrected,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // What wears the card out, read every minute; has_wear says these are + // present. Counts of -1 are not reported for this card. + HasWear bool `protobuf:"varint,13,opt,name=has_wear,json=hasWear,proto3" json:"has_wear,omitempty"` + PcieReplays int64 `protobuf:"varint,14,opt,name=pcie_replays,json=pcieReplays,proto3" json:"pcie_replays,omitempty"` + RetiredPagesSbe int64 `protobuf:"varint,15,opt,name=retired_pages_sbe,json=retiredPagesSbe,proto3" json:"retired_pages_sbe,omitempty"` + RetiredPagesDbe int64 `protobuf:"varint,16,opt,name=retired_pages_dbe,json=retiredPagesDbe,proto3" json:"retired_pages_dbe,omitempty"` + RetirementPending bool `protobuf:"varint,17,opt,name=retirement_pending,json=retirementPending,proto3" json:"retirement_pending,omitempty"` + RemappedRowsCorrectable int64 `protobuf:"varint,18,opt,name=remapped_rows_correctable,json=remappedRowsCorrectable,proto3" json:"remapped_rows_correctable,omitempty"` + RemappedRowsUncorrectable int64 `protobuf:"varint,19,opt,name=remapped_rows_uncorrectable,json=remappedRowsUncorrectable,proto3" json:"remapped_rows_uncorrectable,omitempty"` + RemapPending bool `protobuf:"varint,20,opt,name=remap_pending,json=remapPending,proto3" json:"remap_pending,omitempty"` + RemapFailure bool `protobuf:"varint,21,opt,name=remap_failure,json=remapFailure,proto3" json:"remap_failure,omitempty"` + NvlinkCrcErrors int64 `protobuf:"varint,22,opt,name=nvlink_crc_errors,json=nvlinkCrcErrors,proto3" json:"nvlink_crc_errors,omitempty"` + NvlinkReplayErrors int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"` + NvlinkRecoveryErrors int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"` + // xid_errors are the driver's error events since the agent started. + XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"` + LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"` + LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"` + LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPUSensor) Reset() { @@ -1679,6 +1698,118 @@ func (x *GPUSensor) GetEccUncorrected() float64 { return 0 } +func (x *GPUSensor) GetHasWear() bool { + if x != nil { + return x.HasWear + } + return false +} + +func (x *GPUSensor) GetPcieReplays() int64 { + if x != nil { + return x.PcieReplays + } + return 0 +} + +func (x *GPUSensor) GetRetiredPagesSbe() int64 { + if x != nil { + return x.RetiredPagesSbe + } + return 0 +} + +func (x *GPUSensor) GetRetiredPagesDbe() int64 { + if x != nil { + return x.RetiredPagesDbe + } + return 0 +} + +func (x *GPUSensor) GetRetirementPending() bool { + if x != nil { + return x.RetirementPending + } + return false +} + +func (x *GPUSensor) GetRemappedRowsCorrectable() int64 { + if x != nil { + return x.RemappedRowsCorrectable + } + return 0 +} + +func (x *GPUSensor) GetRemappedRowsUncorrectable() int64 { + if x != nil { + return x.RemappedRowsUncorrectable + } + return 0 +} + +func (x *GPUSensor) GetRemapPending() bool { + if x != nil { + return x.RemapPending + } + return false +} + +func (x *GPUSensor) GetRemapFailure() bool { + if x != nil { + return x.RemapFailure + } + return false +} + +func (x *GPUSensor) GetNvlinkCrcErrors() int64 { + if x != nil { + return x.NvlinkCrcErrors + } + return 0 +} + +func (x *GPUSensor) GetNvlinkReplayErrors() int64 { + if x != nil { + return x.NvlinkReplayErrors + } + return 0 +} + +func (x *GPUSensor) GetNvlinkRecoveryErrors() int64 { + if x != nil { + return x.NvlinkRecoveryErrors + } + return 0 +} + +func (x *GPUSensor) GetXidErrors() int64 { + if x != nil { + return x.XidErrors + } + return 0 +} + +func (x *GPUSensor) GetLastXid() int32 { + if x != nil { + return x.LastXid + } + return 0 +} + +func (x *GPUSensor) GetLastXidText() string { + if x != nil { + return x.LastXidText + } + return "" +} + +func (x *GPUSensor) GetLastXidUnix() int64 { + if x != nil { + return x.LastXidUnix + } + return 0 +} + // ShardLoad is one engine process group's use. type ShardLoad struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4051,7 +4182,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + " \x01(\x01R\tcpuPowerW\x12 \n" + "\finlet_temp_c\x18\v \x01(\x01R\n" + "inletTempC\x12'\n" + - "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xbd\x03\n" + + "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" + "\tGPUSensor\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" + "\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" + @@ -4066,7 +4197,24 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + " \x01(\x01R\rmemoryTotalMb\x12 \n" + "\fsm_clock_mhz\x18\v \x01(\x01R\n" + "smClockMhz\x12'\n" + - "\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\"m\n" + + "\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\x12\x19\n" + + "\bhas_wear\x18\r \x01(\bR\ahasWear\x12!\n" + + "\fpcie_replays\x18\x0e \x01(\x03R\vpcieReplays\x12*\n" + + "\x11retired_pages_sbe\x18\x0f \x01(\x03R\x0fretiredPagesSbe\x12*\n" + + "\x11retired_pages_dbe\x18\x10 \x01(\x03R\x0fretiredPagesDbe\x12-\n" + + "\x12retirement_pending\x18\x11 \x01(\bR\x11retirementPending\x12:\n" + + "\x19remapped_rows_correctable\x18\x12 \x01(\x03R\x17remappedRowsCorrectable\x12>\n" + + "\x1bremapped_rows_uncorrectable\x18\x13 \x01(\x03R\x19remappedRowsUncorrectable\x12#\n" + + "\rremap_pending\x18\x14 \x01(\bR\fremapPending\x12#\n" + + "\rremap_failure\x18\x15 \x01(\bR\fremapFailure\x12*\n" + + "\x11nvlink_crc_errors\x18\x16 \x01(\x03R\x0fnvlinkCrcErrors\x120\n" + + "\x14nvlink_replay_errors\x18\x17 \x01(\x03R\x12nvlinkReplayErrors\x124\n" + + "\x16nvlink_recovery_errors\x18\x18 \x01(\x03R\x14nvlinkRecoveryErrors\x12\x1d\n" + + "\n" + + "xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" + + "\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" + + "\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" + + "\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 6a3a03b..3ee090c 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -314,6 +314,25 @@ message GPUSensor { double sm_clock_mhz = 11; // ecc_uncorrected counts uncorrected ECC errors since the driver loaded. double ecc_uncorrected = 12; + // What wears the card out, read every minute; has_wear says these are + // present. Counts of -1 are not reported for this card. + bool has_wear = 13; + int64 pcie_replays = 14; + int64 retired_pages_sbe = 15; + int64 retired_pages_dbe = 16; + bool retirement_pending = 17; + int64 remapped_rows_correctable = 18; + int64 remapped_rows_uncorrectable = 19; + bool remap_pending = 20; + bool remap_failure = 21; + int64 nvlink_crc_errors = 22; + int64 nvlink_replay_errors = 23; + int64 nvlink_recovery_errors = 24; + // xid_errors are the driver's error events since the agent started. + int64 xid_errors = 25; + int32 last_xid = 26; + string last_xid_text = 27; + int64 last_xid_unix = 28; } // ShardLoad is one engine process group's use. diff --git a/deploy/monitoring/grafana/llmgw-overview.json b/deploy/monitoring/grafana/llmgw-overview.json index ea533cc..cfe385b 100644 --- a/deploy/monitoring/grafana/llmgw-overview.json +++ b/deploy/monitoring/grafana/llmgw-overview.json @@ -855,6 +855,59 @@ "mode": "multi" } } + }, + { + "type": "timeseries", + "title": "GPU wear", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "id": 24, + "description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.", + "gridPos": { + "x": 0, + "y": 66, + "w": 24, + "h": 7 + }, + "targets": [ + { + "expr": "increase(llmgw_gpu_xid_errors[15m]) > 0", + "legendFormat": "XID {{node}} gpu {{gpu}}", + "refId": "A" + }, + { + "expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0", + "legendFormat": "rows remapped {{node}} gpu {{gpu}}", + "refId": "B" + }, + { + "expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0", + "legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}", + "refId": "C" + }, + { + "expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0", + "legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } } ] } \ No newline at end of file diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index 16bf5af..e877019 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -224,6 +224,60 @@ groups: card may be computing on corrupted memory. Drain the node (llmgw ctl drain) and have the card looked at. + - alert: GPUXidError + expr: increase(llmgw_gpu_xid_errors[15m]) > 0 + for: 0m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} logged an XID error" + description: >- + The driver reported an error on the card (its code is + llmgw_gpu_last_xid, and the node page says it in words). 79 is a + card fallen off the bus, 48 and 63/64 memory errors, 74 NVLink, + 94/95 contained and uncontained ECC errors: those need the card + looked at. Others are often an engine's own fault, and pass. + + - alert: GPUOutOfSpareRows + expr: llmgw_gpu_remap_failure == 1 + for: 1m + labels: { severity: critical } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has no spare memory rows left" + description: >- + The card cannot remap another failing row. Drain the node and + have the card replaced. + + - alert: GPUNeedsReset + expr: llmgw_gpu_remap_pending == 1 or llmgw_gpu_retirement_pending == 1 + for: 5m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has memory repairs waiting for a reset" + description: >- + Rows remapped or pages retired take effect only after the card is + reset. Drain the node (llmgw ctl drain), reset or reboot, and bring + it back. + + - alert: GPUPCIeRetriesClimbing + expr: increase(llmgw_gpu_pcie_replays[1h]) > 100 + for: 0m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: {{ $value }} PCIe retries in an hour" + description: >- + The card's link to the host is retrying. A riser, the slot or a + cable is the usual cause, and transfers to the card slow down. + + - alert: GPUNVLinkErrorsClimbing + expr: increase(llmgw_gpu_nvlink_errors{kind="crc"}[1h]) > 10 + for: 0m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: NVLink CRC errors climbing" + description: >- + Errors on the card's NVLink links: tensor-parallel replicas on it + slow down, and a link that keeps failing takes a replica with it. + - alert: GPUHot expr: llmgw_gpu_temperature_celsius > 87 for: 10m diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 95f39fc..3a87ca1 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -104,6 +104,8 @@ type Agent struct { // sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs // from nvidia-smi. sensors *discovery.SensorSampler + // wear reads what wears a GPU out: remapped rows, PCIe retries, XIDs. + wear *discovery.HealthSampler // fetchWatchers are the shards waiting on each source, so progress on one // download reaches every shard that asked for it. A fetch is single-flight @@ -150,6 +152,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er provKick: make(chan struct{}, 1), load: discovery.NewLoadSampler(), sensors: discovery.NewSensorSampler(), + wear: discovery.NewHealthSampler(), } a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus} return a, nil @@ -656,14 +659,34 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad { if a.sensors != nil { se := a.sensors.Sample(context.Background()) out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC + var wear map[int]discovery.GPUWear + if a.wear != nil && len(se.GPUs) > 0 { + wear = map[int]discovery.GPUWear{} + for _, w := range a.wear.Sample(context.Background()) { + wear[w.Index] = w + } + } for _, g := range se.GPUs { - out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{ + gs := &llmgwv1.GPUSensor{ Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC, PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle, UtilizationPercent: g.UtilPercent, MemoryUtilizationPercent: g.MemUtilPercent, MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB, SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected, - }) + } + if w, ok := wear[g.Index]; ok { + gs.HasWear = true + gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending = + w.PCIeReplays, w.RetiredSBE, w.RetiredDBE, w.RetirePending + gs.RemappedRowsCorrectable, gs.RemappedRowsUncorrectable, gs.RemapPending, gs.RemapFailure = + w.RemappedCorrectable, w.RemappedUncorrectable, w.RemapPending, w.RemapFailure + gs.NvlinkCrcErrors, gs.NvlinkReplayErrors, gs.NvlinkRecoveryErrors = w.NVLinkCRC, w.NVLinkReplay, w.NVLinkRecovery + gs.XidErrors, gs.LastXid, gs.LastXidText = w.XIDs, int32(w.LastXID), w.LastXIDText + if !w.LastXIDAt.IsZero() { + gs.LastXidUnix = w.LastXIDAt.Unix() + } + } + out.Gpus = append(out.Gpus, gs) } } return out diff --git a/internal/agent/discovery/gpuhealth.go b/internal/agent/discovery/gpuhealth.go new file mode 100644 index 0000000..9552df4 --- /dev/null +++ b/internal/agent/discovery/gpuhealth.go @@ -0,0 +1,280 @@ +package discovery + +import ( + "context" + "encoding/xml" + "os/exec" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +// GPUWear is what wears a card out or takes it down, beyond temperature: +// read every minute, from nvidia-smi and the kernel's log. The early signs of +// a card failing are here — a growing count of rows its memory had to remap, +// PCIe link retries, XID errors — long before a replica on it fails. +// +// A count of -1 is not reported: a card without row remapping (before +// Ampere) has no such count, which is not the same as none. +type GPUWear struct { + Index int + // PCIeReplays are link-level retries on the card's PCIe link: a few are + // noise, a count that keeps climbing is a riser, slot or cable. + PCIeReplays int64 + // RetiredSBE and RetiredDBE are memory pages retired for single and + // double bit errors (cards before Ampere); RetirePending is a page + // waiting for a reset to be retired. + RetiredSBE, RetiredDBE int64 + RetirePending bool + // RemappedCorrectable and RemappedUncorrectable are memory rows remapped + // (Ampere and later). RemapPending waits for a reset; RemapFailure is a + // card out of spare rows — it needs replacing. + RemappedCorrectable, RemappedUncorrectable int64 + RemapPending, RemapFailure bool + // NVLink errors summed over the card's links, by kind. + NVLinkCRC, NVLinkReplay, NVLinkRecovery int64 + // XIDs are the driver's error events for this card since the agent + // started, and the last of them. + XIDs int64 + LastXID int + LastXIDText string + LastXIDAt time.Time +} + +// healthEvery is how often health is read: it changes over hours, and the +// kernel log and nvidia-smi's full report are the heavier reads. +const healthEvery = time.Minute + +// HealthSampler reads GPUWear, keeping the XID counts between reads. +type HealthSampler struct { + run func(ctx context.Context, name string, args ...string) ([]byte, error) + now func() time.Time + + mu sync.Mutex + at time.Time + last []GPUWear + since time.Time + xids map[int]*GPUWear + started bool +} + +// NewHealthSampler reads the machine's own nvidia-smi and kernel log. +func NewHealthSampler() *HealthSampler { + return &HealthSampler{now: time.Now, xids: map[int]*GPUWear{}, + run: func(ctx context.Context, name string, args ...string) ([]byte, error) { + return exec.CommandContext(ctx, name, args...).Output() //nolint:gosec // fixed tools + }} +} + +// Sample is the cards' health, read again once healthEvery has passed. Nil +// on a machine without nvidia-smi. +func (h *HealthSampler) Sample(ctx context.Context) []GPUWear { + h.mu.Lock() + defer h.mu.Unlock() + now := h.now() + if !h.at.IsZero() && now.Sub(h.at) < healthEvery { + return h.last + } + h.at = now + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + report, err := h.run(ctx, "nvidia-smi", "-q", "-x") + if err != nil { + h.last = nil + return nil + } + cards, bus := parseHealthReport(report) + if out, err := h.run(ctx, "nvidia-smi", "nvlink", "-e"); err == nil { + for i, e := range parseNVLinkErrors(string(out)) { + if i < len(cards) { + cards[i].NVLinkCRC, cards[i].NVLinkReplay, cards[i].NVLinkRecovery = e[0], e[1], e[2] + } + } + } + h.readXIDs(ctx, now, bus) + for i := range cards { + if x := h.xids[i]; x != nil { + cards[i].XIDs, cards[i].LastXID, cards[i].LastXIDText, cards[i].LastXIDAt = x.XIDs, x.LastXID, x.LastXIDText, x.LastXIDAt + } + } + h.last = cards + return cards +} + +// readXIDs adds the XID events logged since the last read. The first read +// starts from now: what happened before the agent started was before, and +// counting the whole log would raise every alert on every restart. +func (h *HealthSampler) readXIDs(ctx context.Context, now time.Time, bus map[string]int) { + if !h.started { + h.started, h.since = true, now + return + } + since := h.since + h.since = now + out, err := h.run(ctx, "journalctl", "-k", "--no-pager", "-o", "short-unix", + "--since", "@"+strconv.FormatInt(since.Unix(), 10), "-g", "NVRM: Xid") + if err != nil { + // No journal, or nothing matched (journalctl exits 1 for that). + return + } + for _, e := range parseXIDs(string(out)) { + i, ok := bus[e.bus] + if !ok { + continue + } + x := h.xids[i] + if x == nil { + x = &GPUWear{Index: i} + h.xids[i] = x + } + x.XIDs++ + x.LastXID, x.LastXIDText = e.code, e.text + x.LastXIDAt = now + if e.at > 0 { + x.LastXIDAt = time.Unix(int64(e.at), 0) + } + } +} + +// healthLog is the part of nvidia-smi's XML report health is read from. +type healthLog struct { + GPUs []struct { + ID string `xml:"id,attr"` + PCI struct { + Replay string `xml:"replay_counter"` + } `xml:"pci"` + Retired struct { + SBE string `xml:"multiple_single_bit_retirement>retired_count"` + DBE string `xml:"double_bit_retirement>retired_count"` + Pending string `xml:"pending_retirement"` + // Older drivers name it pending_blacklist. + Blacklist string `xml:"pending_blacklist"` + } `xml:"retired_pages"` + Remapped struct { + Corr string `xml:"remapped_row_corr"` + Unc string `xml:"remapped_row_unc"` + Pending string `xml:"remapped_row_pending"` + Failure string `xml:"remapped_row_failure"` + } `xml:"remapped_rows"` + } `xml:"gpu"` +} + +// parseHealthReport reads each card's health, in index order, and maps each +// card's PCI bus ("3b:00", as the kernel names it) to its index. +func parseHealthReport(report []byte) ([]GPUWear, map[string]int) { + var log healthLog + if err := xml.Unmarshal(report, &log); err != nil { + return nil, nil + } + bus := map[string]int{} + out := make([]GPUWear, 0, len(log.GPUs)) + for i, g := range log.GPUs { + bus[busOf(g.ID)] = i + out = append(out, GPUWear{ + Index: i, + PCIeReplays: count(g.PCI.Replay), + RetiredSBE: count(g.Retired.SBE), + RetiredDBE: count(g.Retired.DBE), + RetirePending: yes(g.Retired.Pending) || yes(g.Retired.Blacklist), + RemappedCorrectable: count(g.Remapped.Corr), + RemappedUncorrectable: count(g.Remapped.Unc), + RemapPending: yes(g.Remapped.Pending), + RemapFailure: yes(g.Remapped.Failure), + }) + } + return out, bus +} + +// busOf is a PCI address as the kernel's XID lines carry it: nvidia-smi +// says "00000000:3B:00.0", the kernel "0000:3b:00". +func busOf(id string) string { + id = strings.ToLower(strings.TrimSpace(id)) + if i := strings.LastIndex(id, "."); i >= 0 { + id = id[:i] + } + parts := strings.Split(id, ":") + if len(parts) >= 2 { + return parts[len(parts)-2] + ":" + parts[len(parts)-1] + } + return id +} + +func count(s string) int64 { + n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return -1 + } + return n +} + +func yes(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "yes") } + +// nvlinkError matches a line of `nvidia-smi nvlink -e`, whose wording varies +// by driver: "Link 0: Replay Errors: 0", "Link 0: CRC Errors: 0", +// "Link 3: Data CRC Errors: 2". +var ( + nvlinkGPU = regexp.MustCompile(`^GPU (\d+):`) + nvlinkError = regexp.MustCompile(`Link \d+:\s*(.*?)\s*Errors?\s*:\s*(\d+)`) +) + +// parseNVLinkErrors sums each card's link errors by kind: CRC, replay, +// recovery. +func parseNVLinkErrors(out string) map[int][3]int64 { + res := map[int][3]int64{} + gpu := -1 + for _, line := range strings.Split(out, "\n") { + if m := nvlinkGPU.FindStringSubmatch(strings.TrimSpace(line)); m != nil { + gpu, _ = strconv.Atoi(m[1]) + continue + } + m := nvlinkError.FindStringSubmatch(line) + if m == nil || gpu < 0 { + continue + } + n, _ := strconv.ParseInt(m[2], 10, 64) + e := res[gpu] + kind := strings.ToLower(m[1]) + switch { + case strings.Contains(kind, "crc"): + e[0] += n + case strings.Contains(kind, "replay"): + e[1] += n + case strings.Contains(kind, "recovery"): + e[2] += n + } + res[gpu] = e + } + return res +} + +type xidEvent struct { + bus string + code int + text string + at float64 +} + +// xidLine matches the driver's report of an XID: +// "NVRM: Xid (PCI:0000:3b:00): 79, pid=..., GPU has fallen off the bus." +var xidLine = regexp.MustCompile(`NVRM: Xid \(PCI:([0-9a-fA-F:.]+)\): (\d+),\s*(.*)$`) + +// parseXIDs reads journalctl's short-unix lines: a timestamp, then the +// message. +func parseXIDs(out string) []xidEvent { + var evs []xidEvent + for _, line := range strings.Split(out, "\n") { + m := xidLine.FindStringSubmatch(line) + if m == nil { + continue + } + code, _ := strconv.Atoi(m[2]) + e := xidEvent{bus: busOf(m[1]), code: code, text: strings.TrimSpace(m[3])} + if f := strings.Fields(line); len(f) > 0 { + e.at, _ = strconv.ParseFloat(f[0], 64) + } + evs = append(evs, e) + } + return evs +} diff --git a/internal/agent/discovery/gpuhealth_test.go b/internal/agent/discovery/gpuhealth_test.go new file mode 100644 index 0000000..efe5c9e --- /dev/null +++ b/internal/agent/discovery/gpuhealth_test.go @@ -0,0 +1,101 @@ +package discovery + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +const healthXML = ` + + + 0 + + N/A + N/A + N/A + + + 2 + 1 + Yes + No + + + + 4021 + + 3 + 1 + Yes + + N/A + +` + +const nvlinkOut = `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-aaa) + Link 0: Replay Errors: 1 + Link 0: Recovery Errors: 0 + Link 0: CRC Errors: 5 + Link 1: Data CRC Errors: 2 +GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-bbb) + Link 0: Replay Errors: 0 +` + +func TestAGPUsWearAndFailuresAreReadAndCountedFromWhenTheAgentStarted(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + journal := "" + h := &HealthSampler{now: func() time.Time { return now }, xids: map[int]*GPUWear{}, + run: func(_ context.Context, name string, args ...string) ([]byte, error) { + switch { + case name == "nvidia-smi" && args[0] == "-q": + return []byte(healthXML), nil + case name == "nvidia-smi" && args[0] == "nvlink": + return []byte(nvlinkOut), nil + case name == "journalctl": + if journal == "" { + return nil, errors.New("exit status 1") + } + return []byte(journal), nil + } + return nil, errors.New("unexpected " + name) + }} + got := h.Sample(t.Context()) + if len(got) != 2 { + t.Fatalf("%+v", got) + } + a, b := got[0], got[1] + if a.RemappedCorrectable != 2 || a.RemappedUncorrectable != 1 || !a.RemapPending || a.RemapFailure || + a.RetiredSBE != -1 || a.PCIeReplays != 0 || a.NVLinkCRC != 7 || a.NVLinkReplay != 1 { + t.Errorf("card 0 %+v", a) + } + if b.PCIeReplays != 4021 || b.RetiredSBE != 3 || b.RetiredDBE != 1 || !b.RetirePending || b.RemappedCorrectable != -1 { + t.Errorf("card 1 %+v", b) + } + + // An XID on card 1, logged after the agent started; read on the next + // sample a minute on. + journal = "1800000030.123456 gpu07 kernel: NVRM: Xid (PCI:0000:5e:00): 79, pid=1234, name=llama-server, GPU has fallen off the bus.\n" + now = now.Add(61 * time.Second) + got = h.Sample(t.Context()) + if got[1].XIDs != 1 || got[1].LastXID != 79 || !strings.Contains(got[1].LastXIDText, "fallen off the bus") || + got[1].LastXIDAt.Unix() != 1_800_000_030 || got[0].XIDs != 0 { + t.Errorf("after an XID: %+v / %+v", got[0], got[1]) + } + // Within the minute, the last reading. + journal = "" + now = now.Add(10 * time.Second) + if again := h.Sample(t.Context()); again[1].XIDs != 1 { + t.Errorf("%+v", again[1]) + } +} + +func TestBusIDsAreReadAsTheKernelWritesThem(t *testing.T) { + for in, want := range map[string]string{"00000000:3B:00.0": "3b:00", "0000:3b:00": "3b:00", "0000:5e:00.0": "5e:00"} { + if got := busOf(in); got != want { + t.Errorf("%s: %s, want %s", in, got, want) + } + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 0ca6387..1a825fd 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -588,6 +588,7 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(), MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(), SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(), + Wear: wearFrom(g), }) } return out @@ -964,3 +965,22 @@ func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string) return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID) } } + +// wearFrom is a card's wear, when the agent read it. +func wearFrom(g *llmgwv1.GPUSensor) *inventory.GPUWear { + if !g.GetHasWear() { + return nil + } + w := &inventory.GPUWear{ + PCIeReplays: g.GetPcieReplays(), RetiredSBE: g.GetRetiredPagesSbe(), RetiredDBE: g.GetRetiredPagesDbe(), + RetirePending: g.GetRetirementPending(), RemappedCorrectable: g.GetRemappedRowsCorrectable(), + RemappedUncorrectable: g.GetRemappedRowsUncorrectable(), RemapPending: g.GetRemapPending(), + RemapFailure: g.GetRemapFailure(), NVLinkCRC: g.GetNvlinkCrcErrors(), NVLinkReplay: g.GetNvlinkReplayErrors(), + NVLinkRecovery: g.GetNvlinkRecoveryErrors(), XIDs: g.GetXidErrors(), LastXID: int(g.GetLastXid()), + LastXIDText: g.GetLastXidText(), + } + if u := g.GetLastXidUnix(); u > 0 { + w.LastXIDAt = time.Unix(u, 0) + } + return w +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 7cbf5e5..f8a3c97 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -74,6 +74,15 @@ type Collector struct { gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc gpuAssigned *prometheus.Desc + gpuPCIeReplay *prometheus.Desc + gpuRetired *prometheus.Desc + gpuRetirePend *prometheus.Desc + gpuRemapped *prometheus.Desc + gpuRemapPend *prometheus.Desc + gpuRemapFail *prometheus.Desc + gpuNVLinkErr *prometheus.Desc + gpuXID *prometheus.Desc + gpuLastXID *prometheus.Desc gpuMemUtil *prometheus.Desc gpuSMClock *prometheus.Desc gpuTemp *prometheus.Desc @@ -152,6 +161,24 @@ func New(src Source) *Collector { "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), + gpuPCIeReplay: prometheus.NewDesc("llmgw_gpu_pcie_replays", + "PCIe link retries on a GPU's link since the driver loaded; one that keeps climbing is a riser, slot or cable.", gpu, nil), + gpuRetired: prometheus.NewDesc("llmgw_gpu_retired_pages", + "Memory pages retired, by cause (sbe, dbe), on GPUs before Ampere.", append(gpu, "cause"), nil), + gpuRetirePend: prometheus.NewDesc("llmgw_gpu_retirement_pending", + "1 when retired pages wait for a GPU reset to take effect.", gpu, nil), + gpuRemapped: prometheus.NewDesc("llmgw_gpu_remapped_rows", + "Memory rows remapped, by kind (correctable, uncorrectable), on Ampere and later.", append(gpu, "kind"), nil), + gpuRemapPend: prometheus.NewDesc("llmgw_gpu_remap_pending", + "1 when remapped rows wait for a GPU reset to take effect.", gpu, nil), + gpuRemapFail: prometheus.NewDesc("llmgw_gpu_remap_failure", + "1 when a GPU has run out of spare memory rows: it needs replacing.", gpu, nil), + gpuNVLinkErr: prometheus.NewDesc("llmgw_gpu_nvlink_errors", + "NVLink errors summed over a GPU's links, by kind (crc, replay, recovery).", append(gpu, "kind"), nil), + gpuXID: prometheus.NewDesc("llmgw_gpu_xid_errors", + "The driver's XID error events for a GPU since its node's agent started.", gpu, nil), + gpuLastXID: prometheus.NewDesc("llmgw_gpu_last_xid", + "The code of a GPU's last XID error since its agent started; absent when there was none.", gpu, nil), gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned", "1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+ "Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.", @@ -284,6 +311,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy, + c.gpuPCIeReplay, c.gpuRetired, c.gpuRetirePend, c.gpuRemapped, c.gpuRemapPend, c.gpuRemapFail, + c.gpuNVLinkErr, c.gpuXID, c.gpuLastXID, c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -628,6 +657,44 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor for _, reason := range s.Throttle { ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...) } + if w := s.Wear; w != nil { + c.gpuWear(ch, w, labels) + } +} + +// gpuWear is what wears a card out. A count the card does not keep (-1) is +// left out, not published as zero. +func (c *Collector) gpuWear(ch chan<- prometheus.Metric, w *inventory.GPUWear, labels []string) { + gauge := func(d *prometheus.Desc, v float64, extra ...string) { + ch <- prometheus.MustNewConstMetric(d, prometheus.GaugeValue, v, append(append([]string{}, labels...), extra...)...) + } + flag := func(b bool) float64 { + if b { + return 1 + } + return 0 + } + if w.PCIeReplays >= 0 { + gauge(c.gpuPCIeReplay, float64(w.PCIeReplays)) + } + if w.RetiredSBE >= 0 || w.RetiredDBE >= 0 { + gauge(c.gpuRetired, float64(max(w.RetiredSBE, 0)), "sbe") + gauge(c.gpuRetired, float64(max(w.RetiredDBE, 0)), "dbe") + gauge(c.gpuRetirePend, flag(w.RetirePending)) + } + if w.RemappedCorrectable >= 0 || w.RemappedUncorrectable >= 0 { + gauge(c.gpuRemapped, float64(max(w.RemappedCorrectable, 0)), "correctable") + gauge(c.gpuRemapped, float64(max(w.RemappedUncorrectable, 0)), "uncorrectable") + gauge(c.gpuRemapPend, flag(w.RemapPending)) + gauge(c.gpuRemapFail, flag(w.RemapFailure)) + } + gauge(c.gpuNVLinkErr, float64(w.NVLinkCRC), "crc") + gauge(c.gpuNVLinkErr, float64(w.NVLinkReplay), "replay") + gauge(c.gpuNVLinkErr, float64(w.NVLinkRecovery), "recovery") + gauge(c.gpuXID, float64(w.XIDs)) + if w.XIDs > 0 { + gauge(c.gpuLastXID, float64(w.LastXID)) + } } // collectAssignments says which replica holds each GPU. diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go index b761de8..8204fcb 100644 --- a/internal/control/fleetmetrics/sensors_test.go +++ b/internal/control/fleetmetrics/sensors_test.go @@ -102,3 +102,28 @@ func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) { t.Error("a stopped replica's GPU was counted as held") } } + +func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) { + n := inventory.Node{ID: "g1", State: inventory.Ready, + GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}}, + Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 40, MemoryTotalMB: 81559, + Wear: &inventory.GPUWear{PCIeReplays: 12, RetiredSBE: -1, RetiredDBE: -1, RemappedCorrectable: 2, + RemappedUncorrectable: 1, RemapPending: true, NVLinkCRC: 7, XIDs: 3, LastXID: 79}}}}} + out := gather(t, fakeSource{nodes: []inventory.Node{n}}) + for _, want := range []string{ + "llmgw_gpu_pcie_replays class=h100-80g gpu=0 node=g1 -> 12\n", + "llmgw_gpu_remapped_rows class=h100-80g gpu=0 kind=uncorrectable node=g1 -> 1\n", + "llmgw_gpu_remap_pending class=h100-80g gpu=0 node=g1 -> 1\n", + "llmgw_gpu_nvlink_errors class=h100-80g gpu=0 kind=crc node=g1 -> 7\n", + "llmgw_gpu_xid_errors class=h100-80g gpu=0 node=g1 -> 3\n", + "llmgw_gpu_last_xid class=h100-80g gpu=0 node=g1 -> 79\n", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s", want) + } + } + // An H100 keeps no retired pages; none are published. + if strings.Contains(out, "llmgw_gpu_retired_pages") { + t.Errorf("a count the card does not keep was published:\n%s", out) + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 04356a6..1f86b0e 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -3,6 +3,7 @@ package inventory import ( + "fmt" "maps" "slices" "strings" @@ -328,6 +329,60 @@ type GPUSensor struct { MemoryTotalMB float64 `json:"memory_total_mb,omitempty"` SMClockMHz float64 `json:"sm_clock_mhz,omitempty"` ECCUncorrected float64 `json:"ecc_uncorrected,omitempty"` + // Wear is what wears the card out, read every minute; nil from an agent + // that does not read it. + Wear *GPUWear `json:"wear,omitempty"` +} + +// GPUWear is a card's wear and failures. Counts of -1 are not reported for +// it: a card before Ampere has no remapped rows, one after no retired pages. +type GPUWear struct { + PCIeReplays int64 `json:"pcie_replays"` + RetiredSBE int64 `json:"retired_pages_sbe"` + RetiredDBE int64 `json:"retired_pages_dbe"` + RetirePending bool `json:"retire_pending,omitempty"` + RemappedCorrectable int64 `json:"remapped_rows_correctable"` + RemappedUncorrectable int64 `json:"remapped_rows_uncorrectable"` + RemapPending bool `json:"remap_pending,omitempty"` + RemapFailure bool `json:"remap_failure,omitempty"` + NVLinkCRC int64 `json:"nvlink_crc_errors"` + NVLinkReplay int64 `json:"nvlink_replay_errors"` + NVLinkRecovery int64 `json:"nvlink_recovery_errors"` + XIDs int64 `json:"xid_errors"` + LastXID int `json:"last_xid,omitempty"` + LastXIDText string `json:"last_xid_text,omitempty"` + LastXIDAt time.Time `json:"last_xid_at,omitempty"` +} + +// Problems says, in a few words each, what is wrong with the card; empty +// when nothing is. +func (w *GPUWear) Problems() []string { + if w == nil { + return nil + } + var out []string + if w.RemapFailure { + out = append(out, "out of spare memory rows: replace the card") + } + if w.RemapPending { + out = append(out, "memory rows remapped, waiting for a reset") + } + if w.RetirePending { + out = append(out, "memory pages waiting for a reset to be retired") + } + if w.RemappedUncorrectable > 0 { + out = append(out, fmt.Sprintf("%d rows remapped for uncorrectable errors", w.RemappedUncorrectable)) + } + if w.RetiredDBE > 0 { + out = append(out, fmt.Sprintf("%d pages retired for double-bit errors", w.RetiredDBE)) + } + if w.XIDs > 0 { + out = append(out, fmt.Sprintf("XID %d (%s)", w.LastXID, w.LastXIDText)) + } + if w.NVLinkCRC > 0 || w.NVLinkReplay > 0 || w.NVLinkRecovery > 0 { + out = append(out, fmt.Sprintf("NVLink errors: %d CRC, %d replay, %d recovery", w.NVLinkCRC, w.NVLinkReplay, w.NVLinkRecovery)) + } + return out } // Busy reports whether the sample carries how busy the card is. diff --git a/internal/web/gpumap.go b/internal/web/gpumap.go index 652e202..19d86d8 100644 --- a/internal/web/gpumap.go +++ b/internal/web/gpumap.go @@ -44,6 +44,8 @@ type gpuCell struct { Title string // Label is the figure written in the cell. Label string + // Faulty is a card whose wear needs seeing to. + Faulty bool } // idleHolding is a replica holding GPUs that are doing nothing. @@ -108,12 +110,19 @@ func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, pre if len(s.Throttle) > 0 { title = append(title, "held back by "+strings.Join(s.Throttle, ", ")) } + title = append(title, s.Problems...) } c.Title = strings.Join(title, " · ") style, label := cellColour(m.By, s, sampled) if c.Model == "" { style += ";border-style:dashed" } + // A card wearing out is outlined whatever the map is coloured + // by: it is the one thing on the map that needs doing. + if len(s.Problems) > 0 { + style += ";border:2px solid #c0392b" + c.Faulty = true + } c.Style, c.Label = template.CSS(style), label row.Cells = append(row.Cells, c) } @@ -216,6 +225,7 @@ func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu type sensorView = struct { TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64 Throttle []string + Problems []string } func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) { @@ -225,7 +235,8 @@ func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) { for _, s := range n.Load.GPUs { if s.Index == gpu { return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent, - MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true + MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle, + Problems: s.Wear.Problems()}, true } } return sensorView{}, false diff --git a/internal/web/gpumap_test.go b/internal/web/gpumap_test.go index 6e4cc2c..ec0fdd7 100644 --- a/internal/web/gpumap_test.go +++ b/internal/web/gpumap_test.go @@ -20,7 +20,12 @@ func (mapFleet) Nodes() []adminapi.NodeView { return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300} } return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"}, - Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}} + Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}} +} + +func worn(s inventory.GPUSensor) inventory.GPUSensor { + s.Wear = &inventory.GPUWear{RemapPending: true, XIDs: 1, LastXID: 79, LastXIDText: "GPU has fallen off the bus."} + return s } func (mapFleet) Replicas(string) []adminapi.ReplicaView { @@ -38,6 +43,8 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) { for _, want := range []string{ "gpu07 GPU 2 · code (code-1) · 93% busy", "gpu07 GPU 3 · free", + "memory rows remapped, waiting for a reset · XID 79 (GPU has fallen off the bus.)", + "border:2px solid #c0392b", "border-style:dashed", "Cards held and doing nothing", "chat-1gpu07", "30m0s ago", diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 0d4a34a..37b2f28 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -55,7 +55,7 @@
{{range .Rows}}
{{.Node}}{{range .Cells}}{{.Label}}{{end}}
{{end}}
-

{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.

+

{{.Legend}} A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.

{{end}} {{with .Efficiency}}

What each model gets from its cards last 15 minutes

diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index 0409ad0..5a60a1a 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -35,7 +35,7 @@ {{if .Unhealthy}}

{{join .Unhealthy}}

{{end}} {{with .Load}}{{if .GPUs}}

GPUs

- +
GPUBusyMemory usedTemperatureMemory temp.PowerClockHeld back by
{{range .GPUs}} {{/* From the last sample; a dash where the driver or an older agent did @@ -46,7 +46,7 @@ - + {{end}}
GPUBusyMemory usedTemperatureMemory temp.PowerClockHeld back by, and wear
{{.Index}}{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}} {{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}{{printf "%.0f" .PowerW}} W of {{printf "%.0f" .PowerLimitW}}{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}} {{if .SMClockMHz}}{{printf "%.0f" .SMClockMHz}} MHz{{else}}—{{end}}{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if gt .ECCUncorrected 0.0}} {{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors{{end}}{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if gt .ECCUncorrected 0.0}} {{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors{{end}}{{with .Wear}}{{with .Problems}}
{{join .}}{{end}}{{if gt .PCIeReplays 0}} {{.PCIeReplays}} PCIe retries{{end}}{{end}}
{{end}}{{end}} {{end}}