Read what wears a GPU out: XIDs, remapped rows, PCIe and NVLink errors
Every minute the agent reads nvidia-smi's full report, its NVLink error counters and the kernel's XID lines. They are published per card, shown on the node page and the GPU map, charted in Grafana, and five alerts watch them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ff554c7062
commit
82a63cfc6e
16 files changed
+885
-11
No files matched your search
@@ -56,6 +56,17 @@ and a figure the driver does not give is left out rather than shown as zero.
|
||||
- **What each model gets from its cards**, on the cluster page and in
|
||||
Grafana: output tokens a second per GPU and per watt-hour, with the cards'
|
||||
draw and average utilisation, over the last quarter hour.
|
||||
- **What wears a card out**, read every minute: XID errors from the kernel's
|
||||
log (counted from when the agent started), memory rows remapped or pages
|
||||
retired and whether either waits for a reset, a card out of spare rows,
|
||||
PCIe link retries and NVLink errors. Published as `llmgw_gpu_xid_errors`,
|
||||
`llmgw_gpu_last_xid`, `llmgw_gpu_remapped_rows`, `llmgw_gpu_remap_pending`,
|
||||
`llmgw_gpu_remap_failure`, `llmgw_gpu_retired_pages`,
|
||||
`llmgw_gpu_retirement_pending`, `llmgw_gpu_pcie_replays`,
|
||||
`llmgw_gpu_nvlink_errors`; said in words on the node page; a red outline
|
||||
on the GPU map; a Grafana panel; and alerts — `GPUXidError`,
|
||||
`GPUOutOfSpareRows`, `GPUNeedsReset`, `GPUPCIeRetriesClimbing`,
|
||||
`GPUNVLinkErrorsClimbing`.
|
||||
- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds
|
||||
each card and whose quota it counts against, for joining the GPU series to
|
||||
models and tenants.
|
||||
|
||||
@@ -1561,8 +1561,27 @@ type GPUSensor struct {
|
||||
SmClockMhz float64 `protobuf:"fixed64,11,opt,name=sm_clock_mhz,json=smClockMhz,proto3" json:"sm_clock_mhz,omitempty"`
|
||||
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
|
||||
EccUncorrected float64 `protobuf:"fixed64,12,opt,name=ecc_uncorrected,json=eccUncorrected,proto3" json:"ecc_uncorrected,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// What wears the card out, read every minute; has_wear says these are
|
||||
// present. Counts of -1 are not reported for this card.
|
||||
HasWear bool `protobuf:"varint,13,opt,name=has_wear,json=hasWear,proto3" json:"has_wear,omitempty"`
|
||||
PcieReplays int64 `protobuf:"varint,14,opt,name=pcie_replays,json=pcieReplays,proto3" json:"pcie_replays,omitempty"`
|
||||
RetiredPagesSbe int64 `protobuf:"varint,15,opt,name=retired_pages_sbe,json=retiredPagesSbe,proto3" json:"retired_pages_sbe,omitempty"`
|
||||
RetiredPagesDbe int64 `protobuf:"varint,16,opt,name=retired_pages_dbe,json=retiredPagesDbe,proto3" json:"retired_pages_dbe,omitempty"`
|
||||
RetirementPending bool `protobuf:"varint,17,opt,name=retirement_pending,json=retirementPending,proto3" json:"retirement_pending,omitempty"`
|
||||
RemappedRowsCorrectable int64 `protobuf:"varint,18,opt,name=remapped_rows_correctable,json=remappedRowsCorrectable,proto3" json:"remapped_rows_correctable,omitempty"`
|
||||
RemappedRowsUncorrectable int64 `protobuf:"varint,19,opt,name=remapped_rows_uncorrectable,json=remappedRowsUncorrectable,proto3" json:"remapped_rows_uncorrectable,omitempty"`
|
||||
RemapPending bool `protobuf:"varint,20,opt,name=remap_pending,json=remapPending,proto3" json:"remap_pending,omitempty"`
|
||||
RemapFailure bool `protobuf:"varint,21,opt,name=remap_failure,json=remapFailure,proto3" json:"remap_failure,omitempty"`
|
||||
NvlinkCrcErrors int64 `protobuf:"varint,22,opt,name=nvlink_crc_errors,json=nvlinkCrcErrors,proto3" json:"nvlink_crc_errors,omitempty"`
|
||||
NvlinkReplayErrors int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"`
|
||||
NvlinkRecoveryErrors int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"`
|
||||
// xid_errors are the driver's error events since the agent started.
|
||||
XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
|
||||
LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
|
||||
LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
|
||||
LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *GPUSensor) Reset() {
|
||||
@@ -1679,6 +1698,118 @@ func (x *GPUSensor) GetEccUncorrected() float64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetHasWear() bool {
|
||||
if x != nil {
|
||||
return x.HasWear
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetPcieReplays() int64 {
|
||||
if x != nil {
|
||||
return x.PcieReplays
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRetiredPagesSbe() int64 {
|
||||
if x != nil {
|
||||
return x.RetiredPagesSbe
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRetiredPagesDbe() int64 {
|
||||
if x != nil {
|
||||
return x.RetiredPagesDbe
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRetirementPending() bool {
|
||||
if x != nil {
|
||||
return x.RetirementPending
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRemappedRowsCorrectable() int64 {
|
||||
if x != nil {
|
||||
return x.RemappedRowsCorrectable
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRemappedRowsUncorrectable() int64 {
|
||||
if x != nil {
|
||||
return x.RemappedRowsUncorrectable
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRemapPending() bool {
|
||||
if x != nil {
|
||||
return x.RemapPending
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetRemapFailure() bool {
|
||||
if x != nil {
|
||||
return x.RemapFailure
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetNvlinkCrcErrors() int64 {
|
||||
if x != nil {
|
||||
return x.NvlinkCrcErrors
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetNvlinkReplayErrors() int64 {
|
||||
if x != nil {
|
||||
return x.NvlinkReplayErrors
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetNvlinkRecoveryErrors() int64 {
|
||||
if x != nil {
|
||||
return x.NvlinkRecoveryErrors
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetXidErrors() int64 {
|
||||
if x != nil {
|
||||
return x.XidErrors
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetLastXid() int32 {
|
||||
if x != nil {
|
||||
return x.LastXid
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetLastXidText() string {
|
||||
if x != nil {
|
||||
return x.LastXidText
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *GPUSensor) GetLastXidUnix() int64 {
|
||||
if x != nil {
|
||||
return x.LastXidUnix
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
type ShardLoad struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -4051,7 +4182,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
" \x01(\x01R\tcpuPowerW\x12 \n" +
|
||||
"\finlet_temp_c\x18\v \x01(\x01R\n" +
|
||||
"inletTempC\x12'\n" +
|
||||
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xbd\x03\n" +
|
||||
"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" +
|
||||
"\tGPUSensor\x12\x14\n" +
|
||||
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" +
|
||||
"\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" +
|
||||
@@ -4066,7 +4197,24 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
" \x01(\x01R\rmemoryTotalMb\x12 \n" +
|
||||
"\fsm_clock_mhz\x18\v \x01(\x01R\n" +
|
||||
"smClockMhz\x12'\n" +
|
||||
"\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\"m\n" +
|
||||
"\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\x12\x19\n" +
|
||||
"\bhas_wear\x18\r \x01(\bR\ahasWear\x12!\n" +
|
||||
"\fpcie_replays\x18\x0e \x01(\x03R\vpcieReplays\x12*\n" +
|
||||
"\x11retired_pages_sbe\x18\x0f \x01(\x03R\x0fretiredPagesSbe\x12*\n" +
|
||||
"\x11retired_pages_dbe\x18\x10 \x01(\x03R\x0fretiredPagesDbe\x12-\n" +
|
||||
"\x12retirement_pending\x18\x11 \x01(\bR\x11retirementPending\x12:\n" +
|
||||
"\x19remapped_rows_correctable\x18\x12 \x01(\x03R\x17remappedRowsCorrectable\x12>\n" +
|
||||
"\x1bremapped_rows_uncorrectable\x18\x13 \x01(\x03R\x19remappedRowsUncorrectable\x12#\n" +
|
||||
"\rremap_pending\x18\x14 \x01(\bR\fremapPending\x12#\n" +
|
||||
"\rremap_failure\x18\x15 \x01(\bR\fremapFailure\x12*\n" +
|
||||
"\x11nvlink_crc_errors\x18\x16 \x01(\x03R\x0fnvlinkCrcErrors\x120\n" +
|
||||
"\x14nvlink_replay_errors\x18\x17 \x01(\x03R\x12nvlinkReplayErrors\x124\n" +
|
||||
"\x16nvlink_recovery_errors\x18\x18 \x01(\x03R\x14nvlinkRecoveryErrors\x12\x1d\n" +
|
||||
"\n" +
|
||||
"xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" +
|
||||
"\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" +
|
||||
"\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" +
|
||||
"\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" +
|
||||
"\tShardLoad\x12\x19\n" +
|
||||
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
|
||||
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
|
||||
|
||||
@@ -314,6 +314,25 @@ message GPUSensor {
|
||||
double sm_clock_mhz = 11;
|
||||
// ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
|
||||
double ecc_uncorrected = 12;
|
||||
// What wears the card out, read every minute; has_wear says these are
|
||||
// present. Counts of -1 are not reported for this card.
|
||||
bool has_wear = 13;
|
||||
int64 pcie_replays = 14;
|
||||
int64 retired_pages_sbe = 15;
|
||||
int64 retired_pages_dbe = 16;
|
||||
bool retirement_pending = 17;
|
||||
int64 remapped_rows_correctable = 18;
|
||||
int64 remapped_rows_uncorrectable = 19;
|
||||
bool remap_pending = 20;
|
||||
bool remap_failure = 21;
|
||||
int64 nvlink_crc_errors = 22;
|
||||
int64 nvlink_replay_errors = 23;
|
||||
int64 nvlink_recovery_errors = 24;
|
||||
// xid_errors are the driver's error events since the agent started.
|
||||
int64 xid_errors = 25;
|
||||
int32 last_xid = 26;
|
||||
string last_xid_text = 27;
|
||||
int64 last_xid_unix = 28;
|
||||
}
|
||||
|
||||
// ShardLoad is one engine process group's use.
|
||||
|
||||
@@ -855,6 +855,59 @@
|
||||
"mode": "multi"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "GPU wear",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "llmgw-prom"
|
||||
},
|
||||
"id": 24,
|
||||
"description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 66,
|
||||
"w": 24,
|
||||
"h": 7
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "increase(llmgw_gpu_xid_errors[15m]) > 0",
|
||||
"legendFormat": "XID {{node}} gpu {{gpu}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0",
|
||||
"legendFormat": "rows remapped {{node}} gpu {{gpu}}",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0",
|
||||
"legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0",
|
||||
"legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}",
|
||||
"refId": "D"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -224,6 +224,60 @@ groups:
|
||||
card may be computing on corrupted memory. Drain the node
|
||||
(llmgw ctl drain) and have the card looked at.
|
||||
|
||||
- alert: GPUXidError
|
||||
expr: increase(llmgw_gpu_xid_errors[15m]) > 0
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "{{ $labels.node }} GPU {{ $labels.gpu }} logged an XID error"
|
||||
description: >-
|
||||
The driver reported an error on the card (its code is
|
||||
llmgw_gpu_last_xid, and the node page says it in words). 79 is a
|
||||
card fallen off the bus, 48 and 63/64 memory errors, 74 NVLink,
|
||||
94/95 contained and uncontained ECC errors: those need the card
|
||||
looked at. Others are often an engine's own fault, and pass.
|
||||
|
||||
- alert: GPUOutOfSpareRows
|
||||
expr: llmgw_gpu_remap_failure == 1
|
||||
for: 1m
|
||||
labels: { severity: critical }
|
||||
annotations:
|
||||
summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has no spare memory rows left"
|
||||
description: >-
|
||||
The card cannot remap another failing row. Drain the node and
|
||||
have the card replaced.
|
||||
|
||||
- alert: GPUNeedsReset
|
||||
expr: llmgw_gpu_remap_pending == 1 or llmgw_gpu_retirement_pending == 1
|
||||
for: 5m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has memory repairs waiting for a reset"
|
||||
description: >-
|
||||
Rows remapped or pages retired take effect only after the card is
|
||||
reset. Drain the node (llmgw ctl drain), reset or reboot, and bring
|
||||
it back.
|
||||
|
||||
- alert: GPUPCIeRetriesClimbing
|
||||
expr: increase(llmgw_gpu_pcie_replays[1h]) > 100
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: {{ $value }} PCIe retries in an hour"
|
||||
description: >-
|
||||
The card's link to the host is retrying. A riser, the slot or a
|
||||
cable is the usual cause, and transfers to the card slow down.
|
||||
|
||||
- alert: GPUNVLinkErrorsClimbing
|
||||
expr: increase(llmgw_gpu_nvlink_errors{kind="crc"}[1h]) > 10
|
||||
for: 0m
|
||||
labels: { severity: warning }
|
||||
annotations:
|
||||
summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: NVLink CRC errors climbing"
|
||||
description: >-
|
||||
Errors on the card's NVLink links: tensor-parallel replicas on it
|
||||
slow down, and a link that keeps failing takes a replica with it.
|
||||
|
||||
- alert: GPUHot
|
||||
expr: llmgw_gpu_temperature_celsius > 87
|
||||
for: 10m
|
||||
|
||||
+25
-2
@@ -104,6 +104,8 @@ type Agent struct {
|
||||
// sensors reads heat and power: CPU and board from hwmon and RAPL, GPUs
|
||||
// from nvidia-smi.
|
||||
sensors *discovery.SensorSampler
|
||||
// wear reads what wears a GPU out: remapped rows, PCIe retries, XIDs.
|
||||
wear *discovery.HealthSampler
|
||||
|
||||
// fetchWatchers are the shards waiting on each source, so progress on one
|
||||
// download reaches every shard that asked for it. A fetch is single-flight
|
||||
@@ -150,6 +152,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
provKick: make(chan struct{}, 1),
|
||||
load: discovery.NewLoadSampler(),
|
||||
sensors: discovery.NewSensorSampler(),
|
||||
wear: discovery.NewHealthSampler(),
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
return a, nil
|
||||
@@ -656,14 +659,34 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
|
||||
if a.sensors != nil {
|
||||
se := a.sensors.Sample(context.Background())
|
||||
out.CpuTempC, out.CpuPowerW, out.InletTempC = se.CPUTempC, se.CPUPowerW, se.InletTempC
|
||||
var wear map[int]discovery.GPUWear
|
||||
if a.wear != nil && len(se.GPUs) > 0 {
|
||||
wear = map[int]discovery.GPUWear{}
|
||||
for _, w := range a.wear.Sample(context.Background()) {
|
||||
wear[w.Index] = w
|
||||
}
|
||||
}
|
||||
for _, g := range se.GPUs {
|
||||
out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{
|
||||
gs := &llmgwv1.GPUSensor{
|
||||
Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC,
|
||||
PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle,
|
||||
UtilizationPercent: g.UtilPercent, MemoryUtilizationPercent: g.MemUtilPercent,
|
||||
MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB,
|
||||
SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected,
|
||||
})
|
||||
}
|
||||
if w, ok := wear[g.Index]; ok {
|
||||
gs.HasWear = true
|
||||
gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending =
|
||||
w.PCIeReplays, w.RetiredSBE, w.RetiredDBE, w.RetirePending
|
||||
gs.RemappedRowsCorrectable, gs.RemappedRowsUncorrectable, gs.RemapPending, gs.RemapFailure =
|
||||
w.RemappedCorrectable, w.RemappedUncorrectable, w.RemapPending, w.RemapFailure
|
||||
gs.NvlinkCrcErrors, gs.NvlinkReplayErrors, gs.NvlinkRecoveryErrors = w.NVLinkCRC, w.NVLinkReplay, w.NVLinkRecovery
|
||||
gs.XidErrors, gs.LastXid, gs.LastXidText = w.XIDs, int32(w.LastXID), w.LastXIDText
|
||||
if !w.LastXIDAt.IsZero() {
|
||||
gs.LastXidUnix = w.LastXIDAt.Unix()
|
||||
}
|
||||
}
|
||||
out.Gpus = append(out.Gpus, gs)
|
||||
}
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GPUWear is what wears a card out or takes it down, beyond temperature:
|
||||
// read every minute, from nvidia-smi and the kernel's log. The early signs of
|
||||
// a card failing are here — a growing count of rows its memory had to remap,
|
||||
// PCIe link retries, XID errors — long before a replica on it fails.
|
||||
//
|
||||
// A count of -1 is not reported: a card without row remapping (before
|
||||
// Ampere) has no such count, which is not the same as none.
|
||||
type GPUWear struct {
|
||||
Index int
|
||||
// PCIeReplays are link-level retries on the card's PCIe link: a few are
|
||||
// noise, a count that keeps climbing is a riser, slot or cable.
|
||||
PCIeReplays int64
|
||||
// RetiredSBE and RetiredDBE are memory pages retired for single and
|
||||
// double bit errors (cards before Ampere); RetirePending is a page
|
||||
// waiting for a reset to be retired.
|
||||
RetiredSBE, RetiredDBE int64
|
||||
RetirePending bool
|
||||
// RemappedCorrectable and RemappedUncorrectable are memory rows remapped
|
||||
// (Ampere and later). RemapPending waits for a reset; RemapFailure is a
|
||||
// card out of spare rows — it needs replacing.
|
||||
RemappedCorrectable, RemappedUncorrectable int64
|
||||
RemapPending, RemapFailure bool
|
||||
// NVLink errors summed over the card's links, by kind.
|
||||
NVLinkCRC, NVLinkReplay, NVLinkRecovery int64
|
||||
// XIDs are the driver's error events for this card since the agent
|
||||
// started, and the last of them.
|
||||
XIDs int64
|
||||
LastXID int
|
||||
LastXIDText string
|
||||
LastXIDAt time.Time
|
||||
}
|
||||
|
||||
// healthEvery is how often health is read: it changes over hours, and the
|
||||
// kernel log and nvidia-smi's full report are the heavier reads.
|
||||
const healthEvery = time.Minute
|
||||
|
||||
// HealthSampler reads GPUWear, keeping the XID counts between reads.
|
||||
type HealthSampler struct {
|
||||
run func(ctx context.Context, name string, args ...string) ([]byte, error)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
last []GPUWear
|
||||
since time.Time
|
||||
xids map[int]*GPUWear
|
||||
started bool
|
||||
}
|
||||
|
||||
// NewHealthSampler reads the machine's own nvidia-smi and kernel log.
|
||||
func NewHealthSampler() *HealthSampler {
|
||||
return &HealthSampler{now: time.Now, xids: map[int]*GPUWear{},
|
||||
run: func(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
return exec.CommandContext(ctx, name, args...).Output() //nolint:gosec // fixed tools
|
||||
}}
|
||||
}
|
||||
|
||||
// Sample is the cards' health, read again once healthEvery has passed. Nil
|
||||
// on a machine without nvidia-smi.
|
||||
func (h *HealthSampler) Sample(ctx context.Context) []GPUWear {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
now := h.now()
|
||||
if !h.at.IsZero() && now.Sub(h.at) < healthEvery {
|
||||
return h.last
|
||||
}
|
||||
h.at = now
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
report, err := h.run(ctx, "nvidia-smi", "-q", "-x")
|
||||
if err != nil {
|
||||
h.last = nil
|
||||
return nil
|
||||
}
|
||||
cards, bus := parseHealthReport(report)
|
||||
if out, err := h.run(ctx, "nvidia-smi", "nvlink", "-e"); err == nil {
|
||||
for i, e := range parseNVLinkErrors(string(out)) {
|
||||
if i < len(cards) {
|
||||
cards[i].NVLinkCRC, cards[i].NVLinkReplay, cards[i].NVLinkRecovery = e[0], e[1], e[2]
|
||||
}
|
||||
}
|
||||
}
|
||||
h.readXIDs(ctx, now, bus)
|
||||
for i := range cards {
|
||||
if x := h.xids[i]; x != nil {
|
||||
cards[i].XIDs, cards[i].LastXID, cards[i].LastXIDText, cards[i].LastXIDAt = x.XIDs, x.LastXID, x.LastXIDText, x.LastXIDAt
|
||||
}
|
||||
}
|
||||
h.last = cards
|
||||
return cards
|
||||
}
|
||||
|
||||
// readXIDs adds the XID events logged since the last read. The first read
|
||||
// starts from now: what happened before the agent started was before, and
|
||||
// counting the whole log would raise every alert on every restart.
|
||||
func (h *HealthSampler) readXIDs(ctx context.Context, now time.Time, bus map[string]int) {
|
||||
if !h.started {
|
||||
h.started, h.since = true, now
|
||||
return
|
||||
}
|
||||
since := h.since
|
||||
h.since = now
|
||||
out, err := h.run(ctx, "journalctl", "-k", "--no-pager", "-o", "short-unix",
|
||||
"--since", "@"+strconv.FormatInt(since.Unix(), 10), "-g", "NVRM: Xid")
|
||||
if err != nil {
|
||||
// No journal, or nothing matched (journalctl exits 1 for that).
|
||||
return
|
||||
}
|
||||
for _, e := range parseXIDs(string(out)) {
|
||||
i, ok := bus[e.bus]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
x := h.xids[i]
|
||||
if x == nil {
|
||||
x = &GPUWear{Index: i}
|
||||
h.xids[i] = x
|
||||
}
|
||||
x.XIDs++
|
||||
x.LastXID, x.LastXIDText = e.code, e.text
|
||||
x.LastXIDAt = now
|
||||
if e.at > 0 {
|
||||
x.LastXIDAt = time.Unix(int64(e.at), 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// healthLog is the part of nvidia-smi's XML report health is read from.
|
||||
type healthLog struct {
|
||||
GPUs []struct {
|
||||
ID string `xml:"id,attr"`
|
||||
PCI struct {
|
||||
Replay string `xml:"replay_counter"`
|
||||
} `xml:"pci"`
|
||||
Retired struct {
|
||||
SBE string `xml:"multiple_single_bit_retirement>retired_count"`
|
||||
DBE string `xml:"double_bit_retirement>retired_count"`
|
||||
Pending string `xml:"pending_retirement"`
|
||||
// Older drivers name it pending_blacklist.
|
||||
Blacklist string `xml:"pending_blacklist"`
|
||||
} `xml:"retired_pages"`
|
||||
Remapped struct {
|
||||
Corr string `xml:"remapped_row_corr"`
|
||||
Unc string `xml:"remapped_row_unc"`
|
||||
Pending string `xml:"remapped_row_pending"`
|
||||
Failure string `xml:"remapped_row_failure"`
|
||||
} `xml:"remapped_rows"`
|
||||
} `xml:"gpu"`
|
||||
}
|
||||
|
||||
// parseHealthReport reads each card's health, in index order, and maps each
|
||||
// card's PCI bus ("3b:00", as the kernel names it) to its index.
|
||||
func parseHealthReport(report []byte) ([]GPUWear, map[string]int) {
|
||||
var log healthLog
|
||||
if err := xml.Unmarshal(report, &log); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
bus := map[string]int{}
|
||||
out := make([]GPUWear, 0, len(log.GPUs))
|
||||
for i, g := range log.GPUs {
|
||||
bus[busOf(g.ID)] = i
|
||||
out = append(out, GPUWear{
|
||||
Index: i,
|
||||
PCIeReplays: count(g.PCI.Replay),
|
||||
RetiredSBE: count(g.Retired.SBE),
|
||||
RetiredDBE: count(g.Retired.DBE),
|
||||
RetirePending: yes(g.Retired.Pending) || yes(g.Retired.Blacklist),
|
||||
RemappedCorrectable: count(g.Remapped.Corr),
|
||||
RemappedUncorrectable: count(g.Remapped.Unc),
|
||||
RemapPending: yes(g.Remapped.Pending),
|
||||
RemapFailure: yes(g.Remapped.Failure),
|
||||
})
|
||||
}
|
||||
return out, bus
|
||||
}
|
||||
|
||||
// busOf is a PCI address as the kernel's XID lines carry it: nvidia-smi
|
||||
// says "00000000:3B:00.0", the kernel "0000:3b:00".
|
||||
func busOf(id string) string {
|
||||
id = strings.ToLower(strings.TrimSpace(id))
|
||||
if i := strings.LastIndex(id, "."); i >= 0 {
|
||||
id = id[:i]
|
||||
}
|
||||
parts := strings.Split(id, ":")
|
||||
if len(parts) >= 2 {
|
||||
return parts[len(parts)-2] + ":" + parts[len(parts)-1]
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func count(s string) int64 {
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func yes(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "yes") }
|
||||
|
||||
// nvlinkError matches a line of `nvidia-smi nvlink -e`, whose wording varies
|
||||
// by driver: "Link 0: Replay Errors: 0", "Link 0: CRC Errors: 0",
|
||||
// "Link 3: Data CRC Errors: 2".
|
||||
var (
|
||||
nvlinkGPU = regexp.MustCompile(`^GPU (\d+):`)
|
||||
nvlinkError = regexp.MustCompile(`Link \d+:\s*(.*?)\s*Errors?\s*:\s*(\d+)`)
|
||||
)
|
||||
|
||||
// parseNVLinkErrors sums each card's link errors by kind: CRC, replay,
|
||||
// recovery.
|
||||
func parseNVLinkErrors(out string) map[int][3]int64 {
|
||||
res := map[int][3]int64{}
|
||||
gpu := -1
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if m := nvlinkGPU.FindStringSubmatch(strings.TrimSpace(line)); m != nil {
|
||||
gpu, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
m := nvlinkError.FindStringSubmatch(line)
|
||||
if m == nil || gpu < 0 {
|
||||
continue
|
||||
}
|
||||
n, _ := strconv.ParseInt(m[2], 10, 64)
|
||||
e := res[gpu]
|
||||
kind := strings.ToLower(m[1])
|
||||
switch {
|
||||
case strings.Contains(kind, "crc"):
|
||||
e[0] += n
|
||||
case strings.Contains(kind, "replay"):
|
||||
e[1] += n
|
||||
case strings.Contains(kind, "recovery"):
|
||||
e[2] += n
|
||||
}
|
||||
res[gpu] = e
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
type xidEvent struct {
|
||||
bus string
|
||||
code int
|
||||
text string
|
||||
at float64
|
||||
}
|
||||
|
||||
// xidLine matches the driver's report of an XID:
|
||||
// "NVRM: Xid (PCI:0000:3b:00): 79, pid=..., GPU has fallen off the bus."
|
||||
var xidLine = regexp.MustCompile(`NVRM: Xid \(PCI:([0-9a-fA-F:.]+)\): (\d+),\s*(.*)$`)
|
||||
|
||||
// parseXIDs reads journalctl's short-unix lines: a timestamp, then the
|
||||
// message.
|
||||
func parseXIDs(out string) []xidEvent {
|
||||
var evs []xidEvent
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
m := xidLine.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
code, _ := strconv.Atoi(m[2])
|
||||
e := xidEvent{bus: busOf(m[1]), code: code, text: strings.TrimSpace(m[3])}
|
||||
if f := strings.Fields(line); len(f) > 0 {
|
||||
e.at, _ = strconv.ParseFloat(f[0], 64)
|
||||
}
|
||||
evs = append(evs, e)
|
||||
}
|
||||
return evs
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const healthXML = `<?xml version="1.0" ?>
|
||||
<nvidia_smi_log>
|
||||
<gpu id="00000000:3B:00.0">
|
||||
<pci><replay_counter>0</replay_counter></pci>
|
||||
<retired_pages>
|
||||
<multiple_single_bit_retirement><retired_count>N/A</retired_count></multiple_single_bit_retirement>
|
||||
<double_bit_retirement><retired_count>N/A</retired_count></double_bit_retirement>
|
||||
<pending_retirement>N/A</pending_retirement>
|
||||
</retired_pages>
|
||||
<remapped_rows>
|
||||
<remapped_row_corr>2</remapped_row_corr>
|
||||
<remapped_row_unc>1</remapped_row_unc>
|
||||
<remapped_row_pending>Yes</remapped_row_pending>
|
||||
<remapped_row_failure>No</remapped_row_failure>
|
||||
</remapped_rows>
|
||||
</gpu>
|
||||
<gpu id="00000000:5E:00.0">
|
||||
<pci><replay_counter>4021</replay_counter></pci>
|
||||
<retired_pages>
|
||||
<multiple_single_bit_retirement><retired_count>3</retired_count></multiple_single_bit_retirement>
|
||||
<double_bit_retirement><retired_count>1</retired_count></double_bit_retirement>
|
||||
<pending_blacklist>Yes</pending_blacklist>
|
||||
</retired_pages>
|
||||
<remapped_rows>N/A</remapped_rows>
|
||||
</gpu>
|
||||
</nvidia_smi_log>`
|
||||
|
||||
const nvlinkOut = `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-aaa)
|
||||
Link 0: Replay Errors: 1
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 5
|
||||
Link 1: Data CRC Errors: 2
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-bbb)
|
||||
Link 0: Replay Errors: 0
|
||||
`
|
||||
|
||||
func TestAGPUsWearAndFailuresAreReadAndCountedFromWhenTheAgentStarted(t *testing.T) {
|
||||
now := time.Unix(1_800_000_000, 0)
|
||||
journal := ""
|
||||
h := &HealthSampler{now: func() time.Time { return now }, xids: map[int]*GPUWear{},
|
||||
run: func(_ context.Context, name string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case name == "nvidia-smi" && args[0] == "-q":
|
||||
return []byte(healthXML), nil
|
||||
case name == "nvidia-smi" && args[0] == "nvlink":
|
||||
return []byte(nvlinkOut), nil
|
||||
case name == "journalctl":
|
||||
if journal == "" {
|
||||
return nil, errors.New("exit status 1")
|
||||
}
|
||||
return []byte(journal), nil
|
||||
}
|
||||
return nil, errors.New("unexpected " + name)
|
||||
}}
|
||||
got := h.Sample(t.Context())
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
a, b := got[0], got[1]
|
||||
if a.RemappedCorrectable != 2 || a.RemappedUncorrectable != 1 || !a.RemapPending || a.RemapFailure ||
|
||||
a.RetiredSBE != -1 || a.PCIeReplays != 0 || a.NVLinkCRC != 7 || a.NVLinkReplay != 1 {
|
||||
t.Errorf("card 0 %+v", a)
|
||||
}
|
||||
if b.PCIeReplays != 4021 || b.RetiredSBE != 3 || b.RetiredDBE != 1 || !b.RetirePending || b.RemappedCorrectable != -1 {
|
||||
t.Errorf("card 1 %+v", b)
|
||||
}
|
||||
|
||||
// An XID on card 1, logged after the agent started; read on the next
|
||||
// sample a minute on.
|
||||
journal = "1800000030.123456 gpu07 kernel: NVRM: Xid (PCI:0000:5e:00): 79, pid=1234, name=llama-server, GPU has fallen off the bus.\n"
|
||||
now = now.Add(61 * time.Second)
|
||||
got = h.Sample(t.Context())
|
||||
if got[1].XIDs != 1 || got[1].LastXID != 79 || !strings.Contains(got[1].LastXIDText, "fallen off the bus") ||
|
||||
got[1].LastXIDAt.Unix() != 1_800_000_030 || got[0].XIDs != 0 {
|
||||
t.Errorf("after an XID: %+v / %+v", got[0], got[1])
|
||||
}
|
||||
// Within the minute, the last reading.
|
||||
journal = ""
|
||||
now = now.Add(10 * time.Second)
|
||||
if again := h.Sample(t.Context()); again[1].XIDs != 1 {
|
||||
t.Errorf("%+v", again[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestBusIDsAreReadAsTheKernelWritesThem(t *testing.T) {
|
||||
for in, want := range map[string]string{"00000000:3B:00.0": "3b:00", "0000:3b:00": "3b:00", "0000:5e:00.0": "5e:00"} {
|
||||
if got := busOf(in); got != want {
|
||||
t.Errorf("%s: %s, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -588,6 +588,7 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
|
||||
UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(),
|
||||
MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(),
|
||||
SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(),
|
||||
Wear: wearFrom(g),
|
||||
})
|
||||
}
|
||||
return out
|
||||
@@ -964,3 +965,22 @@ func (s *Service) Upgrade(ctx context.Context, nodeID, version, releases string)
|
||||
return fmt.Errorf("node %s did not answer the upgrade; an agent older than v0.10.4 cannot upgrade itself", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// wearFrom is a card's wear, when the agent read it.
|
||||
func wearFrom(g *llmgwv1.GPUSensor) *inventory.GPUWear {
|
||||
if !g.GetHasWear() {
|
||||
return nil
|
||||
}
|
||||
w := &inventory.GPUWear{
|
||||
PCIeReplays: g.GetPcieReplays(), RetiredSBE: g.GetRetiredPagesSbe(), RetiredDBE: g.GetRetiredPagesDbe(),
|
||||
RetirePending: g.GetRetirementPending(), RemappedCorrectable: g.GetRemappedRowsCorrectable(),
|
||||
RemappedUncorrectable: g.GetRemappedRowsUncorrectable(), RemapPending: g.GetRemapPending(),
|
||||
RemapFailure: g.GetRemapFailure(), NVLinkCRC: g.GetNvlinkCrcErrors(), NVLinkReplay: g.GetNvlinkReplayErrors(),
|
||||
NVLinkRecovery: g.GetNvlinkRecoveryErrors(), XIDs: g.GetXidErrors(), LastXID: int(g.GetLastXid()),
|
||||
LastXIDText: g.GetLastXidText(),
|
||||
}
|
||||
if u := g.GetLastXidUnix(); u > 0 {
|
||||
w.LastXIDAt = time.Unix(u, 0)
|
||||
}
|
||||
return w
|
||||
}
|
||||
@@ -74,6 +74,15 @@ type Collector struct {
|
||||
gpuMemoryUsed *prometheus.Desc
|
||||
gpuUtil *prometheus.Desc
|
||||
gpuAssigned *prometheus.Desc
|
||||
gpuPCIeReplay *prometheus.Desc
|
||||
gpuRetired *prometheus.Desc
|
||||
gpuRetirePend *prometheus.Desc
|
||||
gpuRemapped *prometheus.Desc
|
||||
gpuRemapPend *prometheus.Desc
|
||||
gpuRemapFail *prometheus.Desc
|
||||
gpuNVLinkErr *prometheus.Desc
|
||||
gpuXID *prometheus.Desc
|
||||
gpuLastXID *prometheus.Desc
|
||||
gpuMemUtil *prometheus.Desc
|
||||
gpuSMClock *prometheus.Desc
|
||||
gpuTemp *prometheus.Desc
|
||||
@@ -152,6 +161,24 @@ func New(src Source) *Collector {
|
||||
"How busy a GPU was over the last sample period. Absent when the driver does not "+
|
||||
"report it, which is not the same as idle. Memory used says a model is loaded; "+
|
||||
"this says whether it is doing anything.", gpu, nil),
|
||||
gpuPCIeReplay: prometheus.NewDesc("llmgw_gpu_pcie_replays",
|
||||
"PCIe link retries on a GPU's link since the driver loaded; one that keeps climbing is a riser, slot or cable.", gpu, nil),
|
||||
gpuRetired: prometheus.NewDesc("llmgw_gpu_retired_pages",
|
||||
"Memory pages retired, by cause (sbe, dbe), on GPUs before Ampere.", append(gpu, "cause"), nil),
|
||||
gpuRetirePend: prometheus.NewDesc("llmgw_gpu_retirement_pending",
|
||||
"1 when retired pages wait for a GPU reset to take effect.", gpu, nil),
|
||||
gpuRemapped: prometheus.NewDesc("llmgw_gpu_remapped_rows",
|
||||
"Memory rows remapped, by kind (correctable, uncorrectable), on Ampere and later.", append(gpu, "kind"), nil),
|
||||
gpuRemapPend: prometheus.NewDesc("llmgw_gpu_remap_pending",
|
||||
"1 when remapped rows wait for a GPU reset to take effect.", gpu, nil),
|
||||
gpuRemapFail: prometheus.NewDesc("llmgw_gpu_remap_failure",
|
||||
"1 when a GPU has run out of spare memory rows: it needs replacing.", gpu, nil),
|
||||
gpuNVLinkErr: prometheus.NewDesc("llmgw_gpu_nvlink_errors",
|
||||
"NVLink errors summed over a GPU's links, by kind (crc, replay, recovery).", append(gpu, "kind"), nil),
|
||||
gpuXID: prometheus.NewDesc("llmgw_gpu_xid_errors",
|
||||
"The driver's XID error events for a GPU since its node's agent started.", gpu, nil),
|
||||
gpuLastXID: prometheus.NewDesc("llmgw_gpu_last_xid",
|
||||
"The code of a GPU's last XID error since its agent started; absent when there was none.", gpu, nil),
|
||||
gpuAssigned: prometheus.NewDesc("llmgw_gpu_assigned",
|
||||
"1 for each GPU a replica holds: which model, which replica, and whose quota it counts against. "+
|
||||
"Joined on node and gpu with the other llmgw_gpu_ series, it says what each model's cards are doing.",
|
||||
@@ -284,6 +311,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||
c.gpuNVLJoined, c.imexUp,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuPCIeReplay, c.gpuRetired, c.gpuRetirePend, c.gpuRemapped, c.gpuRemapPend, c.gpuRemapFail,
|
||||
c.gpuNVLinkErr, c.gpuXID, c.gpuLastXID,
|
||||
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
@@ -628,6 +657,44 @@ func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor
|
||||
for _, reason := range s.Throttle {
|
||||
ch <- prometheus.MustNewConstMetric(c.gpuThrottled, prometheus.GaugeValue, 1, append(labels, reason)...)
|
||||
}
|
||||
if w := s.Wear; w != nil {
|
||||
c.gpuWear(ch, w, labels)
|
||||
}
|
||||
}
|
||||
|
||||
// gpuWear is what wears a card out. A count the card does not keep (-1) is
|
||||
// left out, not published as zero.
|
||||
func (c *Collector) gpuWear(ch chan<- prometheus.Metric, w *inventory.GPUWear, labels []string) {
|
||||
gauge := func(d *prometheus.Desc, v float64, extra ...string) {
|
||||
ch <- prometheus.MustNewConstMetric(d, prometheus.GaugeValue, v, append(append([]string{}, labels...), extra...)...)
|
||||
}
|
||||
flag := func(b bool) float64 {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if w.PCIeReplays >= 0 {
|
||||
gauge(c.gpuPCIeReplay, float64(w.PCIeReplays))
|
||||
}
|
||||
if w.RetiredSBE >= 0 || w.RetiredDBE >= 0 {
|
||||
gauge(c.gpuRetired, float64(max(w.RetiredSBE, 0)), "sbe")
|
||||
gauge(c.gpuRetired, float64(max(w.RetiredDBE, 0)), "dbe")
|
||||
gauge(c.gpuRetirePend, flag(w.RetirePending))
|
||||
}
|
||||
if w.RemappedCorrectable >= 0 || w.RemappedUncorrectable >= 0 {
|
||||
gauge(c.gpuRemapped, float64(max(w.RemappedCorrectable, 0)), "correctable")
|
||||
gauge(c.gpuRemapped, float64(max(w.RemappedUncorrectable, 0)), "uncorrectable")
|
||||
gauge(c.gpuRemapPend, flag(w.RemapPending))
|
||||
gauge(c.gpuRemapFail, flag(w.RemapFailure))
|
||||
}
|
||||
gauge(c.gpuNVLinkErr, float64(w.NVLinkCRC), "crc")
|
||||
gauge(c.gpuNVLinkErr, float64(w.NVLinkReplay), "replay")
|
||||
gauge(c.gpuNVLinkErr, float64(w.NVLinkRecovery), "recovery")
|
||||
gauge(c.gpuXID, float64(w.XIDs))
|
||||
if w.XIDs > 0 {
|
||||
gauge(c.gpuLastXID, float64(w.LastXID))
|
||||
}
|
||||
}
|
||||
|
||||
// collectAssignments says which replica holds each GPU.
|
||||
|
||||
@@ -102,3 +102,28 @@ func TestEachGPUAReplicaHoldsSaysWhoseItIs(t *testing.T) {
|
||||
t.Error("a stopped replica's GPU was counted as held")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) {
|
||||
n := inventory.Node{ID: "g1", State: inventory.Ready,
|
||||
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}},
|
||||
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 40, MemoryTotalMB: 81559,
|
||||
Wear: &inventory.GPUWear{PCIeReplays: 12, RetiredSBE: -1, RetiredDBE: -1, RemappedCorrectable: 2,
|
||||
RemappedUncorrectable: 1, RemapPending: true, NVLinkCRC: 7, XIDs: 3, LastXID: 79}}}}}
|
||||
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
|
||||
for _, want := range []string{
|
||||
"llmgw_gpu_pcie_replays class=h100-80g gpu=0 node=g1 -> 12\n",
|
||||
"llmgw_gpu_remapped_rows class=h100-80g gpu=0 kind=uncorrectable node=g1 -> 1\n",
|
||||
"llmgw_gpu_remap_pending class=h100-80g gpu=0 node=g1 -> 1\n",
|
||||
"llmgw_gpu_nvlink_errors class=h100-80g gpu=0 kind=crc node=g1 -> 7\n",
|
||||
"llmgw_gpu_xid_errors class=h100-80g gpu=0 node=g1 -> 3\n",
|
||||
"llmgw_gpu_last_xid class=h100-80g gpu=0 node=g1 -> 79\n",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %s", want)
|
||||
}
|
||||
}
|
||||
// An H100 keeps no retired pages; none are published.
|
||||
if strings.Contains(out, "llmgw_gpu_retired_pages") {
|
||||
t.Errorf("a count the card does not keep was published:\n%s", out)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -328,6 +329,60 @@ type GPUSensor struct {
|
||||
MemoryTotalMB float64 `json:"memory_total_mb,omitempty"`
|
||||
SMClockMHz float64 `json:"sm_clock_mhz,omitempty"`
|
||||
ECCUncorrected float64 `json:"ecc_uncorrected,omitempty"`
|
||||
// Wear is what wears the card out, read every minute; nil from an agent
|
||||
// that does not read it.
|
||||
Wear *GPUWear `json:"wear,omitempty"`
|
||||
}
|
||||
|
||||
// GPUWear is a card's wear and failures. Counts of -1 are not reported for
|
||||
// it: a card before Ampere has no remapped rows, one after no retired pages.
|
||||
type GPUWear struct {
|
||||
PCIeReplays int64 `json:"pcie_replays"`
|
||||
RetiredSBE int64 `json:"retired_pages_sbe"`
|
||||
RetiredDBE int64 `json:"retired_pages_dbe"`
|
||||
RetirePending bool `json:"retire_pending,omitempty"`
|
||||
RemappedCorrectable int64 `json:"remapped_rows_correctable"`
|
||||
RemappedUncorrectable int64 `json:"remapped_rows_uncorrectable"`
|
||||
RemapPending bool `json:"remap_pending,omitempty"`
|
||||
RemapFailure bool `json:"remap_failure,omitempty"`
|
||||
NVLinkCRC int64 `json:"nvlink_crc_errors"`
|
||||
NVLinkReplay int64 `json:"nvlink_replay_errors"`
|
||||
NVLinkRecovery int64 `json:"nvlink_recovery_errors"`
|
||||
XIDs int64 `json:"xid_errors"`
|
||||
LastXID int `json:"last_xid,omitempty"`
|
||||
LastXIDText string `json:"last_xid_text,omitempty"`
|
||||
LastXIDAt time.Time `json:"last_xid_at,omitempty"`
|
||||
}
|
||||
|
||||
// Problems says, in a few words each, what is wrong with the card; empty
|
||||
// when nothing is.
|
||||
func (w *GPUWear) Problems() []string {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
if w.RemapFailure {
|
||||
out = append(out, "out of spare memory rows: replace the card")
|
||||
}
|
||||
if w.RemapPending {
|
||||
out = append(out, "memory rows remapped, waiting for a reset")
|
||||
}
|
||||
if w.RetirePending {
|
||||
out = append(out, "memory pages waiting for a reset to be retired")
|
||||
}
|
||||
if w.RemappedUncorrectable > 0 {
|
||||
out = append(out, fmt.Sprintf("%d rows remapped for uncorrectable errors", w.RemappedUncorrectable))
|
||||
}
|
||||
if w.RetiredDBE > 0 {
|
||||
out = append(out, fmt.Sprintf("%d pages retired for double-bit errors", w.RetiredDBE))
|
||||
}
|
||||
if w.XIDs > 0 {
|
||||
out = append(out, fmt.Sprintf("XID %d (%s)", w.LastXID, w.LastXIDText))
|
||||
}
|
||||
if w.NVLinkCRC > 0 || w.NVLinkReplay > 0 || w.NVLinkRecovery > 0 {
|
||||
out = append(out, fmt.Sprintf("NVLink errors: %d CRC, %d replay, %d recovery", w.NVLinkCRC, w.NVLinkReplay, w.NVLinkRecovery))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Busy reports whether the sample carries how busy the card is.
|
||||
|
||||
+12
-1
@@ -44,6 +44,8 @@ type gpuCell struct {
|
||||
Title string
|
||||
// Label is the figure written in the cell.
|
||||
Label string
|
||||
// Faulty is a card whose wear needs seeing to.
|
||||
Faulty bool
|
||||
}
|
||||
|
||||
// idleHolding is a replica holding GPUs that are doing nothing.
|
||||
@@ -108,12 +110,19 @@ func buildGPUMap(nodes []adminapi.NodeView, reps []adminapi.ReplicaView, by, pre
|
||||
if len(s.Throttle) > 0 {
|
||||
title = append(title, "held back by "+strings.Join(s.Throttle, ", "))
|
||||
}
|
||||
title = append(title, s.Problems...)
|
||||
}
|
||||
c.Title = strings.Join(title, " · ")
|
||||
style, label := cellColour(m.By, s, sampled)
|
||||
if c.Model == "" {
|
||||
style += ";border-style:dashed"
|
||||
}
|
||||
// A card wearing out is outlined whatever the map is coloured
|
||||
// by: it is the one thing on the map that needs doing.
|
||||
if len(s.Problems) > 0 {
|
||||
style += ";border:2px solid #c0392b"
|
||||
c.Faulty = true
|
||||
}
|
||||
c.Style, c.Label = template.CSS(style), label
|
||||
row.Cells = append(row.Cells, c)
|
||||
}
|
||||
@@ -216,6 +225,7 @@ func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu
|
||||
type sensorView = struct {
|
||||
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
|
||||
Throttle []string
|
||||
Problems []string
|
||||
}
|
||||
|
||||
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
|
||||
@@ -225,7 +235,8 @@ func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
|
||||
for _, s := range n.Load.GPUs {
|
||||
if s.Index == gpu {
|
||||
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
|
||||
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle}, true
|
||||
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle,
|
||||
Problems: s.Wear.Problems()}, true
|
||||
}
|
||||
}
|
||||
return sensorView{}, false
|
||||
|
||||
@@ -20,7 +20,12 @@ func (mapFleet) Nodes() []adminapi.NodeView {
|
||||
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
|
||||
}
|
||||
return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
|
||||
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), busy(3, 0, 30)}}}}
|
||||
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
|
||||
}
|
||||
|
||||
func worn(s inventory.GPUSensor) inventory.GPUSensor {
|
||||
s.Wear = &inventory.GPUWear{RemapPending: true, XIDs: 1, LastXID: 79, LastXIDText: "GPU has fallen off the bus."}
|
||||
return s
|
||||
}
|
||||
|
||||
func (mapFleet) Replicas(string) []adminapi.ReplicaView {
|
||||
@@ -38,6 +43,8 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"gpu07 GPU 2 · code (code-1) · 93% busy",
|
||||
"gpu07 GPU 3 · free",
|
||||
"memory rows remapped, waiting for a reset · XID 79 (GPU has fallen off the bus.)",
|
||||
"border:2px solid #c0392b",
|
||||
"border-style:dashed",
|
||||
"Cards held and doing nothing",
|
||||
"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
|
||||
|
||||
@@ -55,7 +55,7 @@
|
||||
<div class="gpu-map">
|
||||
{{range .Rows}}<div class="gpu-row"><span class="gpu-node">{{.Node}}</span>{{range .Cells}}<span class="gpu-cell" style="{{.Style}}" title="{{.Title}}">{{.Label}}</span>{{end}}</div>
|
||||
{{end}}</div>
|
||||
<p class="note">{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.</p>
|
||||
<p class="note">{{.Legend}} A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.</p>
|
||||
{{end}}
|
||||
{{with .Efficiency}}
|
||||
<h3>What each model gets from its cards <span class="opt">last 15 minutes</span></h3>
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
{{if .Unhealthy}}<p class="note bad">{{join .Unhealthy}}</p>{{end}}
|
||||
{{with .Load}}{{if .GPUs}}
|
||||
<h2>GPUs</h2>
|
||||
<table class="rows"><tr><th class="num">GPU</th><th>Busy</th><th>Memory used</th><th class="num">Temperature</th><th class="num">Memory temp.</th><th>Power</th><th class="num">Clock</th><th>Held back by</th></tr>
|
||||
<table class="rows"><tr><th class="num">GPU</th><th>Busy</th><th>Memory used</th><th class="num">Temperature</th><th class="num">Memory temp.</th><th>Power</th><th class="num">Clock</th><th>Held back by, and wear</th></tr>
|
||||
{{range .GPUs}}<tr>
|
||||
<td class="num">{{.Index}}</td>
|
||||
{{/* From the last sample; a dash where the driver or an older agent did
|
||||
@@ -46,7 +46,7 @@
|
||||
<td class="num">{{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}}</td>
|
||||
<td class="load">{{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}<span class="load-v">{{printf "%.0f" .PowerW}} W <span class="note">of {{printf "%.0f" .PowerLimitW}}</span></span>{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}</td>
|
||||
<td class="num">{{if .SMClockMHz}}{{printf "%.0f" .SMClockMHz}} MHz{{else}}—{{end}}</td>
|
||||
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors</span>{{end}}</td>
|
||||
<td>{{if .Throttle}}<span class="bad">{{join .Throttle}}</span>{{else}}<span class="note">nothing</span>{{end}}{{if gt .ECCUncorrected 0.0}} <span class="bad">{{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors</span>{{end}}{{with .Wear}}{{with .Problems}}<br><span class="bad">{{join .}}</span>{{end}}{{if gt .PCIeReplays 0}} <span class="note">{{.PCIeReplays}} PCIe retries</span>{{end}}{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}{{end}}
|
||||
{{end}}
|
||||
|
||||
Reference in new issue
Block a user