diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aaef40..c00c4ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,31 @@ APIs change between minors without a deprecation period. From the field report in issue #2, after a deployment on GPU nodes. +### GPU metrics that move, and charts of them + +A GPU's utilisation and memory in use were read once, when the agent +started, and published from then on: a card busy since looked as idle as it +was at registration, and one idle since as busy. They are now sampled with +the temperatures and power every heartbeat (ten seconds at most), with the +memory bandwidth in use, the SM clock and uncorrected ECC errors besides. +An agent too old to send them still has its registration figures published, +and a figure the driver does not give is left out rather than shown as zero. + +- **New series**: `llmgw_gpu_memory_utilization_percent`, + `llmgw_gpu_sm_clock_mhz`; `llmgw_gpu_utilization_percent`, + `llmgw_gpu_memory_used_bytes` and `llmgw_gpu_ecc_errors` now from the + sample. +- **The node page** shows each card's utilisation, memory, clock and ECC + errors in its table, and charts of utilisation, memory, temperature and + power, one line per card. +- **The cluster page** charts the fleet's GPUs by class: average utilisation + and memory in use. +- **Grafana** (`deploy/monitoring/grafana/llmgw-overview.json`) has a GPU row: + utilisation, memory, memory bandwidth, temperature, power against the + limit, clock, and what holds cards back. +- **Alerts**: `GPUThermalThrottling` (ten minutes slowed by heat), + `GPUUncorrectedECCErrors`, `GPUHot` (above 87 °C for ten minutes). + ### A GPU placement no longer runs on the CPU unnoticed llama.cpp placed on a GPU, with its CUDA backend unable to load (cuBLAS not diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 5e9f4fd71..d4f4efb 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1550,9 +1550,19 @@ type GPUSensor struct { PowerLimitW float64 `protobuf:"fixed64,5,opt,name=power_limit_w,json=powerLimitW,proto3" json:"power_limit_w,omitempty"` // throttle names what is holding the clocks down now: sw_power_cap, // hw_slowdown, sw_thermal, hw_thermal, power_brake. - Throttle []string `protobuf:"bytes,6,rep,name=throttle,proto3" json:"throttle,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Throttle []string `protobuf:"bytes,6,rep,name=throttle,proto3" json:"throttle,omitempty"` + // How busy the card is, sampled with the rest, rather than read once at + // registration. -1 is not reported by the driver. memory_total_mb is set + // by every agent that sends these, so zero there is an older agent. + UtilizationPercent float64 `protobuf:"fixed64,7,opt,name=utilization_percent,json=utilizationPercent,proto3" json:"utilization_percent,omitempty"` + MemoryUtilizationPercent float64 `protobuf:"fixed64,8,opt,name=memory_utilization_percent,json=memoryUtilizationPercent,proto3" json:"memory_utilization_percent,omitempty"` + MemoryUsedMb float64 `protobuf:"fixed64,9,opt,name=memory_used_mb,json=memoryUsedMb,proto3" json:"memory_used_mb,omitempty"` + MemoryTotalMb float64 `protobuf:"fixed64,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"` + SmClockMhz float64 `protobuf:"fixed64,11,opt,name=sm_clock_mhz,json=smClockMhz,proto3" json:"sm_clock_mhz,omitempty"` + // ecc_uncorrected counts uncorrected ECC errors since the driver loaded. + EccUncorrected float64 `protobuf:"fixed64,12,opt,name=ecc_uncorrected,json=eccUncorrected,proto3" json:"ecc_uncorrected,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPUSensor) Reset() { @@ -1627,6 +1637,48 @@ func (x *GPUSensor) GetThrottle() []string { return nil } +func (x *GPUSensor) GetUtilizationPercent() float64 { + if x != nil { + return x.UtilizationPercent + } + return 0 +} + +func (x *GPUSensor) GetMemoryUtilizationPercent() float64 { + if x != nil { + return x.MemoryUtilizationPercent + } + return 0 +} + +func (x *GPUSensor) GetMemoryUsedMb() float64 { + if x != nil { + return x.MemoryUsedMb + } + return 0 +} + +func (x *GPUSensor) GetMemoryTotalMb() float64 { + if x != nil { + return x.MemoryTotalMb + } + return 0 +} + +func (x *GPUSensor) GetSmClockMhz() float64 { + if x != nil { + return x.SmClockMhz + } + return 0 +} + +func (x *GPUSensor) GetEccUncorrected() float64 { + if x != nil { + return x.EccUncorrected + } + return 0 +} + // ShardLoad is one engine process group's use. type ShardLoad struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -3999,14 +4051,22 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + " \x01(\x01R\tcpuPowerW\x12 \n" + "\finlet_temp_c\x18\v \x01(\x01R\n" + "inletTempC\x12'\n" + - "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xb5\x01\n" + + "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xbd\x03\n" + "\tGPUSensor\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" + "\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" + "\rmemory_temp_c\x18\x03 \x01(\x01R\vmemoryTempC\x12\x17\n" + "\apower_w\x18\x04 \x01(\x01R\x06powerW\x12\"\n" + "\rpower_limit_w\x18\x05 \x01(\x01R\vpowerLimitW\x12\x1a\n" + - "\bthrottle\x18\x06 \x03(\tR\bthrottle\"m\n" + + "\bthrottle\x18\x06 \x03(\tR\bthrottle\x12/\n" + + "\x13utilization_percent\x18\a \x01(\x01R\x12utilizationPercent\x12<\n" + + "\x1amemory_utilization_percent\x18\b \x01(\x01R\x18memoryUtilizationPercent\x12$\n" + + "\x0ememory_used_mb\x18\t \x01(\x01R\fmemoryUsedMb\x12&\n" + + "\x0fmemory_total_mb\x18\n" + + " \x01(\x01R\rmemoryTotalMb\x12 \n" + + "\fsm_clock_mhz\x18\v \x01(\x01R\n" + + "smClockMhz\x12'\n" + + "\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\"m\n" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 16d4613..6a3a03b 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -304,6 +304,16 @@ message GPUSensor { // throttle names what is holding the clocks down now: sw_power_cap, // hw_slowdown, sw_thermal, hw_thermal, power_brake. repeated string throttle = 6; + // How busy the card is, sampled with the rest, rather than read once at + // registration. -1 is not reported by the driver. memory_total_mb is set + // by every agent that sends these, so zero there is an older agent. + double utilization_percent = 7; + double memory_utilization_percent = 8; + double memory_used_mb = 9; + double memory_total_mb = 10; + double sm_clock_mhz = 11; + // ecc_uncorrected counts uncorrected ECC errors since the driver loaded. + double ecc_uncorrected = 12; } // ShardLoad is one engine process group's use. diff --git a/deploy/monitoring/grafana/llmgw-overview.json b/deploy/monitoring/grafana/llmgw-overview.json index 5b36716..27583df 100644 --- a/deploy/monitoring/grafana/llmgw-overview.json +++ b/deploy/monitoring/grafana/llmgw-overview.json @@ -487,6 +487,298 @@ "defaults": {}, "overrides": [] } + }, + { + "type": "row", + "title": "GPUs", + "id": 14, + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "GPU busy, by card", + "description": "Share of each sample period a kernel was running, sampled every heartbeat.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 31, + "w": 12, + "h": 7 + }, + "id": 15, + "targets": [ + { + "expr": "llmgw_gpu_utilization_percent", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU memory in use, by card", + "description": "Framebuffer in use. Engines reserve most of it at start.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 31, + "w": 12, + "h": 7 + }, + "id": 16, + "targets": [ + { + "expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU memory bandwidth busy, by card", + "description": "Decode is bound by memory: high here with compute low is a card waiting on memory.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 38, + "w": 12, + "h": 7 + }, + "id": 17, + "targets": [ + { + "expr": "llmgw_gpu_memory_utilization_percent", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU temperature", + "description": "Throttling usually starts in the high 80s.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 38, + "w": 12, + "h": 7 + }, + "id": 18, + "targets": [ + { + "expr": "llmgw_gpu_temperature_celsius", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU power against its limit", + "description": "1 is the enforced limit; a card held there is power-capped.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 45, + "w": 12, + "h": 7 + }, + "id": 19, + "targets": [ + { + "expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1.1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU clock", + "description": "A clock held low under load is the card throttled; see the next panel for why.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 45, + "w": 12, + "h": 7 + }, + "id": 20, + "targets": [ + { + "expr": "llmgw_gpu_sm_clock_mhz", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "rotmhz" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "What holds GPUs back", + "description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 52, + "w": 24, + "h": 7 + }, + "id": 21, + "targets": [ + { + "expr": "sum by (reason) (llmgw_gpu_throttled)", + "legendFormat": "throttled: {{reason}}", + "refId": "A" + }, + { + "expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)", + "legendFormat": "ECC errors {{node}} gpu {{gpu}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } } ] } \ No newline at end of file diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index 4393b0e..16bf5af 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -200,6 +200,40 @@ groups: is skipped by placement rather than failing, so this is quiet until somebody wonders why a model will not land there. + - name: llmgw-gpus + rules: + - alert: GPUThermalThrottling + expr: llmgw_gpu_throttled{reason=~"sw_thermal|hw_thermal|hw_slowdown"} == 1 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} is slowed by heat ({{ $labels.reason }})" + description: >- + Ten minutes held back by temperature: every replica on the card is + slower than its measurements say. Usually airflow, a fan, or the + room — the node page shows the card's temperature over time. + + - alert: GPUUncorrectedECCErrors + expr: llmgw_gpu_ecc_errors > 0 + for: 1m + labels: { severity: critical } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has {{ $value }} uncorrected ECC errors" + description: >- + Uncorrected memory errors since the driver loaded: what runs on the + card may be computing on corrupted memory. Drain the node + (llmgw ctl drain) and have the card looked at. + + - alert: GPUHot + expr: llmgw_gpu_temperature_celsius > 87 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} at {{ $value }} °C" + description: >- + Ten minutes above 87 °C, where most data-centre cards begin to + throttle. + - name: monitoring-host rules: - alert: MonitoringDiskFilling diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 3cb8ea2..95f39fc 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -660,6 +660,9 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad { out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{ Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC, PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle, + UtilizationPercent: g.UtilPercent, MemoryUtilizationPercent: g.MemUtilPercent, + MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB, + SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected, }) } } diff --git a/internal/agent/discovery/sensors.go b/internal/agent/discovery/sensors.go index d415e31..d34dde9 100644 --- a/internal/agent/discovery/sensors.go +++ b/internal/agent/discovery/sensors.go @@ -41,6 +41,14 @@ type GPUSensor struct { // is running free. Idle and application clock settings are left out; // they are not the card being held back. Throttle []string + // How busy it is, read with the rest every sample. Discovery reads the + // same once, at start, which made a busy card look as idle as it was + // when the agent came up. -1 is not reported by the driver; zero + // MemoryTotalMB is a sample without these at all. + UtilPercent, MemUtilPercent float64 + MemoryUsedMB, MemoryTotalMB float64 + SMClockMHz float64 + ECCUncorrected float64 } // sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often @@ -187,13 +195,33 @@ var throttleBits = []struct { // gpus asks nvidia-smi for each card's heat and power. The memory temperature // is not known to every driver, and a field nvidia-smi does not know fails the // whole query, so it is asked for first and dropped on failure. +// gpuFields are what each sample asks nvidia-smi for, in order. The busy +// figures follow the throttle reasons; the memory temperature is last, being +// the one field a driver may reject outright. +const ( + gpuBase = "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active" + gpuBusy = ",utilization.gpu,utilization.memory,memory.used,memory.total,clocks.sm,ecc.errors.uncorrected.volatile.total" + gpuHBMTC = ",temperature.memory" +) + func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor { - withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory" - out, err := s.smi(ctx, withMem) - if err != nil { - if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil { - return nil + // Most fields first, fewer on a driver that refuses one: a query naming + // a field the driver does not know fails as a whole. + var out []byte + var busy bool + for _, q := range []struct { + fields string + busy bool + }{{gpuBase + gpuBusy + gpuHBMTC, true}, {gpuBase + gpuBusy, true}, {gpuBase + gpuHBMTC, false}, {gpuBase, false}} { + var err error + if out, err = s.smi(ctx, q.fields); err == nil { + busy = q.busy + break } + out = nil + } + if out == nil { + return nil } var gs []GPUSensor for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { @@ -216,14 +244,31 @@ func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor { } } } - if len(f) > 5 { - g.MemoryTempC = num(f[5]) + rest := f[5:] + if busy && len(rest) >= 6 { + g.UtilPercent, g.MemUtilPercent = numOrUnknown(rest[0]), numOrUnknown(rest[1]) + g.MemoryUsedMB, g.MemoryTotalMB = num(rest[2]), num(rest[3]) + g.SMClockMHz, g.ECCUncorrected = num(rest[4]), numOrUnknown(rest[5]) + rest = rest[6:] + } + if len(rest) > 0 { + g.MemoryTempC = num(rest[0]) } gs = append(gs, g) } return gs } +// numOrUnknown is a figure, or -1 where nvidia-smi says it has none: a card +// without ECC has no error count, which is not the same as none. +func numOrUnknown(s string) float64 { + v, err := strconv.ParseFloat(s, 64) + if err != nil || v < 0 { + return -1 + } + return v +} + // num reads a figure nvidia-smi may give as "[N/A]" or "N/A". func num(s string) float64 { v, err := strconv.ParseFloat(s, 64) diff --git a/internal/agent/discovery/sensors_test.go b/internal/agent/discovery/sensors_test.go index ce1507e..dc9ae07 100644 --- a/internal/agent/discovery/sensors_test.go +++ b/internal/agent/discovery/sensors_test.go @@ -167,3 +167,44 @@ func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T t.Errorf("%+v", got.GPUs) } } + +func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) { + // What the card is doing now, every sample: discovery read it once, at + // start, and a card busy since looked as idle as it was then. + smi := func(_ context.Context, fields string) ([]byte, error) { + if !strings.Contains(fields, "utilization.gpu") { + return nil, errors.New("asked without how busy the cards are") + } + return []byte( + "0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" + + "1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil + } + now := time.Now() + got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs + if len(got) != 2 { + t.Fatalf("%+v", got) + } + g0, g1 := got[0], got[1] + if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 || + g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 { + t.Errorf("card 0 %+v", g0) + } + // Idle is zero; a count the card does not keep is unknown, not zero. + if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 { + t.Errorf("card 1 %+v", g1) + } +} + +func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) { + smi := func(_ context.Context, fields string) ([]byte, error) { + if strings.Contains(fields, "utilization.gpu") { + return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`) + } + return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil + } + now := time.Now() + got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs + if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 { + t.Errorf("%+v", got) + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index c2951ed..0ca6387 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -585,6 +585,9 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { out.GPUs = append(out.GPUs, inventory.GPUSensor{ Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(), PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(), + UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(), + MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(), + SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(), }) } return out diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 81f5923..09144e4 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -73,6 +73,8 @@ type Collector struct { gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc + gpuMemUtil *prometheus.Desc + gpuSMClock *prometheus.Desc gpuTemp *prometheus.Desc gpuMemTemp *prometheus.Desc gpuPower *prometheus.Desc @@ -149,6 +151,11 @@ func New(src Source) *Collector { "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), + gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent", + "How much of the sample period a GPU's memory was being read or written: "+ + "decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil), + gpuSMClock: prometheus.NewDesc("llmgw_gpu_sm_clock_mhz", + "A GPU's streaming multiprocessor clock now; one held low under load is the card being throttled.", gpu, nil), gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius", "A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil), gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts", @@ -271,7 +278,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, - c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, + c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy, c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -388,20 +395,28 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { byClass[class]++ labels := []string{n.ID, strconv.Itoa(g.Index), class} - ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, - float64(g.MemoryTotalMB)*1024*1024, labels...) - ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, - float64(g.MemoryUsedMB)*1024*1024, labels...) - if g.UtilizationPercent > 0 { - // Only when reported. A driver too old to give it would - // otherwise show every card at a flat zero, which reads as an - // idle fleet rather than a missing measurement. - ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, - float64(g.UtilizationPercent), labels...) + s, fresh := sensorFor(n.Load, g.Index, now) + if fresh && s.Busy() { + // What the card is doing now, sampled every heartbeat. + c.gpuBusy(ch, s, labels) + } else { + // An agent too old to sample it: what it read at + // registration, which is all there is. + ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, + float64(g.MemoryTotalMB)*1024*1024, labels...) + ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, + float64(g.MemoryUsedMB)*1024*1024, labels...) + if g.UtilizationPercent > 0 { + // Only when reported. A driver too old to give it would + // otherwise show every card at a flat zero, which reads as + // an idle fleet rather than a missing measurement. + ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, + float64(g.UtilizationPercent), labels...) + } } // The sensors' fresh reading when there is one; the temperature // read at registration otherwise, which is all an older agent sends. - if s, ok := sensorFor(n.Load, g.Index, now); ok { + if fresh { c.gpuSensor(ch, s, labels) } else if g.Health.TemperatureC > 0 { ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue, @@ -414,8 +429,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...) ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue, float64(len(g.NVLinkPeers)), labels...) - ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue, - float64(g.Health.ECCErrors), labels...) + ecc := float64(g.Health.ECCErrors) + if fresh && s.Busy() && s.ECCUncorrected >= 0 { + ecc = s.ECCUncorrected + } + ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue, ecc, labels...) } for class, n2 := range byClass { ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue, @@ -569,6 +587,23 @@ func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor return inventory.GPUSensor{}, false } +// gpuBusy is how busy a card is, from a sample that carries it. +func (c *Collector) gpuBusy(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) { + ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, s.MemoryTotalMB*1024*1024, labels...) + ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, s.MemoryUsedMB*1024*1024, labels...) + // Zero is a real reading here — an idle card — and only -1, the driver + // not saying, leaves the series out. + if s.UtilPercent >= 0 { + ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, s.UtilPercent, labels...) + } + if s.MemUtilPercent >= 0 { + ch <- prometheus.MustNewConstMetric(c.gpuMemUtil, prometheus.GaugeValue, s.MemUtilPercent, labels...) + } + if s.SMClockMHz > 0 { + ch <- prometheus.MustNewConstMetric(c.gpuSMClock, prometheus.GaugeValue, s.SMClockMHz, labels...) + } +} + func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) { for _, v := range []struct { d *prometheus.Desc diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go index cd73f6b..3e5358a 100644 --- a/internal/control/fleetmetrics/sensors_test.go +++ b/internal/control/fleetmetrics/sensors_test.go @@ -39,3 +39,41 @@ func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) { } } } + +func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) { + // Discovery read 87% and 70 GB used when the agent came up; the card has + // been idle since. The sample says so, zero included. + reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, + MemoryUsedMB: 70000, UtilizationPercent: 87} + n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg}, + Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35, + UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559, + SMClockMHz: 345, ECCUncorrected: 2}}}} + out := gather(t, fakeSource{nodes: []inventory.Node{n}}) + for _, want := range []string{ + "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n", + "llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n", + "llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n", + "llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n", + "llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s in:\n%s", want, out) + } + } + if strings.Contains(out, "-> 87\n") { + t.Error("the utilization read at registration was published over the sample's") + } + + // An agent too old to sample it: registration's figures, and no zero + // standing in for a utilization the driver did not give. + n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}} + out = gather(t, fakeSource{nodes: []inventory.Node{n}}) + if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") { + t.Errorf("an older agent's registration figure was dropped:\n%s", out) + } + n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}} + if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") { + t.Errorf("a utilization the driver did not report was published:\n%s", out) + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 4e9df64..04356a6 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -319,8 +319,20 @@ type GPUSensor struct { PowerW float64 `json:"power_w,omitempty"` PowerLimitW float64 `json:"power_limit_w,omitempty"` Throttle []string `json:"throttle,omitempty"` + // How busy the card is, as of this sample; -1 is not reported. Zero + // MemoryTotalMB is an agent too old to send these, whose figures are + // the ones it read at registration. + UtilPercent float64 `json:"util_percent,omitempty"` + MemUtilPercent float64 `json:"mem_util_percent,omitempty"` + MemoryUsedMB float64 `json:"memory_used_mb,omitempty"` + MemoryTotalMB float64 `json:"memory_total_mb,omitempty"` + SMClockMHz float64 `json:"sm_clock_mhz,omitempty"` + ECCUncorrected float64 `json:"ecc_uncorrected,omitempty"` } +// Busy reports whether the sample carries how busy the card is. +func (s GPUSensor) Busy() bool { return s.MemoryTotalMB > 0 } + // ShardLoad is one engine's use: cores busy, and resident memory. type ShardLoad struct { CPUCores float64 diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 536e184..ba0f7cc 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -201,6 +201,9 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat for _, m := range d.Models { d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0 } + if d.AnyGPUs { + d.Fleet.LoadGPU(r.Context(), nav.History, nav.Window) + } d.Summary = src.Summary() if fs, ok := src.(FabricSource); ok { r := fs.Fabrics() diff --git a/internal/web/dashcharts.go b/internal/web/dashcharts.go index 5821e63..a6e83dd 100644 --- a/internal/web/dashcharts.go +++ b/internal/web/dashcharts.go @@ -24,7 +24,10 @@ type FleetCharts struct { Queue TimeSeries KV TimeSeries Ready TimeSeries - Window string + // GPUBusy and GPUMemory are the fleet's cards by class, for a fleet that + // has them. + GPUBusy, GPUMemory TimeSeries + Window string // Err is set when the metrics store could not be read. Shown as a line of // text where the charts would be: an empty panel reads as a healthy cluster // with nothing happening, which is the opposite of not knowing. @@ -90,3 +93,25 @@ func LoadFleetCharts(ctx context.Context, h History, window time.Duration, } return out } + +// LoadGPU adds the fleet's GPUs, by class: how busy on average, and how full +// their memory. Asked only of a fleet with GPUs. A failure leaves these two +// empty and the rest of the page as it was; they are not what the other +// panels' all-or-nothing is about. +func (f *FleetCharts) LoadGPU(ctx context.Context, h History, window time.Duration) { + if f == nil || h == nil || f.Err != "" { + return + } + if window <= 0 { + window = time.Hour + } + step := window / 60 + if s, err := h.RangeQuery(ctx, `avg by (class) (llmgw_gpu_utilization_percent) / 100`, "class", window, step); err == nil { + f.GPUBusy = LineChart(s, "%", 6) + } + if s, err := h.RangeQuery(ctx, + `sum by (class) (llmgw_gpu_memory_used_bytes) / sum by (class) (llmgw_gpu_memory_total_bytes)`, + "class", window, step); err == nil { + f.GPUMemory = LineChart(s, "%", 6) + } +} diff --git a/internal/web/node_page.go b/internal/web/node_page.go index 59f8303..35cabd8 100644 --- a/internal/web/node_page.go +++ b/internal/web/node_page.go @@ -40,8 +40,10 @@ type nodeEngine struct { // nodeCharts are the node's recent history, from the metrics store. type nodeCharts struct { CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries - Window string - Err string + // The GPUs', one line per card; empty on a node without them. + GPUUtil, GPUMemory, GPUTemp, GPUPower TimeSeries + Window string + Err string } // nodePanels are the charts, each a query scoped to this node. %s is the @@ -63,6 +65,23 @@ var nodePanels = []struct { func(c *nodeCharts, t TimeSeries) { c.EngineMemory = t }}, } +// gpuPanels are the GPUs' charts, one line per card, for a node that has +// them. Read from the heartbeat's samples, not from what the node said when +// it registered. +var gpuPanels = []struct { + expr, unit string + into func(*nodeCharts, TimeSeries) +}{ + {`llmgw_gpu_utilization_percent{node=%s} / 100`, "%", + func(c *nodeCharts, t TimeSeries) { c.GPUUtil = t }}, + {`llmgw_gpu_memory_used_bytes{node=%s}`, "bytes", + func(c *nodeCharts, t TimeSeries) { c.GPUMemory = t }}, + {`llmgw_gpu_temperature_celsius{node=%s}`, "", + func(c *nodeCharts, t TimeSeries) { c.GPUTemp = t }}, + {`llmgw_gpu_power_watts{node=%s}`, "", + func(c *nodeCharts, t TimeSeries) { c.GPUPower = t }}, +} + // NodePage shows one node. Its charts come from the metrics store when one // is configured; the current figures come from the node's last heartbeat // either way. @@ -115,14 +134,14 @@ func NodePage(src Source, history History, window time.Duration, onAdminListener sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores }) if history != nil { - d.Charts = loadNodeCharts(r.Context(), history, id, replicas, window, log) + d.Charts = loadNodeCharts(r.Context(), history, id, replicas, d.Node.GPUTotal, window, log) } noStore(w) render(w, "node", d) } } -func loadNodeCharts(ctx context.Context, h History, node string, replicas []string, +func loadNodeCharts(ctx context.Context, h History, node string, replicas []string, gpus int, window time.Duration, log *slog.Logger, ) *nodeCharts { step := window / 60 @@ -139,6 +158,19 @@ func loadNodeCharts(ctx context.Context, h History, node string, replicas []stri } p.into(out, LineChart(series, p.unit, 4)) } + // Every card its own line, up to the eight a node commonly has. + if gpus > 0 { + for _, p := range gpuPanels { + series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), "gpu", window, step) + if err != nil { + continue + } + for i := range series { + series[i].Name = "GPU " + series[i].Name + } + p.into(out, LineChart(series, p.unit, max(gpus, 8))) + } + } // The engines' queues, for the replicas that are here now. if len(replicas) > 0 { re := make([]string, len(replicas)) diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index 6e91dbd..f947477 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -205,3 +205,68 @@ func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) { } } } + +// gpuSource is a node with two GPUs, one busy and one idle. +type gpuSource struct{ fakeSource } + +func (gpuSource) Nodes() []adminapi.NodeView { + n := fakeSource{}.Nodes()[0] + n.GPUTotal = 2 + n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39, + GPUs: []inventory.GPUSensor{ + {Index: 0, TempC: 71, PowerW: 612, PowerLimitW: 700, UtilPercent: 97, MemUtilPercent: 64, + MemoryUsedMB: 72909, MemoryTotalMB: 81559, SMClockMHz: 1980, ECCUncorrected: 0}, + {Index: 1, TempC: 30, PowerW: 70, PowerLimitW: 700, UtilPercent: 0, MemUtilPercent: 0, + MemoryUsedMB: 512, MemoryTotalMB: 81559, SMClockMHz: 345, ECCUncorrected: 3}, + }} + return []adminapi.NodeView{n} +} + +func TestAGPUNodesPageShowsHowBusyEachCardIsAndOverTime(t *testing.T) { + q := func(expr string) string { return strings.Replace(expr, "%s", `"llmgw-node01"`, 1) } + h := &fakeHistory{byExpr: map[string][]promq.Series{ + q(`llmgw_gpu_utilization_percent{node=%s} / 100`): {ts("0", 0.9, 0.97), ts("1", 0, 0)}, + q(`llmgw_gpu_memory_used_bytes{node=%s}`): {ts("0", 7e10, 7.6e10)}, + q(`llmgw_gpu_temperature_celsius{node=%s}`): {ts("0", 65, 71)}, + q(`llmgw_gpu_power_watts{node=%s}`): {ts("0", 500, 612)}, + }} + _, body := get(t, nodeMux(gpuSource{}, h), "http://x/admin/nodes/llmgw-node01", nil) + for _, want := range []string{ + "97%", "memory 64%", "71.2 of 80 GB", "1980 MHz", + "3 uncorrected ECC errors", + "GPU busy, by card", "GPU memory used, by card", "GPU temperature, °C", "GPU power, W", "GPU 0", + } { + if !strings.Contains(body, want) { + t.Errorf("the page lacks %q", want) + } + } + // A node without GPUs is not asked about them. + h2 := &fakeHistory{} + get(t, nodeMux(cpuFleet{}, h2), "http://x/admin/nodes/llmgw-node01", nil) + for _, e := range h2.asked { + if strings.Contains(e, "llmgw_gpu_") { + t.Errorf("a node with no GPU was asked for %s", e) + } + } +} + +func TestTheClusterPageDrawsItsGPUsByClass(t *testing.T) { + h := &fakeHistory{byExpr: map[string][]promq.Series{ + `avg by (class) (llmgw_gpu_utilization_percent) / 100`: {ts("h100-80g", 0.2, 0.6)}, + `sum by (class) (llmgw_gpu_memory_used_bytes) / sum by (class) (llmgw_gpu_memory_total_bytes)`: {ts("h100-80g", 0.8, 0.9)}, + }} + _, body := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{History: h, Window: time.Hour}, nil), "http://127.0.0.1:9090/", nil) + if !strings.Contains(body, "GPUs busy, by class") || !strings.Contains(body, "GPU memory in use, by class") { + t.Error("a fleet with GPUs has no GPU charts") + } + h2 := &fakeHistory{} + _, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{History: h2, Window: time.Hour}, nil), "http://127.0.0.1:9090/", nil) + if strings.Contains(cpu, "GPUs busy") { + t.Error("a fleet without GPUs was drawn GPU charts") + } + for _, e := range h2.asked { + if strings.Contains(e, "llmgw_gpu_") { + t.Errorf("a fleet without GPUs was asked for %s", e) + } + } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 195ca69..20e6742 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -213,6 +213,14 @@ unused nodes, which is what decides whether such a replica can land.
{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model" "note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}} +{{if $.AnyGPUs}} +{{join .Unhealthy}}
{{end}} {{with .Load}}{{if .GPUs}}| GPU | Temperature | Memory | Power | Held back by |
|---|
| GPU | Busy | Memory used | Temperature | Memory temp. | Power | Clock | Held back by | |
|---|---|---|---|---|---|---|---|---|
| {{.Index}} | + {{/* From the last sample; a dash where the driver or an older agent did + not say, rather than a zero nobody measured. */}} +{{if and .MemoryTotalMB (ge .UtilPercent 0.0)}}{{template "minibar" pctbar .UtilPercent}}{{printf "%.0f" .UtilPercent}}%{{if ge .MemUtilPercent 0.0}} memory {{printf "%.0f" .MemUtilPercent}}%{{end}}{{else}}—{{end}} | +{{if .MemoryTotalMB}}{{template "minibar" pctbar (percent .MemoryUsedMB .MemoryTotalMB)}}{{printf "%.1f" (gib .MemoryUsedMB)}} of {{printf "%.0f" (gib .MemoryTotalMB)}} GB{{else}}—{{end}} | {{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}} | {{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}} | {{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}{{printf "%.0f" .PowerW}} W of {{printf "%.0f" .PowerLimitW}}{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}} | -{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}} | +{{if .SMClockMHz}}{{printf "%.0f" .SMClockMHz}} MHz{{else}}—{{end}} | +{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if gt .ECCUncorrected 0.0}} {{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors{{end}} |