From bd36798b15f831137e4a8ce962651fc707c9e016 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 15:42:17 +0200 Subject: [PATCH] Sample GPU utilisation and memory every heartbeat, and chart them They were read once, at agent start, and published unchanged since. Each sample now carries utilisation, memory and bandwidth in use, the SM clock and uncorrected ECC errors. The node page charts each card, the cluster page the fleet by class, Grafana has a GPU row, and three GPU alerts are added. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 25 ++ api/gen/llmgw/v1/agent.pb.go | 70 ++++- api/proto/llmgw/v1/agent.proto | 10 + deploy/monitoring/grafana/llmgw-overview.json | 292 ++++++++++++++++++ deploy/monitoring/llmgw-rules.yml | 34 ++ internal/agent/agent.go | 3 + internal/agent/discovery/sensors.go | 59 +++- internal/agent/discovery/sensors_test.go | 41 +++ internal/control/agentsvc/service.go | 3 + internal/control/fleetmetrics/fleetmetrics.go | 63 +++- internal/control/fleetmetrics/sensors_test.go | 38 +++ internal/control/inventory/inventory.go | 12 + internal/web/dashboard.go | 3 + internal/web/dashcharts.go | 27 +- internal/web/node_page.go | 40 ++- internal/web/node_page_test.go | 65 ++++ internal/web/templates/dashboard.html | 8 + internal/web/templates/node.html | 19 +- internal/web/web.go | 2 + 19 files changed, 781 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aaef40..c00c4ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,31 @@ APIs change between minors without a deprecation period. From the field report in issue #2, after a deployment on GPU nodes. +### GPU metrics that move, and charts of them + +A GPU's utilisation and memory in use were read once, when the agent +started, and published from then on: a card busy since looked as idle as it +was at registration, and one idle since as busy. They are now sampled with +the temperatures and power every heartbeat (ten seconds at most), with the +memory bandwidth in use, the SM clock and uncorrected ECC errors besides. +An agent too old to send them still has its registration figures published, +and a figure the driver does not give is left out rather than shown as zero. + +- **New series**: `llmgw_gpu_memory_utilization_percent`, + `llmgw_gpu_sm_clock_mhz`; `llmgw_gpu_utilization_percent`, + `llmgw_gpu_memory_used_bytes` and `llmgw_gpu_ecc_errors` now from the + sample. +- **The node page** shows each card's utilisation, memory, clock and ECC + errors in its table, and charts of utilisation, memory, temperature and + power, one line per card. +- **The cluster page** charts the fleet's GPUs by class: average utilisation + and memory in use. +- **Grafana** (`deploy/monitoring/grafana/llmgw-overview.json`) has a GPU row: + utilisation, memory, memory bandwidth, temperature, power against the + limit, clock, and what holds cards back. +- **Alerts**: `GPUThermalThrottling` (ten minutes slowed by heat), + `GPUUncorrectedECCErrors`, `GPUHot` (above 87 °C for ten minutes). + ### A GPU placement no longer runs on the CPU unnoticed llama.cpp placed on a GPU, with its CUDA backend unable to load (cuBLAS not diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 5e9f4fd71..d4f4efb 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1550,9 +1550,19 @@ type GPUSensor struct { PowerLimitW float64 `protobuf:"fixed64,5,opt,name=power_limit_w,json=powerLimitW,proto3" json:"power_limit_w,omitempty"` // throttle names what is holding the clocks down now: sw_power_cap, // hw_slowdown, sw_thermal, hw_thermal, power_brake. - Throttle []string `protobuf:"bytes,6,rep,name=throttle,proto3" json:"throttle,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Throttle []string `protobuf:"bytes,6,rep,name=throttle,proto3" json:"throttle,omitempty"` + // How busy the card is, sampled with the rest, rather than read once at + // registration. -1 is not reported by the driver. memory_total_mb is set + // by every agent that sends these, so zero there is an older agent. + UtilizationPercent float64 `protobuf:"fixed64,7,opt,name=utilization_percent,json=utilizationPercent,proto3" json:"utilization_percent,omitempty"` + MemoryUtilizationPercent float64 `protobuf:"fixed64,8,opt,name=memory_utilization_percent,json=memoryUtilizationPercent,proto3" json:"memory_utilization_percent,omitempty"` + MemoryUsedMb float64 `protobuf:"fixed64,9,opt,name=memory_used_mb,json=memoryUsedMb,proto3" json:"memory_used_mb,omitempty"` + MemoryTotalMb float64 `protobuf:"fixed64,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"` + SmClockMhz float64 `protobuf:"fixed64,11,opt,name=sm_clock_mhz,json=smClockMhz,proto3" json:"sm_clock_mhz,omitempty"` + // ecc_uncorrected counts uncorrected ECC errors since the driver loaded. + EccUncorrected float64 `protobuf:"fixed64,12,opt,name=ecc_uncorrected,json=eccUncorrected,proto3" json:"ecc_uncorrected,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPUSensor) Reset() { @@ -1627,6 +1637,48 @@ func (x *GPUSensor) GetThrottle() []string { return nil } +func (x *GPUSensor) GetUtilizationPercent() float64 { + if x != nil { + return x.UtilizationPercent + } + return 0 +} + +func (x *GPUSensor) GetMemoryUtilizationPercent() float64 { + if x != nil { + return x.MemoryUtilizationPercent + } + return 0 +} + +func (x *GPUSensor) GetMemoryUsedMb() float64 { + if x != nil { + return x.MemoryUsedMb + } + return 0 +} + +func (x *GPUSensor) GetMemoryTotalMb() float64 { + if x != nil { + return x.MemoryTotalMb + } + return 0 +} + +func (x *GPUSensor) GetSmClockMhz() float64 { + if x != nil { + return x.SmClockMhz + } + return 0 +} + +func (x *GPUSensor) GetEccUncorrected() float64 { + if x != nil { + return x.EccUncorrected + } + return 0 +} + // ShardLoad is one engine process group's use. type ShardLoad struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -3999,14 +4051,22 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + " \x01(\x01R\tcpuPowerW\x12 \n" + "\finlet_temp_c\x18\v \x01(\x01R\n" + "inletTempC\x12'\n" + - "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xb5\x01\n" + + "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xbd\x03\n" + "\tGPUSensor\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" + "\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" + "\rmemory_temp_c\x18\x03 \x01(\x01R\vmemoryTempC\x12\x17\n" + "\apower_w\x18\x04 \x01(\x01R\x06powerW\x12\"\n" + "\rpower_limit_w\x18\x05 \x01(\x01R\vpowerLimitW\x12\x1a\n" + - "\bthrottle\x18\x06 \x03(\tR\bthrottle\"m\n" + + "\bthrottle\x18\x06 \x03(\tR\bthrottle\x12/\n" + + "\x13utilization_percent\x18\a \x01(\x01R\x12utilizationPercent\x12<\n" + + "\x1amemory_utilization_percent\x18\b \x01(\x01R\x18memoryUtilizationPercent\x12$\n" + + "\x0ememory_used_mb\x18\t \x01(\x01R\fmemoryUsedMb\x12&\n" + + "\x0fmemory_total_mb\x18\n" + + " \x01(\x01R\rmemoryTotalMb\x12 \n" + + "\fsm_clock_mhz\x18\v \x01(\x01R\n" + + "smClockMhz\x12'\n" + + "\x0fecc_uncorrected\x18\f \x01(\x01R\x0eeccUncorrected\"m\n" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 16d4613..6a3a03b 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -304,6 +304,16 @@ message GPUSensor { // throttle names what is holding the clocks down now: sw_power_cap, // hw_slowdown, sw_thermal, hw_thermal, power_brake. repeated string throttle = 6; + // How busy the card is, sampled with the rest, rather than read once at + // registration. -1 is not reported by the driver. memory_total_mb is set + // by every agent that sends these, so zero there is an older agent. + double utilization_percent = 7; + double memory_utilization_percent = 8; + double memory_used_mb = 9; + double memory_total_mb = 10; + double sm_clock_mhz = 11; + // ecc_uncorrected counts uncorrected ECC errors since the driver loaded. + double ecc_uncorrected = 12; } // ShardLoad is one engine process group's use. diff --git a/deploy/monitoring/grafana/llmgw-overview.json b/deploy/monitoring/grafana/llmgw-overview.json index 5b36716..27583df 100644 --- a/deploy/monitoring/grafana/llmgw-overview.json +++ b/deploy/monitoring/grafana/llmgw-overview.json @@ -487,6 +487,298 @@ "defaults": {}, "overrides": [] } + }, + { + "type": "row", + "title": "GPUs", + "id": 14, + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 1 + }, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "GPU busy, by card", + "description": "Share of each sample period a kernel was running, sampled every heartbeat.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 31, + "w": 12, + "h": 7 + }, + "id": 15, + "targets": [ + { + "expr": "llmgw_gpu_utilization_percent", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU memory in use, by card", + "description": "Framebuffer in use. Engines reserve most of it at start.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 31, + "w": 12, + "h": 7 + }, + "id": 16, + "targets": [ + { + "expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU memory bandwidth busy, by card", + "description": "Decode is bound by memory: high here with compute low is a card waiting on memory.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 38, + "w": 12, + "h": 7 + }, + "id": 17, + "targets": [ + { + "expr": "llmgw_gpu_memory_utilization_percent", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU temperature", + "description": "Throttling usually starts in the high 80s.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 38, + "w": 12, + "h": 7 + }, + "id": 18, + "targets": [ + { + "expr": "llmgw_gpu_temperature_celsius", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU power against its limit", + "description": "1 is the enforced limit; a card held there is power-capped.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 45, + "w": 12, + "h": 7 + }, + "id": 19, + "targets": [ + { + "expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1.1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "GPU clock", + "description": "A clock held low under load is the card throttled; see the next panel for why.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 45, + "w": 12, + "h": 7 + }, + "id": 20, + "targets": [ + { + "expr": "llmgw_gpu_sm_clock_mhz", + "legendFormat": "{{node}} gpu {{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "rotmhz" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "What holds GPUs back", + "description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 52, + "w": 24, + "h": 7 + }, + "id": 21, + "targets": [ + { + "expr": "sum by (reason) (llmgw_gpu_throttled)", + "legendFormat": "throttled: {{reason}}", + "refId": "A" + }, + { + "expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)", + "legendFormat": "ECC errors {{node}} gpu {{gpu}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } } ] } \ No newline at end of file diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index 4393b0e..16bf5af 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -200,6 +200,40 @@ groups: is skipped by placement rather than failing, so this is quiet until somebody wonders why a model will not land there. + - name: llmgw-gpus + rules: + - alert: GPUThermalThrottling + expr: llmgw_gpu_throttled{reason=~"sw_thermal|hw_thermal|hw_slowdown"} == 1 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} is slowed by heat ({{ $labels.reason }})" + description: >- + Ten minutes held back by temperature: every replica on the card is + slower than its measurements say. Usually airflow, a fan, or the + room — the node page shows the card's temperature over time. + + - alert: GPUUncorrectedECCErrors + expr: llmgw_gpu_ecc_errors > 0 + for: 1m + labels: { severity: critical } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has {{ $value }} uncorrected ECC errors" + description: >- + Uncorrected memory errors since the driver loaded: what runs on the + card may be computing on corrupted memory. Drain the node + (llmgw ctl drain) and have the card looked at. + + - alert: GPUHot + expr: llmgw_gpu_temperature_celsius > 87 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $labels.node }} GPU {{ $labels.gpu }} at {{ $value }} °C" + description: >- + Ten minutes above 87 °C, where most data-centre cards begin to + throttle. + - name: monitoring-host rules: - alert: MonitoringDiskFilling diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 3cb8ea2..95f39fc 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -660,6 +660,9 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad { out.Gpus = append(out.Gpus, &llmgwv1.GPUSensor{ Index: int32(g.Index), TempC: g.TempC, MemoryTempC: g.MemoryTempC, PowerW: g.PowerW, PowerLimitW: g.PowerLimitW, Throttle: g.Throttle, + UtilizationPercent: g.UtilPercent, MemoryUtilizationPercent: g.MemUtilPercent, + MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB, + SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected, }) } } diff --git a/internal/agent/discovery/sensors.go b/internal/agent/discovery/sensors.go index d415e31..d34dde9 100644 --- a/internal/agent/discovery/sensors.go +++ b/internal/agent/discovery/sensors.go @@ -41,6 +41,14 @@ type GPUSensor struct { // is running free. Idle and application clock settings are left out; // they are not the card being held back. Throttle []string + // How busy it is, read with the rest every sample. Discovery reads the + // same once, at start, which made a busy card look as idle as it was + // when the agent came up. -1 is not reported by the driver; zero + // MemoryTotalMB is a sample without these at all. + UtilPercent, MemUtilPercent float64 + MemoryUsedMB, MemoryTotalMB float64 + SMClockMHz float64 + ECCUncorrected float64 } // sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often @@ -187,13 +195,33 @@ var throttleBits = []struct { // gpus asks nvidia-smi for each card's heat and power. The memory temperature // is not known to every driver, and a field nvidia-smi does not know fails the // whole query, so it is asked for first and dropped on failure. +// gpuFields are what each sample asks nvidia-smi for, in order. The busy +// figures follow the throttle reasons; the memory temperature is last, being +// the one field a driver may reject outright. +const ( + gpuBase = "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active" + gpuBusy = ",utilization.gpu,utilization.memory,memory.used,memory.total,clocks.sm,ecc.errors.uncorrected.volatile.total" + gpuHBMTC = ",temperature.memory" +) + func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor { - withMem := "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active,temperature.memory" - out, err := s.smi(ctx, withMem) - if err != nil { - if out, err = s.smi(ctx, strings.TrimSuffix(withMem, ",temperature.memory")); err != nil { - return nil + // Most fields first, fewer on a driver that refuses one: a query naming + // a field the driver does not know fails as a whole. + var out []byte + var busy bool + for _, q := range []struct { + fields string + busy bool + }{{gpuBase + gpuBusy + gpuHBMTC, true}, {gpuBase + gpuBusy, true}, {gpuBase + gpuHBMTC, false}, {gpuBase, false}} { + var err error + if out, err = s.smi(ctx, q.fields); err == nil { + busy = q.busy + break } + out = nil + } + if out == nil { + return nil } var gs []GPUSensor for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { @@ -216,14 +244,31 @@ func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor { } } } - if len(f) > 5 { - g.MemoryTempC = num(f[5]) + rest := f[5:] + if busy && len(rest) >= 6 { + g.UtilPercent, g.MemUtilPercent = numOrUnknown(rest[0]), numOrUnknown(rest[1]) + g.MemoryUsedMB, g.MemoryTotalMB = num(rest[2]), num(rest[3]) + g.SMClockMHz, g.ECCUncorrected = num(rest[4]), numOrUnknown(rest[5]) + rest = rest[6:] + } + if len(rest) > 0 { + g.MemoryTempC = num(rest[0]) } gs = append(gs, g) } return gs } +// numOrUnknown is a figure, or -1 where nvidia-smi says it has none: a card +// without ECC has no error count, which is not the same as none. +func numOrUnknown(s string) float64 { + v, err := strconv.ParseFloat(s, 64) + if err != nil || v < 0 { + return -1 + } + return v +} + // num reads a figure nvidia-smi may give as "[N/A]" or "N/A". func num(s string) float64 { v, err := strconv.ParseFloat(s, 64) diff --git a/internal/agent/discovery/sensors_test.go b/internal/agent/discovery/sensors_test.go index ce1507e..dc9ae07 100644 --- a/internal/agent/discovery/sensors_test.go +++ b/internal/agent/discovery/sensors_test.go @@ -167,3 +167,44 @@ func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T t.Errorf("%+v", got.GPUs) } } + +func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) { + // What the card is doing now, every sample: discovery read it once, at + // start, and a card busy since looked as idle as it was then. + smi := func(_ context.Context, fields string) ([]byte, error) { + if !strings.Contains(fields, "utilization.gpu") { + return nil, errors.New("asked without how busy the cards are") + } + return []byte( + "0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" + + "1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil + } + now := time.Now() + got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs + if len(got) != 2 { + t.Fatalf("%+v", got) + } + g0, g1 := got[0], got[1] + if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 || + g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 { + t.Errorf("card 0 %+v", g0) + } + // Idle is zero; a count the card does not keep is unknown, not zero. + if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 { + t.Errorf("card 1 %+v", g1) + } +} + +func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) { + smi := func(_ context.Context, fields string) ([]byte, error) { + if strings.Contains(fields, "utilization.gpu") { + return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`) + } + return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil + } + now := time.Now() + got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs + if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 { + t.Errorf("%+v", got) + } +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index c2951ed..0ca6387 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -585,6 +585,9 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { out.GPUs = append(out.GPUs, inventory.GPUSensor{ Index: int(g.GetIndex()), TempC: g.GetTempC(), MemoryTempC: g.GetMemoryTempC(), PowerW: g.GetPowerW(), PowerLimitW: g.GetPowerLimitW(), Throttle: g.GetThrottle(), + UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(), + MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(), + SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(), }) } return out diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 81f5923..09144e4 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -73,6 +73,8 @@ type Collector struct { gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc gpuUtil *prometheus.Desc + gpuMemUtil *prometheus.Desc + gpuSMClock *prometheus.Desc gpuTemp *prometheus.Desc gpuMemTemp *prometheus.Desc gpuPower *prometheus.Desc @@ -149,6 +151,11 @@ func New(src Source) *Collector { "How busy a GPU was over the last sample period. Absent when the driver does not "+ "report it, which is not the same as idle. Memory used says a model is loaded; "+ "this says whether it is doing anything.", gpu, nil), + gpuMemUtil: prometheus.NewDesc("llmgw_gpu_memory_utilization_percent", + "How much of the sample period a GPU's memory was being read or written: "+ + "decode is bound by it, so a card busy here and idle above is waiting on memory.", gpu, nil), + gpuSMClock: prometheus.NewDesc("llmgw_gpu_sm_clock_mhz", + "A GPU's streaming multiprocessor clock now; one held low under load is the card being throttled.", gpu, nil), gpuMemTemp: prometheus.NewDesc("llmgw_gpu_memory_temperature_celsius", "A GPU's memory (HBM) temperature, where the driver reports it.", gpu, nil), gpuPower: prometheus.NewDesc("llmgw_gpu_power_watts", @@ -271,7 +278,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, - c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, + c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuTemp, c.gpuHealthy, c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -388,20 +395,28 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { byClass[class]++ labels := []string{n.ID, strconv.Itoa(g.Index), class} - ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, - float64(g.MemoryTotalMB)*1024*1024, labels...) - ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, - float64(g.MemoryUsedMB)*1024*1024, labels...) - if g.UtilizationPercent > 0 { - // Only when reported. A driver too old to give it would - // otherwise show every card at a flat zero, which reads as an - // idle fleet rather than a missing measurement. - ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, - float64(g.UtilizationPercent), labels...) + s, fresh := sensorFor(n.Load, g.Index, now) + if fresh && s.Busy() { + // What the card is doing now, sampled every heartbeat. + c.gpuBusy(ch, s, labels) + } else { + // An agent too old to sample it: what it read at + // registration, which is all there is. + ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, + float64(g.MemoryTotalMB)*1024*1024, labels...) + ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, + float64(g.MemoryUsedMB)*1024*1024, labels...) + if g.UtilizationPercent > 0 { + // Only when reported. A driver too old to give it would + // otherwise show every card at a flat zero, which reads as + // an idle fleet rather than a missing measurement. + ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, + float64(g.UtilizationPercent), labels...) + } } // The sensors' fresh reading when there is one; the temperature // read at registration otherwise, which is all an older agent sends. - if s, ok := sensorFor(n.Load, g.Index, now); ok { + if fresh { c.gpuSensor(ch, s, labels) } else if g.Health.TemperatureC > 0 { ch <- prometheus.MustNewConstMetric(c.gpuTemp, prometheus.GaugeValue, @@ -414,8 +429,11 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...) ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue, float64(len(g.NVLinkPeers)), labels...) - ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue, - float64(g.Health.ECCErrors), labels...) + ecc := float64(g.Health.ECCErrors) + if fresh && s.Busy() && s.ECCUncorrected >= 0 { + ecc = s.ECCUncorrected + } + ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue, ecc, labels...) } for class, n2 := range byClass { ch <- prometheus.MustNewConstMetric(c.nodeGPUs, prometheus.GaugeValue, @@ -569,6 +587,23 @@ func sensorFor(l *inventory.Load, index int, now time.Time) (inventory.GPUSensor return inventory.GPUSensor{}, false } +// gpuBusy is how busy a card is, from a sample that carries it. +func (c *Collector) gpuBusy(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) { + ch <- prometheus.MustNewConstMetric(c.gpuMemoryTotal, prometheus.GaugeValue, s.MemoryTotalMB*1024*1024, labels...) + ch <- prometheus.MustNewConstMetric(c.gpuMemoryUsed, prometheus.GaugeValue, s.MemoryUsedMB*1024*1024, labels...) + // Zero is a real reading here — an idle card — and only -1, the driver + // not saying, leaves the series out. + if s.UtilPercent >= 0 { + ch <- prometheus.MustNewConstMetric(c.gpuUtil, prometheus.GaugeValue, s.UtilPercent, labels...) + } + if s.MemUtilPercent >= 0 { + ch <- prometheus.MustNewConstMetric(c.gpuMemUtil, prometheus.GaugeValue, s.MemUtilPercent, labels...) + } + if s.SMClockMHz > 0 { + ch <- prometheus.MustNewConstMetric(c.gpuSMClock, prometheus.GaugeValue, s.SMClockMHz, labels...) + } +} + func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) { for _, v := range []struct { d *prometheus.Desc diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go index cd73f6b..3e5358a 100644 --- a/internal/control/fleetmetrics/sensors_test.go +++ b/internal/control/fleetmetrics/sensors_test.go @@ -39,3 +39,41 @@ func TestSensorsArePublishedFreshAndOnlyWhenMeasured(t *testing.T) { } } } + +func TestHowBusyAGPUIsComesFromTheLatestSampleNotRegistration(t *testing.T) { + // Discovery read 87% and 70 GB used when the agent came up; the card has + // been idle since. The sample says so, zero included. + reg := discovery.GPU{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, + MemoryUsedMB: 70000, UtilizationPercent: 87} + n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{reg}, + Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 35, + UtilPercent: 0, MemUtilPercent: 0, MemoryUsedMB: 512, MemoryTotalMB: 81559, + SMClockMHz: 345, ECCUncorrected: 2}}}} + out := gather(t, fakeSource{nodes: []inventory.Node{n}}) + for _, want := range []string{ + "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n", + "llmgw_gpu_memory_utilization_percent class=h100-80g gpu=0 node=g1 -> 0\n", + "llmgw_gpu_memory_used_bytes class=h100-80g gpu=0 node=g1 -> 536870912\n", + "llmgw_gpu_sm_clock_mhz class=h100-80g gpu=0 node=g1 -> 345\n", + "llmgw_gpu_ecc_errors class=h100-80g gpu=0 node=g1 -> 2\n", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s in:\n%s", want, out) + } + } + if strings.Contains(out, "-> 87\n") { + t.Error("the utilization read at registration was published over the sample's") + } + + // An agent too old to sample it: registration's figures, and no zero + // standing in for a utilization the driver did not give. + n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35}} + out = gather(t, fakeSource{nodes: []inventory.Node{n}}) + if !strings.Contains(out, "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 87") { + t.Errorf("an older agent's registration figure was dropped:\n%s", out) + } + n.Load.GPUs = []inventory.GPUSensor{{Index: 0, TempC: 35, UtilPercent: -1, MemUtilPercent: -1, MemoryTotalMB: 81559}} + if out := gather(t, fakeSource{nodes: []inventory.Node{n}}); strings.Contains(out, "llmgw_gpu_utilization_percent ") { + t.Errorf("a utilization the driver did not report was published:\n%s", out) + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 4e9df64..04356a6 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -319,8 +319,20 @@ type GPUSensor struct { PowerW float64 `json:"power_w,omitempty"` PowerLimitW float64 `json:"power_limit_w,omitempty"` Throttle []string `json:"throttle,omitempty"` + // How busy the card is, as of this sample; -1 is not reported. Zero + // MemoryTotalMB is an agent too old to send these, whose figures are + // the ones it read at registration. + UtilPercent float64 `json:"util_percent,omitempty"` + MemUtilPercent float64 `json:"mem_util_percent,omitempty"` + MemoryUsedMB float64 `json:"memory_used_mb,omitempty"` + MemoryTotalMB float64 `json:"memory_total_mb,omitempty"` + SMClockMHz float64 `json:"sm_clock_mhz,omitempty"` + ECCUncorrected float64 `json:"ecc_uncorrected,omitempty"` } +// Busy reports whether the sample carries how busy the card is. +func (s GPUSensor) Busy() bool { return s.MemoryTotalMB > 0 } + // ShardLoad is one engine's use: cores busy, and resident memory. type ShardLoad struct { CPUCores float64 diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 536e184..ba0f7cc 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -201,6 +201,9 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat for _, m := range d.Models { d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0 } + if d.AnyGPUs { + d.Fleet.LoadGPU(r.Context(), nav.History, nav.Window) + } d.Summary = src.Summary() if fs, ok := src.(FabricSource); ok { r := fs.Fabrics() diff --git a/internal/web/dashcharts.go b/internal/web/dashcharts.go index 5821e63..a6e83dd 100644 --- a/internal/web/dashcharts.go +++ b/internal/web/dashcharts.go @@ -24,7 +24,10 @@ type FleetCharts struct { Queue TimeSeries KV TimeSeries Ready TimeSeries - Window string + // GPUBusy and GPUMemory are the fleet's cards by class, for a fleet that + // has them. + GPUBusy, GPUMemory TimeSeries + Window string // Err is set when the metrics store could not be read. Shown as a line of // text where the charts would be: an empty panel reads as a healthy cluster // with nothing happening, which is the opposite of not knowing. @@ -90,3 +93,25 @@ func LoadFleetCharts(ctx context.Context, h History, window time.Duration, } return out } + +// LoadGPU adds the fleet's GPUs, by class: how busy on average, and how full +// their memory. Asked only of a fleet with GPUs. A failure leaves these two +// empty and the rest of the page as it was; they are not what the other +// panels' all-or-nothing is about. +func (f *FleetCharts) LoadGPU(ctx context.Context, h History, window time.Duration) { + if f == nil || h == nil || f.Err != "" { + return + } + if window <= 0 { + window = time.Hour + } + step := window / 60 + if s, err := h.RangeQuery(ctx, `avg by (class) (llmgw_gpu_utilization_percent) / 100`, "class", window, step); err == nil { + f.GPUBusy = LineChart(s, "%", 6) + } + if s, err := h.RangeQuery(ctx, + `sum by (class) (llmgw_gpu_memory_used_bytes) / sum by (class) (llmgw_gpu_memory_total_bytes)`, + "class", window, step); err == nil { + f.GPUMemory = LineChart(s, "%", 6) + } +} diff --git a/internal/web/node_page.go b/internal/web/node_page.go index 59f8303..35cabd8 100644 --- a/internal/web/node_page.go +++ b/internal/web/node_page.go @@ -40,8 +40,10 @@ type nodeEngine struct { // nodeCharts are the node's recent history, from the metrics store. type nodeCharts struct { CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries - Window string - Err string + // The GPUs', one line per card; empty on a node without them. + GPUUtil, GPUMemory, GPUTemp, GPUPower TimeSeries + Window string + Err string } // nodePanels are the charts, each a query scoped to this node. %s is the @@ -63,6 +65,23 @@ var nodePanels = []struct { func(c *nodeCharts, t TimeSeries) { c.EngineMemory = t }}, } +// gpuPanels are the GPUs' charts, one line per card, for a node that has +// them. Read from the heartbeat's samples, not from what the node said when +// it registered. +var gpuPanels = []struct { + expr, unit string + into func(*nodeCharts, TimeSeries) +}{ + {`llmgw_gpu_utilization_percent{node=%s} / 100`, "%", + func(c *nodeCharts, t TimeSeries) { c.GPUUtil = t }}, + {`llmgw_gpu_memory_used_bytes{node=%s}`, "bytes", + func(c *nodeCharts, t TimeSeries) { c.GPUMemory = t }}, + {`llmgw_gpu_temperature_celsius{node=%s}`, "", + func(c *nodeCharts, t TimeSeries) { c.GPUTemp = t }}, + {`llmgw_gpu_power_watts{node=%s}`, "", + func(c *nodeCharts, t TimeSeries) { c.GPUPower = t }}, +} + // NodePage shows one node. Its charts come from the metrics store when one // is configured; the current figures come from the node's last heartbeat // either way. @@ -115,14 +134,14 @@ func NodePage(src Source, history History, window time.Duration, onAdminListener sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores }) if history != nil { - d.Charts = loadNodeCharts(r.Context(), history, id, replicas, window, log) + d.Charts = loadNodeCharts(r.Context(), history, id, replicas, d.Node.GPUTotal, window, log) } noStore(w) render(w, "node", d) } } -func loadNodeCharts(ctx context.Context, h History, node string, replicas []string, +func loadNodeCharts(ctx context.Context, h History, node string, replicas []string, gpus int, window time.Duration, log *slog.Logger, ) *nodeCharts { step := window / 60 @@ -139,6 +158,19 @@ func loadNodeCharts(ctx context.Context, h History, node string, replicas []stri } p.into(out, LineChart(series, p.unit, 4)) } + // Every card its own line, up to the eight a node commonly has. + if gpus > 0 { + for _, p := range gpuPanels { + series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), "gpu", window, step) + if err != nil { + continue + } + for i := range series { + series[i].Name = "GPU " + series[i].Name + } + p.into(out, LineChart(series, p.unit, max(gpus, 8))) + } + } // The engines' queues, for the replicas that are here now. if len(replicas) > 0 { re := make([]string, len(replicas)) diff --git a/internal/web/node_page_test.go b/internal/web/node_page_test.go index 6e91dbd..f947477 100644 --- a/internal/web/node_page_test.go +++ b/internal/web/node_page_test.go @@ -205,3 +205,68 @@ func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) { } } } + +// gpuSource is a node with two GPUs, one busy and one idle. +type gpuSource struct{ fakeSource } + +func (gpuSource) Nodes() []adminapi.NodeView { + n := fakeSource{}.Nodes()[0] + n.GPUTotal = 2 + n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39, + GPUs: []inventory.GPUSensor{ + {Index: 0, TempC: 71, PowerW: 612, PowerLimitW: 700, UtilPercent: 97, MemUtilPercent: 64, + MemoryUsedMB: 72909, MemoryTotalMB: 81559, SMClockMHz: 1980, ECCUncorrected: 0}, + {Index: 1, TempC: 30, PowerW: 70, PowerLimitW: 700, UtilPercent: 0, MemUtilPercent: 0, + MemoryUsedMB: 512, MemoryTotalMB: 81559, SMClockMHz: 345, ECCUncorrected: 3}, + }} + return []adminapi.NodeView{n} +} + +func TestAGPUNodesPageShowsHowBusyEachCardIsAndOverTime(t *testing.T) { + q := func(expr string) string { return strings.Replace(expr, "%s", `"llmgw-node01"`, 1) } + h := &fakeHistory{byExpr: map[string][]promq.Series{ + q(`llmgw_gpu_utilization_percent{node=%s} / 100`): {ts("0", 0.9, 0.97), ts("1", 0, 0)}, + q(`llmgw_gpu_memory_used_bytes{node=%s}`): {ts("0", 7e10, 7.6e10)}, + q(`llmgw_gpu_temperature_celsius{node=%s}`): {ts("0", 65, 71)}, + q(`llmgw_gpu_power_watts{node=%s}`): {ts("0", 500, 612)}, + }} + _, body := get(t, nodeMux(gpuSource{}, h), "http://x/admin/nodes/llmgw-node01", nil) + for _, want := range []string{ + "97%", "memory 64%", "71.2 of 80 GB", "1980 MHz", + "3 uncorrected ECC errors", + "GPU busy, by card", "GPU memory used, by card", "GPU temperature, °C", "GPU power, W", "GPU 0", + } { + if !strings.Contains(body, want) { + t.Errorf("the page lacks %q", want) + } + } + // A node without GPUs is not asked about them. + h2 := &fakeHistory{} + get(t, nodeMux(cpuFleet{}, h2), "http://x/admin/nodes/llmgw-node01", nil) + for _, e := range h2.asked { + if strings.Contains(e, "llmgw_gpu_") { + t.Errorf("a node with no GPU was asked for %s", e) + } + } +} + +func TestTheClusterPageDrawsItsGPUsByClass(t *testing.T) { + h := &fakeHistory{byExpr: map[string][]promq.Series{ + `avg by (class) (llmgw_gpu_utilization_percent) / 100`: {ts("h100-80g", 0.2, 0.6)}, + `sum by (class) (llmgw_gpu_memory_used_bytes) / sum by (class) (llmgw_gpu_memory_total_bytes)`: {ts("h100-80g", 0.8, 0.9)}, + }} + _, body := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{History: h, Window: time.Hour}, nil), "http://127.0.0.1:9090/", nil) + if !strings.Contains(body, "GPUs busy, by class") || !strings.Contains(body, "GPU memory in use, by class") { + t.Error("a fleet with GPUs has no GPU charts") + } + h2 := &fakeHistory{} + _, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{History: h2, Window: time.Hour}, nil), "http://127.0.0.1:9090/", nil) + if strings.Contains(cpu, "GPUs busy") { + t.Error("a fleet without GPUs was drawn GPU charts") + } + for _, e := range h2.asked { + if strings.Contains(e, "llmgw_gpu_") { + t.Errorf("a fleet without GPUs was asked for %s", e) + } + } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 195ca69..20e6742 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -213,6 +213,14 @@ unused nodes, which is what decides whether such a replica can land.

{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model" "note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}} +{{if $.AnyGPUs}} +
+ {{template "tschart" dict "t" .GPUBusy "title" "GPUs busy, by class" + "note" "The average card of each class. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one."}} + {{template "tschart" dict "t" .GPUMemory "title" "GPU memory in use, by class" + "note" "Engines reserve most of a card's memory when they start, so this follows replicas placed more than requests served."}} +
+{{end}}
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started. The first thing to move when a replica is falling behind."}} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index c316e7d..0409ad0 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -35,13 +35,18 @@ {{if .Unhealthy}}

{{join .Unhealthy}}

{{end}} {{with .Load}}{{if .GPUs}}

GPUs

- +
GPUTemperatureMemoryPowerHeld back by
{{range .GPUs}} + {{/* From the last sample; a dash where the driver or an older agent did + not say, rather than a zero nobody measured. */}} + + - + + {{end}}
GPUBusyMemory usedTemperatureMemory temp.PowerClockHeld back by
{{.Index}}{{if and .MemoryTotalMB (ge .UtilPercent 0.0)}}{{template "minibar" pctbar .UtilPercent}}{{printf "%.0f" .UtilPercent}}%{{if ge .MemUtilPercent 0.0}} memory {{printf "%.0f" .MemUtilPercent}}%{{end}}{{else}}—{{end}}{{if .MemoryTotalMB}}{{template "minibar" pctbar (percent .MemoryUsedMB .MemoryTotalMB)}}{{printf "%.1f" (gib .MemoryUsedMB)}} of {{printf "%.0f" (gib .MemoryTotalMB)}} GB{{else}}—{{end}} {{if .TempC}}{{printf "%.0f" .TempC}} °C{{else}}—{{end}} {{if .MemoryTempC}}{{printf "%.0f" .MemoryTempC}} °C{{else}}—{{end}} {{if and .PowerW .PowerLimitW}}{{template "minibar" pctbar (percent .PowerW .PowerLimitW)}}{{printf "%.0f" .PowerW}} W of {{printf "%.0f" .PowerLimitW}}{{else if .PowerW}}{{printf "%.0f" .PowerW}} W{{else}}—{{end}}{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if .SMClockMHz}}{{printf "%.0f" .SMClockMHz}} MHz{{else}}—{{end}}{{if .Throttle}}{{join .Throttle}}{{else}}nothing{{end}}{{if gt .ECCUncorrected 0.0}} {{printf "%.0f" .ECCUncorrected}} uncorrected ECC errors{{end}}
{{end}}{{end}} {{end}} @@ -120,6 +125,16 @@ traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS {{template "tschart" dict "t" .EngineCPU "title" "Cores busy, by model" "note" "Each engine's process group, the way top counts it."}} {{template "tschart" dict "t" .EngineMemory "title" "Engine memory, by model" "note" "Resident memory, including the weights mapped from disk. The kernel can take those pages back, so this can exceed RAM in use."}}
+{{if $.Node.GPUTotal}} +
+ {{template "tschart" dict "t" .GPUUtil "title" "GPU busy, by card" "note" "Of each sample period, the share a kernel was running. A card held by a replica and idle here is one nobody is asking."}} + {{template "tschart" dict "t" .GPUMemory "title" "GPU memory used, by card" "note" "Framebuffer in use: the weights and the KV cache an engine reserved, and anything not llmgw's."}} +
+
+ {{template "tschart" dict "t" .GPUTemp "title" "GPU temperature, °C" "note" "Throttling usually starts in the high 80s; see what holds a card back in the table above."}} + {{template "tschart" dict "t" .GPUPower "title" "GPU power, W" "note" "Draw against the card's enforced limit; at the limit is sw_power_cap."}} +
+{{end}}
{{template "tschart" dict "t" .Load "title" "Load average" "note" "Above the core count means work is waiting for a CPU."}} {{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica" "note" "Requests accepted and not yet started, on the replicas here now."}} diff --git a/internal/web/web.go b/internal/web/web.go index 534aebd..8f44fe9 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -110,6 +110,8 @@ var funcs = template.FuncMap{ return p }, "ago": ago, + // gib is mebibytes as gibibytes, for a GPU's memory. + "gib": func(mib float64) float64 { return mib / 1024 }, // percent is part of whole, 0 to 100, for a bar. "percent": func(part, whole float64) float64 { if whole <= 0 {