Ilker ManapandClaude Opus 5.5
c866b3a8ee
CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
93 lines
3.1 KiB
Go
93 lines
3.1 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
// modelEfficiency is what a model gets out of the GPUs it holds, over the
|
|
// last quarter hour.
|
|
type modelEfficiency struct {
|
|
Model string
|
|
// TokensPerSec is output tokens a second; GPUs the cards it holds now.
|
|
TokensPerSec float64
|
|
GPUs float64
|
|
// Watts is what those cards draw; Util the average share of time a
|
|
// kernel ran on them, which says nothing of how much of each chip it ran
|
|
// on. PowerLoad (draw against the power limit) and SMActive (the SMs'
|
|
// share of time with work, where GPM or DCGM reads it) are how much, in
|
|
// percent; -1 is not known.
|
|
Watts, Util float64
|
|
PowerLoad, SMActive float64
|
|
// PerGPU and PerWh are the two ways of saying how much work the cards
|
|
// do: output tokens a second per card, and per watt-hour drawn.
|
|
PerGPU, PerWh float64
|
|
}
|
|
|
|
// The queries behind the table. Each model's cards are found by joining the
|
|
// GPU series to llmgw_gpu_assigned on the card, which carries the model.
|
|
var efficiencyQueries = []struct {
|
|
expr string
|
|
into func(*modelEfficiency, float64)
|
|
}{
|
|
{`sum by (model) (rate(llmgw_tokens_total{direction="completion"}[15m]))`,
|
|
func(e *modelEfficiency, v float64) { e.TokensPerSec = v }},
|
|
{`count by (model) (llmgw_gpu_assigned)`,
|
|
func(e *modelEfficiency, v float64) { e.GPUs = v }},
|
|
{`sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
|
|
func(e *modelEfficiency, v float64) { e.Watts = v }},
|
|
{`avg by (model) (llmgw_gpu_utilization_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
|
|
func(e *modelEfficiency, v float64) { e.Util = v }},
|
|
{`avg by (model) ((llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts) * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
|
|
func(e *modelEfficiency, v float64) { e.PowerLoad = 100 * v }},
|
|
{`avg by (model) (llmgw_gpu_sm_active_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
|
|
func(e *modelEfficiency, v float64) { e.SMActive = v }},
|
|
}
|
|
|
|
// loadEfficiency is the table, best per card first; nil when the metrics
|
|
// store cannot be read or no model holds a GPU.
|
|
func loadEfficiency(ctx context.Context, h History) []modelEfficiency {
|
|
if h == nil {
|
|
return nil
|
|
}
|
|
byModel := map[string]*modelEfficiency{}
|
|
for _, q := range efficiencyQueries {
|
|
// The last of a short range: what History offers, and the one point
|
|
// the table needs.
|
|
series, err := h.RangeQuery(ctx, q.expr, "model", 10*time.Minute, 5*time.Minute)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
for _, s := range series {
|
|
if len(s.Points) == 0 {
|
|
continue
|
|
}
|
|
e := byModel[s.Name]
|
|
if e == nil {
|
|
e = &modelEfficiency{Model: s.Name, PowerLoad: -1, SMActive: -1}
|
|
byModel[s.Name] = e
|
|
}
|
|
q.into(e, s.Last())
|
|
}
|
|
}
|
|
var out []modelEfficiency
|
|
for _, e := range byModel {
|
|
if e.GPUs == 0 {
|
|
continue // a model on the CPU; this table is about cards
|
|
}
|
|
e.PerGPU = e.TokensPerSec / e.GPUs
|
|
if e.Watts > 0 {
|
|
e.PerWh = e.TokensPerSec * 3600 / e.Watts
|
|
}
|
|
out = append(out, *e)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if out[i].PerGPU != out[j].PerGPU {
|
|
return out[i].PerGPU > out[j].PerGPU
|
|
}
|
|
return out[i].Model < out[j].Model
|
|
})
|
|
return out
|
|
}
|