From ff554c7062226831d5972b206298d6dd07eb4991 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 15:56:42 +0200 Subject: [PATCH] Show what each model gets from its GPUs, per card and per watt-hour Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 + deploy/monitoring/grafana/llmgw-overview.json | 76 +++++++++++++++++ internal/web/dashboard.go | 4 + internal/web/efficiency.go | 83 +++++++++++++++++++ internal/web/gpumap_test.go | 20 +++++ internal/web/templates/dashboard.html | 11 +++ 6 files changed, 197 insertions(+) create mode 100644 internal/web/efficiency.go diff --git a/CHANGELOG.md b/CHANGELOG.md index dc0aa06..905ccca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,9 @@ and a figure the driver does not give is left out rather than shown as zero. with what holds it and its figures under the pointer; free cards dashed. - **Cards held and doing nothing**: the replicas whose every card reads under 5% busy, with when they were last asked — the first place to find room. +- **What each model gets from its cards**, on the cluster page and in + Grafana: output tokens a second per GPU and per watt-hour, with the cards' + draw and average utilisation, over the last quarter hour. - **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds each card and whose quota it counts against, for joining the GPU series to models and tenants. diff --git a/deploy/monitoring/grafana/llmgw-overview.json b/deploy/monitoring/grafana/llmgw-overview.json index 27583df..ea533cc 100644 --- a/deploy/monitoring/grafana/llmgw-overview.json +++ b/deploy/monitoring/grafana/llmgw-overview.json @@ -779,6 +779,82 @@ "mode": "multi" } } + }, + { + "type": "timeseries", + "title": "Output tokens/s per GPU, by model", + "description": "What each model gets out of each card it holds.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 59, + "w": 12, + "h": 7 + }, + "id": 22, + "targets": [ + { + "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Output tokens per watt-hour, by model", + "description": "Tokens for the energy the model's cards draw.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 12, + "y": 59, + "w": 12, + "h": 7 + }, + "id": 23, + "targets": [ + { + "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } } ] } \ No newline at end of file diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 3b7ba3b..14283a6 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -24,6 +24,9 @@ type DashboardData struct { // holding cards that are doing nothing; both from the last heartbeats. GPUMap *gpuMap IdleGPUs []idleHolding + // Efficiency is what each model gets from its cards, from the metrics + // store; nil without one. + Efficiency []modelEfficiency // Notice and Problem are what the last change made from this page did, // carried over the redirect back to it. Notice, Problem string @@ -211,6 +214,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat } if d.AnyGPUs { d.Fleet.LoadGPU(r.Context(), nav.History, nav.Window) + d.Efficiency = loadEfficiency(r.Context(), nav.History) } d.Summary = src.Summary() if fs, ok := src.(FabricSource); ok { diff --git a/internal/web/efficiency.go b/internal/web/efficiency.go new file mode 100644 index 0000000..4a86638 --- /dev/null +++ b/internal/web/efficiency.go @@ -0,0 +1,83 @@ +package web + +import ( + "context" + "sort" + "time" +) + +// modelEfficiency is what a model gets out of the GPUs it holds, over the +// last quarter hour. +type modelEfficiency struct { + Model string + // TokensPerSec is output tokens a second; GPUs the cards it holds now. + TokensPerSec float64 + GPUs float64 + // Watts is what those cards draw; Util their average utilisation. + Watts, Util float64 + // PerGPU and PerWh are the two ways of saying how much work the cards + // do: output tokens a second per card, and per watt-hour drawn. + PerGPU, PerWh float64 +} + +// The queries behind the table. Each model's cards are found by joining the +// GPU series to llmgw_gpu_assigned on the card, which carries the model. +var efficiencyQueries = []struct { + expr string + into func(*modelEfficiency, float64) +}{ + {`sum by (model) (rate(llmgw_tokens_total{direction="completion"}[15m]))`, + func(e *modelEfficiency, v float64) { e.TokensPerSec = v }}, + {`count by (model) (llmgw_gpu_assigned)`, + func(e *modelEfficiency, v float64) { e.GPUs = v }}, + {`sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)`, + func(e *modelEfficiency, v float64) { e.Watts = v }}, + {`avg by (model) (llmgw_gpu_utilization_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`, + func(e *modelEfficiency, v float64) { e.Util = v }}, +} + +// loadEfficiency is the table, best per card first; nil when the metrics +// store cannot be read or no model holds a GPU. +func loadEfficiency(ctx context.Context, h History) []modelEfficiency { + if h == nil { + return nil + } + byModel := map[string]*modelEfficiency{} + for _, q := range efficiencyQueries { + // The last of a short range: what History offers, and the one point + // the table needs. + series, err := h.RangeQuery(ctx, q.expr, "model", 10*time.Minute, 5*time.Minute) + if err != nil { + return nil + } + for _, s := range series { + if len(s.Points) == 0 { + continue + } + e := byModel[s.Name] + if e == nil { + e = &modelEfficiency{Model: s.Name} + byModel[s.Name] = e + } + q.into(e, s.Last()) + } + } + var out []modelEfficiency + for _, e := range byModel { + if e.GPUs == 0 { + continue // a model on the CPU; this table is about cards + } + e.PerGPU = e.TokensPerSec / e.GPUs + if e.Watts > 0 { + e.PerWh = e.TokensPerSec * 3600 / e.Watts + } + out = append(out, *e) + } + sort.Slice(out, func(i, j int) bool { + if out[i].PerGPU != out[j].PerGPU { + return out[i].PerGPU > out[j].PerGPU + } + return out[i].Model < out[j].Model + }) + return out +} diff --git a/internal/web/gpumap_test.go b/internal/web/gpumap_test.go index e704dde..6e4cc2c 100644 --- a/internal/web/gpumap_test.go +++ b/internal/web/gpumap_test.go @@ -1,12 +1,14 @@ package web import ( + "context" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/promq" ) // mapFleet is a node of four cards: chat holds two and is idle, code holds @@ -56,3 +58,21 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) { t.Error("a fleet without GPUs has a GPU map") } } + +func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) { + h := &fakeHistory{byExpr: map[string][]promq.Series{}} + for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}} { + h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])} + } + got := loadEfficiency(context.Background(), h) + if len(got) != 2 || got[0].Model != "chat" { + t.Fatalf("%+v", got) + } + c := got[0] + if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 { + t.Errorf("chat: %+v", c) + } + if got[1].PerGPU != 15 { + t.Errorf("code: %+v", got[1]) + } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index ff3513d..0d4a34a 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -57,6 +57,17 @@ {{end}}

{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.

{{end}} +{{with .Efficiency}} +

What each model gets from its cards last 15 minutes

+{{/* Per card, because a model on eight cards making twice the tokens of one + on two is the worse of the two; per watt-hour, because that is what the + room is paying. */}} + +{{range .}} + + +{{end}}
ModelGPUsOutput tokens/sPer GPUBusyDrawTokens per Wh
{{.Model}}{{printf "%.0f" .GPUs}}{{printf "%.1f" .TokensPerSec}}{{printf "%.1f" .PerGPU}}{{printf "%.0f" .Util}}%{{if .Watts}}{{printf "%.0f" .Watts}} W{{else}}—{{end}}{{if .PerWh}}{{printf "%.0f" .PerWh}}{{else}}—{{end}}
+{{end}} {{with .IdleGPUs}}

Cards held and doing nothing

{{/* The first place to find room: replicas whose every card reads under 5%