Show what each model gets from its GPUs, per card and per watt-hour

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-01 15:56:42 +02:00
1 parent b6a802c03b
commit ff554c7062
6 files changed
+197

No files matched your search

+3
View File
@@ -53,6 +53,9 @@ and a figure the driver does not give is left out rather than shown as zero.
with what holds it and its figures under the pointer; free cards dashed.
- **Cards held and doing nothing**: the replicas whose every card reads under
5% busy, with when they were last asked — the first place to find room.
- **What each model gets from its cards**, on the cluster page and in
Grafana: output tokens a second per GPU and per watt-hour, with the cards'
draw and average utilisation, over the last quarter hour.
- **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds
each card and whose quota it counts against, for joining the GPU series to
models and tenants.
@@ -779,6 +779,82 @@
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "Output tokens/s per GPU, by model",
"description": "What each model gets out of each card it holds.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 59,
"w": 12,
"h": 7
},
"id": 22,
"targets": [
{
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "Output tokens per watt-hour, by model",
"description": "Tokens for the energy the model's cards draw.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 59,
"w": 12,
"h": 7
},
"id": 23,
"targets": [
{
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
}
]
}
+4
View File
@@ -24,6 +24,9 @@ type DashboardData struct {
// holding cards that are doing nothing; both from the last heartbeats.
GPUMap *gpuMap
IdleGPUs []idleHolding
// Efficiency is what each model gets from its cards, from the metrics
// store; nil without one.
Efficiency []modelEfficiency
// Notice and Problem are what the last change made from this page did,
// carried over the redirect back to it.
Notice, Problem string
@@ -211,6 +214,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
}
if d.AnyGPUs {
d.Fleet.LoadGPU(r.Context(), nav.History, nav.Window)
d.Efficiency = loadEfficiency(r.Context(), nav.History)
}
d.Summary = src.Summary()
if fs, ok := src.(FabricSource); ok {
+83
View File
@@ -0,0 +1,83 @@
package web
import (
"context"
"sort"
"time"
)
// modelEfficiency is what a model gets out of the GPUs it holds, over the
// last quarter hour.
type modelEfficiency struct {
Model string
// TokensPerSec is output tokens a second; GPUs the cards it holds now.
TokensPerSec float64
GPUs float64
// Watts is what those cards draw; Util their average utilisation.
Watts, Util float64
// PerGPU and PerWh are the two ways of saying how much work the cards
// do: output tokens a second per card, and per watt-hour drawn.
PerGPU, PerWh float64
}
// The queries behind the table. Each model's cards are found by joining the
// GPU series to llmgw_gpu_assigned on the card, which carries the model.
var efficiencyQueries = []struct {
expr string
into func(*modelEfficiency, float64)
}{
{`sum by (model) (rate(llmgw_tokens_total{direction="completion"}[15m]))`,
func(e *modelEfficiency, v float64) { e.TokensPerSec = v }},
{`count by (model) (llmgw_gpu_assigned)`,
func(e *modelEfficiency, v float64) { e.GPUs = v }},
{`sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
func(e *modelEfficiency, v float64) { e.Watts = v }},
{`avg by (model) (llmgw_gpu_utilization_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
func(e *modelEfficiency, v float64) { e.Util = v }},
}
// loadEfficiency is the table, best per card first; nil when the metrics
// store cannot be read or no model holds a GPU.
func loadEfficiency(ctx context.Context, h History) []modelEfficiency {
if h == nil {
return nil
}
byModel := map[string]*modelEfficiency{}
for _, q := range efficiencyQueries {
// The last of a short range: what History offers, and the one point
// the table needs.
series, err := h.RangeQuery(ctx, q.expr, "model", 10*time.Minute, 5*time.Minute)
if err != nil {
return nil
}
for _, s := range series {
if len(s.Points) == 0 {
continue
}
e := byModel[s.Name]
if e == nil {
e = &modelEfficiency{Model: s.Name}
byModel[s.Name] = e
}
q.into(e, s.Last())
}
}
var out []modelEfficiency
for _, e := range byModel {
if e.GPUs == 0 {
continue // a model on the CPU; this table is about cards
}
e.PerGPU = e.TokensPerSec / e.GPUs
if e.Watts > 0 {
e.PerWh = e.TokensPerSec * 3600 / e.Watts
}
out = append(out, *e)
}
sort.Slice(out, func(i, j int) bool {
if out[i].PerGPU != out[j].PerGPU {
return out[i].PerGPU > out[j].PerGPU
}
return out[i].Model < out[j].Model
})
return out
}
+20
View File
@@ -1,12 +1,14 @@
package web
import (
"context"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)
// mapFleet is a node of four cards: chat holds two and is idle, code holds
@@ -56,3 +58,21 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
t.Error("a fleet without GPUs has a GPU map")
}
}
func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
h := &fakeHistory{byExpr: map[string][]promq.Series{}}
for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}} {
h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])}
}
got := loadEfficiency(context.Background(), h)
if len(got) != 2 || got[0].Model != "chat" {
t.Fatalf("%+v", got)
}
c := got[0]
if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 {
t.Errorf("chat: %+v", c)
}
if got[1].PerGPU != 15 {
t.Errorf("code: %+v", got[1])
}
}
+11
View File
@@ -57,6 +57,17 @@
{{end}}</div>
<p class="note">{{.Legend}} The figure is the one coloured by; the rest of a card is under the pointer.</p>
{{end}}
{{with .Efficiency}}
<h3>What each model gets from its cards <span class="opt">last 15 minutes</span></h3>
{{/* Per card, because a model on eight cards making twice the tokens of one
on two is the worse of the two; per watt-hour, because that is what the
room is paying. */}}
<table class="rows"><tr><th>Model</th><th class="num">GPUs</th><th class="num">Output tokens/s</th><th class="num">Per GPU</th><th class="num">Busy</th><th class="num">Draw</th><th class="num">Tokens per Wh</th></tr>
{{range .}}<tr><td>{{.Model}}</td><td class="num">{{printf "%.0f" .GPUs}}</td><td class="num">{{printf "%.1f" .TokensPerSec}}</td>
<td class="num">{{printf "%.1f" .PerGPU}}</td><td class="num">{{printf "%.0f" .Util}}%</td>
<td class="num">{{if .Watts}}{{printf "%.0f" .Watts}} W{{else}}—{{end}}</td><td class="num">{{if .PerWh}}{{printf "%.0f" .PerWh}}{{else}}—{{end}}</td></tr>
{{end}}</table>
{{end}}
{{with .IdleGPUs}}
<h3>Cards held and doing nothing</h3>
{{/* The first place to find room: replicas whose every card reads under 5%