How much of each GPU works, beside whether anything does
CI / test (push) Failing after 6m42s

The busy figure is nvidia-smi's utilization.gpu: the share of time any
kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy
while the SMs had work 69% of the time and issued on 18% of their cycles.

Busy is now said to be that, and beside it: each card's draw against its
power limit (efficiency table, fleet and node charts, the console's GPU
work chart), and where the profiling counters can be read, SMs active, SM
occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's
GPM on Hopper and later or a dcgm-exporter on the node
(discovery.dcgm_exporter). The GPU map colours by work by default and keeps
busy as a mode. A MIG instance is never published with its card's profile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-31 20:11:26 +01:00
1 parent 7bfe70c8df
commit c866b3a8ee
34 files changed
+954 -52

No files matched your search

+13 -1
View File
@@ -27,7 +27,11 @@ type FleetCharts struct {
// GPUBusy and GPUMemory are the fleet's cards by class, for a fleet that
// has them.
GPUBusy, GPUMemory TimeSeries
Window string
// GPUPowerLoad is the cards' draw against their power limit, and
// GPUSMActive their SMs' share of time with work, where something reads
// it: how much of each chip the busy figure is using.
GPUPowerLoad, GPUSMActive TimeSeries
Window string
// Err is set when the metrics store could not be read. Shown as a line of
// text where the charts would be: an empty panel reads as a healthy cluster
// with nothing happening, which is the opposite of not knowing. The
@@ -144,4 +148,12 @@ func (f *FleetCharts) LoadGPU(ctx context.Context, h History, window time.Durati
"class", window, step); err == nil {
f.GPUMemory = l.LineChart(s, "%", 6)
}
if s, err := h.RangeQuery(ctx,
`sum by (class) (llmgw_gpu_power_watts) / sum by (class) (llmgw_gpu_power_limit_watts)`,
"class", window, step); err == nil {
f.GPUPowerLoad = l.LineChart(s, "%", 6)
}
if s, err := h.RangeQuery(ctx, `avg by (class) (llmgw_gpu_sm_active_percent) / 100`, "class", window, step); err == nil {
f.GPUSMActive = l.LineChart(s, "%", 6)
}
}
+12 -3
View File
@@ -13,8 +13,13 @@ type modelEfficiency struct {
// TokensPerSec is output tokens a second; GPUs the cards it holds now.
TokensPerSec float64
GPUs float64
// Watts is what those cards draw; Util their average utilisation.
Watts, Util float64
// Watts is what those cards draw; Util the average share of time a
// kernel ran on them, which says nothing of how much of each chip it ran
// on. PowerLoad (draw against the power limit) and SMActive (the SMs'
// share of time with work, where GPM or DCGM reads it) are how much, in
// percent; -1 is not known.
Watts, Util float64
PowerLoad, SMActive float64
// PerGPU and PerWh are the two ways of saying how much work the cards
// do: output tokens a second per card, and per watt-hour drawn.
PerGPU, PerWh float64
@@ -34,6 +39,10 @@ var efficiencyQueries = []struct {
func(e *modelEfficiency, v float64) { e.Watts = v }},
{`avg by (model) (llmgw_gpu_utilization_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
func(e *modelEfficiency, v float64) { e.Util = v }},
{`avg by (model) ((llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts) * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
func(e *modelEfficiency, v float64) { e.PowerLoad = 100 * v }},
{`avg by (model) (llmgw_gpu_sm_active_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
func(e *modelEfficiency, v float64) { e.SMActive = v }},
}
// loadEfficiency is the table, best per card first; nil when the metrics
@@ -56,7 +65,7 @@ func loadEfficiency(ctx context.Context, h History) []modelEfficiency {
}
e := byModel[s.Name]
if e == nil {
e = &modelEfficiency{Model: s.Name}
e = &modelEfficiency{Model: s.Name, PowerLoad: -1, SMActive: -1}
byModel[s.Name] = e
}
q.into(e, s.Last())
+44 -13
View File
@@ -8,16 +8,25 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// The GPU map is every card in the fleet, a square each, a row per node,
// coloured by one figure: how busy, how hot, or how full its memory is. A
// coloured by one figure: how much of the chip is at work, how busy, how
// hot, or how full its memory is. A
// hundred cards in a table are a hundred rows to read; on the map the one
// that is not like the others is seen before anything is read.
// gpuMapModes are what the map can be coloured by; each is named by the key
// dash.gpumap.mode.<mode>.
var gpuMapModes = []string{"util", "memory", "temp"}
//
// work is the default. util, the driver's busy figure, is the share of time
// any kernel ran, and a kernel on one SM of a hundred reads 100%: a map of
// it is green wherever a model is loaded and asked anything. work is the
// SMs' share of time with work where the card's counters can be read, and
// its power draw against its limit elsewhere, which rises with how much of
// the chip runs rather than with whether anything does.
var gpuMapModes = []string{"work", "util", "memory", "temp"}
func init() {
for _, m := range gpuMapModes {
@@ -72,17 +81,16 @@ func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView
for _, k := range gpuMapModes {
m.Modes = append(m.Modes, struct{ Key, Label string }{k, l.T("dash.gpumap.mode." + k)})
}
if m.By == "" {
m.By = "util"
}
switch m.By {
case "util":
m.Legend = l.T("dash.gpumap.legend.util")
case "temp":
m.Legend = l.T("dash.gpumap.legend.temp")
case "memory":
m.Legend = l.T("dash.gpumap.legend.memory")
default:
m.By = "util"
m.Legend = l.T("dash.gpumap.legend.util")
m.By = "work"
m.Legend = l.T("dash.gpumap.legend.work")
}
for _, n := range nodes {
if n.GPUTotal == 0 {
@@ -106,13 +114,21 @@ func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView
if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
title = append(title, l.T("dash.gpumap.busy", l.Num(s.UtilPercent, 0)))
}
if p := s.Profile; p != nil && p.SMActive >= 0 {
title = append(title, l.T("dash.gpumap.sm_active", l.Num(p.SMActive, 0)))
if p.TensorActive >= 0 {
title = append(title, l.T("dash.gpumap.tensor_active", l.Num(p.TensorActive, 0)))
}
}
if s.MemoryTotalMB > 0 {
title = append(title, l.T("dash.gpumap.memory", l.Num(s.MemoryUsedMB/1024, 1), l.Num(s.MemoryTotalMB/1024, 0)))
}
if s.TempC > 0 {
title = append(title, l.Num(s.TempC, 0)+" °C")
}
if s.PowerW > 0 {
if s.PowerW > 0 && s.PowerLimitW > 0 {
title = append(title, l.T("dash.gpumap.power_of", l.Num(s.PowerW, 0), l.Num(s.PowerLimitW, 0)))
} else if s.PowerW > 0 {
title = append(title, l.Num(s.PowerW, 0)+" W")
}
if len(s.Throttle) > 0 {
@@ -162,6 +178,20 @@ func cellColour(by string, s sensorView, sampled bool) (string, string) {
}
f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
case "work":
if !sampled {
return unknown, ""
}
var f float64
switch {
case s.Profile != nil && s.Profile.SMActive >= 0:
f = clamp01(s.Profile.SMActive / 100)
case s.PowerW > 0 && s.PowerLimitW > 0:
f = clamp01(s.PowerW / s.PowerLimitW)
default:
return unknown, ""
}
return fmt.Sprintf("background:hsl(200,%.0f%%,%.0f%%)", 15+55*f, 88-50*f), fmt.Sprintf("%.0f", 100*f)
}
if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
return unknown, ""
@@ -286,9 +316,10 @@ func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu
// sensorView is a card's last sample.
type sensorView = struct {
TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
Throttle []string
Problems []string
TempC, PowerW, PowerLimitW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
Throttle []string
Problems []string
Profile *inventory.GPUProfile
}
func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
@@ -297,9 +328,9 @@ func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
}
for _, s := range n.Load.GPUs {
if s.Index == gpu {
return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
return sensorView{TempC: s.TempC, PowerW: s.PowerW, PowerLimitW: s.PowerLimitW, UtilPercent: s.UtilPercent,
MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle,
Problems: s.Wear.Problems()}, true
Problems: s.Wear.Problems(), Profile: s.Profile}, true
}
}
return sensorView{}, false
+22 -4
View File
@@ -17,10 +17,14 @@ type mapFleet struct{ fakeSource }
func (mapFleet) Nodes() []adminapi.NodeView {
busy := func(i int, util, temp float64) inventory.GPUSensor {
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300, PowerLimitW: 700}
}
profiled := func(s inventory.GPUSensor) inventory.GPUSensor {
s.Profile = &inventory.GPUProfile{SMActive: 0, SMOccupancy: 0, TensorActive: 0, DRAMActive: -1, Source: "dcgm"}
return s
}
return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{profiled(busy(0, 1, 35)), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
}
func worn(s inventory.GPUSensor) inventory.GPUSensor {
@@ -60,6 +64,20 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
t.Error("the map coloured by temperature does not say so, or show it")
}
// The default colours by how much of the chip works: card 2's draw of
// 300 W of 700, where nothing reads its SMs; card 0's SMs where
// something does. Busy stays a mode of its own.
if !strings.Contains(body, "<strong>work</strong>") || !strings.Contains(body, ">43</span>") ||
!strings.Contains(body, "300 of 700 W") {
t.Error("the map does not colour by work, or say the draw against the limit")
}
if !strings.Contains(body, "gpu07 GPU 0 · chat (chat-1) · 1% busy · SMs 0% active · tensor cores 0%") {
t.Error("a profiled card does not say how much of its chip is at work")
}
_, util := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=util", nil)
if !strings.Contains(util, "<strong>busy</strong>") || !strings.Contains(util, ">93</span>") {
t.Error("the map coloured by busy does not say so, or show it")
}
_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
if strings.Contains(cpu, "gpu-map") {
t.Error("a fleet without GPUs has a GPU map")
@@ -68,7 +86,7 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
h := &fakeHistory{byExpr: map[string][]promq.Series{}}
for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}} {
for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}, {0.6, 0.2}, {55, 0}} {
h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])}
}
got := loadEfficiency(context.Background(), h)
@@ -76,7 +94,7 @@ func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
t.Fatalf("%+v", got)
}
c := got[0]
if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 {
if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 || c.PowerLoad != 60 || c.SMActive != 55 {
t.Errorf("chat: %+v", c)
}
if got[1].PerGPU != 15 {
+1 -1
View File
@@ -37,7 +37,7 @@ func TestTheDashboardSpeaksTurkishWhenAsked(t *testing.T) {
// A state is still its class, and a Turkish word.
`<td class="state state-ready">hazır</td>`,
// The GPU map: its modes, and what a card says under the pointer.
"<strong>meşguliyet</strong>", "gpu07 GPU 2 · code (code-1) · %93 meşgul · 39,1 / 80 GB",
"<strong>iş</strong>", "gpu07 GPU 2 · code (code-1) · %93 meşgul · 39,1 / 80 GB",
// How long a replica has sat idle, in Turkish units.
"30dk0sn önce",
// The traffic chart's peak, with Turkish thousands.
+14 -2
View File
@@ -26,11 +26,16 @@
"dash.gpumap.mode.util": "busy",
"dash.gpumap.mode.memory": "memory",
"dash.gpumap.mode.temp": "temperature",
"dash.gpumap.mode.work": "work",
"dash.gpumap.legend.temp": "Blue is cool, red is 90 °C and over.",
"dash.gpumap.legend.memory": "Pale is empty, dark is full. Engines reserve most of a card's memory when they start.",
"dash.gpumap.legend.util": "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds.",
"dash.gpumap.legend.util": "Pale is idle, dark green is busy: the share of time any kernel ran, however little of the chip it used. A dashed border is a card no replica holds.",
"dash.gpumap.legend.work": "Pale is idle, dark blue is the whole chip at work: the SMs' share of time with work where the card's counters can be read (GPM, DCGM), its power draw against its limit elsewhere. A dashed border is a card no replica holds.",
"dash.gpumap.free": "free",
"dash.gpumap.busy": "%s%% busy",
"dash.gpumap.sm_active": "SMs %s%% active",
"dash.gpumap.tensor_active": "tensor cores %s%%",
"dash.gpumap.power_of": "%s of %s W",
"dash.gpumap.memory": "%s of %s GB",
"dash.gpumap.held_back": "held back by %s",
"dash.gpumap.note": "A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.",
@@ -41,8 +46,11 @@
"dash.eff.tps": "Output tokens/s",
"dash.eff.per_gpu": "Per GPU",
"dash.eff.busy": "Busy",
"dash.eff.power_load": "Power load",
"dash.eff.sm_active": "SM active",
"dash.eff.draw": "Draw",
"dash.eff.per_wh": "Tokens per Wh",
"dash.eff.note": "Busy is the share of time any kernel ran on the cards, not how much of each chip it ran on: a kernel on one SM reads 100%. Power load (draw against the power limit) and SM active (the SMs' share of time with work, where the cards' counters can be read) say how much.",
"dash.idle.title": "Cards held and doing nothing",
"dash.col.replica": "Replica",
@@ -150,7 +158,11 @@
"dash.fleet.ready": "Replicas ready, by model",
"dash.fleet.ready_note": "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here.",
"dash.fleet.gpu_busy": "GPUs busy, by class",
"dash.fleet.gpu_busy_note": "The average card of each class. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.",
"dash.fleet.gpu_busy_note": "The average card of each class: the share of time any kernel ran, not how much of the chip it used. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.",
"dash.fleet.gpu_power_load": "GPU power load, by class",
"dash.fleet.gpu_power_load_note": "The cards' draw against their power limit. It rises with how much of each chip is working, where busy only says that something is; a class busy above and low here runs small kernels or waits between them.",
"dash.fleet.gpu_sm_active": "SMs active, by class",
"dash.fleet.gpu_sm_active_note": "The share of time the cards' SMs had work, from their profiling counters (NVML GPM on Hopper and later, or a dcgm-exporter on the node). The gap to busy is the part of the chip a running kernel leaves idle.",
"dash.fleet.gpu_memory": "GPU memory in use, by class",
"dash.fleet.gpu_memory_note": "Engines reserve most of a card's memory when they start, so this follows replicas placed more than requests served.",
"dash.fleet.queue": "Engine queue depth, by replica",
+5 -1
View File
@@ -77,7 +77,11 @@
"node.chart.engine_memory": "Engine memory, by model",
"node.chart.engine_memory_note": "Resident memory, including the weights mapped from disk. The kernel can take those pages back, so this can exceed RAM in use.",
"node.chart.gpu_busy": "GPU busy, by card",
"node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running. A card held by a replica and idle here is one nobody is asking.",
"node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running — not how much of the chip it used: one SM busy reads 100%. A card held by a replica and idle here is one nobody is asking.",
"node.chart.gpu_power_load": "GPU power load, by card",
"node.chart.gpu_power_load_note": "Draw against the card's power limit. Busy says a kernel ran; this rises with how much of the chip it ran on.",
"node.chart.gpu_sm_active": "SMs active, by card",
"node.chart.gpu_sm_active_note": "The share of time the card's SMs had work, from its profiling counters. When busy reads 90 and this 60, a third of the chip sits idle while a kernel runs.",
"node.chart.gpu_memory": "GPU memory used, by card",
"node.chart.gpu_memory_note": "Framebuffer in use: the weights and the KV cache an engine reserved, and anything not llmgw's.",
"node.chart.gpu_temp": "GPU temperature, °C",
+14 -2
View File
@@ -26,11 +26,16 @@
"dash.gpumap.mode.util": "meşguliyet",
"dash.gpumap.mode.memory": "bellek",
"dash.gpumap.mode.temp": "sıcaklık",
"dash.gpumap.mode.work": "iş",
"dash.gpumap.legend.temp": "Mavi serin, kırmızı 90 °C ve üstü.",
"dash.gpumap.legend.memory": "Açık renk boş, koyu renk dolu. Motorlar başlarken bir kartın belleğinin çoğunu ayırır.",
"dash.gpumap.legend.util": "Açık renk boşta, koyu yeşil tamamen meşgul. Kesikli kenarlık, hiçbir replikanın tutmadığı bir karttır.",
"dash.gpumap.legend.util": "Açık renk boşta, koyu yeşil meşgul demektir: yonganın ne kadarını kullanırsa kullansın, herhangi bir çekirdek işlevinin çalıştığı zaman payı. Kesik kenarlık hiçbir replikanın tutmadığı karttır.",
"dash.gpumap.legend.work": "Açık renk boşta, koyu mavi tüm yongası çalışan kart demektir: kartın sayaçları okunabiliyorsa (GPM, DCGM) SM'lerin iş yaptığı zaman payı, okunamıyorsa çektiği gücün güç sınırına oranı. Kesik kenarlık hiçbir replikanın tutmadığı karttır.",
"dash.gpumap.free": "boş",
"dash.gpumap.busy": "%%%s meşgul",
"dash.gpumap.sm_active": "SM'ler %%%s etkin",
"dash.gpumap.tensor_active": "tensör çekirdekleri %%%s",
"dash.gpumap.power_of": "%s / %s W",
"dash.gpumap.memory": "%s / %s GB",
"dash.gpumap.held_back": "kısıtlayan: %s",
"dash.gpumap.note": "Kırmızı çerçeve, yıpranmakta olan bir karttır: XID hataları, yeniden eşlenmiş ya da sıfırlama bekleyen bellek satırları, NVLink hataları. Karttaki rakam, renklendirmede kullanılan değerdir; kartın geri kalanı imlecin altındadır.",
@@ -41,8 +46,11 @@
"dash.eff.tps": "Çıkış token/sn",
"dash.eff.per_gpu": "GPU başına",
"dash.eff.busy": "Meşgul",
"dash.eff.power_load": "Güç yükü",
"dash.eff.sm_active": "Etkin SM",
"dash.eff.draw": "Çekilen güç",
"dash.eff.per_wh": "Wh başına token",
"dash.eff.note": "Meşgul, kartlarda herhangi bir çekirdek işlevinin çalıştığı zaman payıdır; her yonganın ne kadarında çalıştığı değil: tek bir SM'de çalışan işlev %100 okunur. Ne kadarını kullandığını güç yükü (çekilen gücün güç sınırına oranı) ve etkin SM (kartların sayaçları okunabildiğinde SM'lerin iş yaptığı zaman payı) söyler.",
"dash.idle.title": "Tutulan ama hiçbir şey yapmayan kartlar",
"dash.col.replica": "Replika",
@@ -148,7 +156,11 @@
"dash.fleet.ready": "Hazır replikalar, modele göre",
"dash.fleet.ready_note": "Bilerek durdurulmuş bir modelin hiç çizgisi olmaz; isteğe bağlı çalışan bir katalogun burada sessiz görünmesinin nedeni budur.",
"dash.fleet.gpu_busy": "Meşgul GPU'lar, sınıfa göre",
"dash.fleet.gpu_busy_note": "Her sınıfın ortalama kartı. Replikalar yerleştirilmişken burada düşük olması, tutulan ama istenmeyen kapasite demektir; her düğümün sayfası kartlarını tek tek gösterir.",
"dash.fleet.gpu_busy_note": "Her sınıfın ortalama kartı: herhangi bir çekirdek işlevinin çalıştığı zaman payı, yonganın ne kadarını kullandığı değil. Replikalar yerleştirilmişken burada düşük olması, tutulan ama istenmeyen kapasite demektir; her düğümün sayfası kartlarını tek tek gösterir.",
"dash.fleet.gpu_power_load": "GPU güç yükü, sınıfa göre",
"dash.fleet.gpu_power_load_note": "Kartların çektiği gücün güç sınırına oranı. Meşgul yalnızca bir şeyin çalıştığını söylerken bu, her yonganın ne kadarının çalıştığıyla artar; yukarıda meşgul ve burada düşük görünen bir sınıf küçük çekirdek işlevleri çalıştırıyor ya da aralarında bekliyordur.",
"dash.fleet.gpu_sm_active": "Etkin SM'ler, sınıfa göre",
"dash.fleet.gpu_sm_active_note": "Kartların SM'lerinin iş yaptığı zaman payı, profil sayaçlarından (Hopper ve sonrasında NVML GPM ya da düğümdeki bir dcgm-exporter). Meşgul ile arasındaki fark, çalışan bir işlevin yongada boş bıraktığı kısımdır.",
"dash.fleet.gpu_memory": "Kullanılan GPU belleği, sınıfa göre",
"dash.fleet.gpu_memory_note": "Motorlar başlarken bir kartın belleğinin çoğunu ayırır; bu yüzden bu grafik sunulan isteklerden çok yerleştirilen replikaları izler.",
"dash.fleet.queue": "Motor kuyruk derinliği, replikaya göre",
+5 -1
View File
@@ -76,7 +76,11 @@
"node.chart.engine_memory": "Motor belleği, modele göre",
"node.chart.engine_memory_note": "Diskten eşlenen ağırlıklar dâhil yerleşik bellek. Çekirdek o sayfaları geri alabilir, bu yüzden bu değer kullanılan RAM'i aşabilir.",
"node.chart.gpu_busy": "Meşgul GPU, karta göre",
"node.chart.gpu_busy_note": "Her örnekleme süresinde bir çekirdek işlevinin çalıştığı pay. Bir replikanın tuttuğu ve burada boşta görünen kart, kimsenin istemediği karttır.",
"node.chart.gpu_busy_note": "Her örnekleme süresinde bir çekirdek işlevinin çalıştığı pay — yonganın ne kadarını kullandığı değil: tek bir SM'nin meşgul olması %100 okunur. Bir replikanın tuttuğu ve burada boşta görünen kart, kimsenin istemediği karttır.",
"node.chart.gpu_power_load": "GPU güç yükü, karta göre",
"node.chart.gpu_power_load_note": "Çekilen gücün kartın güç sınırına oranı. Meşgul bir işlevin çalıştığını söyler; bu, yonganın ne kadarında çalıştığıyla artar.",
"node.chart.gpu_sm_active": "Etkin SM'ler, karta göre",
"node.chart.gpu_sm_active_note": "Kartın SM'lerinin iş yaptığı zaman payı, profil sayaçlarından. Meşgul %90 iken bunun %60 olması, bir işlev çalışırken yonganın üçte birinin boşta olduğu demektir.",
"node.chart.gpu_memory": "Kullanılan GPU belleği, karta göre",
"node.chart.gpu_memory_note": "Kullanımdaki çerçeve arabelleği: bir motorun ayırdığı ağırlıklar ve KV önbelleği, ayrıca llmgw'ye ait olmayan her şey.",
"node.chart.gpu_temp": "GPU sıcaklığı, °C",
+9 -2
View File
@@ -51,8 +51,11 @@ type nodeCharts struct {
CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries
// The GPUs', one line per card; empty on a node without them.
GPUUtil, GPUMemory, GPUTemp, GPUPower TimeSeries
Window string
Err string
// GPUPowerLoad is draw against the power limit; GPUSMActive the SMs'
// share of time with work, empty on a card nothing can profile.
GPUPowerLoad, GPUSMActive TimeSeries
Window string
Err string
}
// nodePanels are the charts, each a query scoped to this node. %s is the
@@ -89,6 +92,10 @@ var gpuPanels = []struct {
func(c *nodeCharts, t TimeSeries) { c.GPUTemp = t }},
{`llmgw_gpu_power_watts{node=%s}`, "",
func(c *nodeCharts, t TimeSeries) { c.GPUPower = t }},
{`llmgw_gpu_power_watts{node=%[1]s} / llmgw_gpu_power_limit_watts{node=%[1]s}`, "%",
func(c *nodeCharts, t TimeSeries) { c.GPUPowerLoad = t }},
{`llmgw_gpu_sm_active_percent{node=%s} / 100`, "%",
func(c *nodeCharts, t TimeSeries) { c.GPUSMActive = t }},
}
// NodePage shows one node. Its charts come from the metrics store when one
+7 -1
View File
@@ -64,11 +64,13 @@
{{/* Per card, because a model on eight cards making twice the tokens of one
on two is the worse of the two; per watt-hour, because that is what the
room is paying. */}}
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.eff.tps"}}</th><th class="num">{{t "dash.eff.per_gpu"}}</th><th class="num">{{t "dash.eff.busy"}}</th><th class="num">{{t "dash.eff.draw"}}</th><th class="num">{{t "dash.eff.per_wh"}}</th></tr>
<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.eff.tps"}}</th><th class="num">{{t "dash.eff.per_gpu"}}</th><th class="num">{{t "dash.eff.busy"}}</th><th class="num">{{t "dash.eff.power_load"}}</th><th class="num">{{t "dash.eff.sm_active"}}</th><th class="num">{{t "dash.eff.draw"}}</th><th class="num">{{t "dash.eff.per_wh"}}</th></tr>
{{range .}}<tr><td>{{.Model}}</td><td class="num">{{num .GPUs 0}}</td><td class="num">{{num .TokensPerSec 1}}</td>
<td class="num">{{num .PerGPU 1}}</td><td class="num">{{pc .Util 0}}</td>
<td class="num">{{if ge .PowerLoad 0.0}}{{pc .PowerLoad 0}}{{else}}—{{end}}</td><td class="num">{{if ge .SMActive 0.0}}{{pc .SMActive 0}}{{else}}—{{end}}</td>
<td class="num">{{if .Watts}}{{num .Watts 0}} W{{else}}—{{end}}</td><td class="num">{{if .PerWh}}{{num .PerWh 0}}{{else}}—{{end}}</td></tr>
{{end}}</table>
<p class="note">{{t "dash.eff.note"}}</p>
{{end}}
{{with .IdleGPUs}}
<h3>{{t "dash.idle.title"}}</h3>
@@ -282,6 +284,10 @@ document.querySelectorAll("form.build-choice select").forEach(function (s) {
{{template "tschart" dict "t" .GPUBusy "title" (t "dash.fleet.gpu_busy") "note" (t "dash.fleet.gpu_busy_note")}}
{{template "tschart" dict "t" .GPUMemory "title" (t "dash.fleet.gpu_memory") "note" (t "dash.fleet.gpu_memory_note")}}
</div>
<div class="chart-row">
{{template "tschart" dict "t" .GPUPowerLoad "title" (t "dash.fleet.gpu_power_load") "note" (t "dash.fleet.gpu_power_load_note")}}
{{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "dash.fleet.gpu_sm_active") "note" (t "dash.fleet.gpu_sm_active_note")}}{{end}}
</div>
{{end}}
<div class="chart-row">
{{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "dash.fleet.queue_note")}}
+4
View File
@@ -141,6 +141,10 @@
{{template "tschart" dict "t" .GPUTemp "title" (t "node.chart.gpu_temp") "note" (t "node.chart.gpu_temp_note")}}
{{template "tschart" dict "t" .GPUPower "title" (t "node.chart.gpu_power") "note" (t "node.chart.gpu_power_note")}}
</div>
<div class="chart-row">
{{template "tschart" dict "t" .GPUPowerLoad "title" (t "node.chart.gpu_power_load") "note" (t "node.chart.gpu_power_load_note")}}
{{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "node.chart.gpu_sm_active") "note" (t "node.chart.gpu_sm_active_note")}}{{end}}
</div>
{{end}}
<div class="chart-row">
{{template "tschart" dict "t" .Load "title" (t "node.chart.load") "note" (t "node.chart.load_note")}}