The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7bfe70c8df
commit
c866b3a8ee
34 files changed
+954
-52
No files matched your search
@@ -1497,6 +1497,10 @@ func (m consoleModel) gpuRows() [][]string {
|
||||
}
|
||||
if !mig {
|
||||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||||
if p := s.Profile; p != nil && p.SMActive >= 0 {
|
||||
// How much of the chip, beside whether anything ran.
|
||||
busy += fmt.Sprintf(" · SM %.0f%%", p.SMActive)
|
||||
}
|
||||
if s.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
|
||||
@@ -80,6 +81,9 @@ func (s *monitorState) sample(d consoleData) {
|
||||
for _, g := range l.GPUs {
|
||||
card := nd.ID + ":" + strconv.Itoa(g.Index)
|
||||
add("gpu/"+card, g.UtilPercent)
|
||||
if w := gpuWork(g); w >= 0 {
|
||||
add("gpuwork/"+card, w)
|
||||
}
|
||||
if g.MemoryTotalMB > 0 {
|
||||
add("vram/"+card, 100*g.MemoryUsedMB/g.MemoryTotalMB)
|
||||
}
|
||||
@@ -142,6 +146,7 @@ func monitorQueries(window time.Duration, points int) (time.Duration, []adminapi
|
||||
{ID: "cpu", Expr: `avg(llmgw_node_cpu_utilization_percent)`},
|
||||
{ID: "mem", Expr: `100 * sum(llmgw_node_memory_used_bytes) / sum(llmgw_node_memory_bytes)`},
|
||||
{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent`), By: "card"},
|
||||
{ID: "gpuwork", Expr: card(gpuWorkExpr("")), By: "card"},
|
||||
{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes`), By: "card"},
|
||||
{ID: "gputemp", Expr: card(`llmgw_gpu_temperature_celsius`), By: "card"},
|
||||
{ID: "gpupow", Expr: card(`llmgw_gpu_power_watts`), By: "card"},
|
||||
@@ -314,7 +319,8 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) {
|
||||
b.WriteString(indent(side(w, box("CPU", pct("cpu"), m.series("cpu"), 100),
|
||||
box("Memory", pct("mem"), m.series("mem"), 100)), " ") + "\n\n")
|
||||
if g := m.series("gpu"); len(g) > 0 || len(m.data.sum.GPUs) > 0 {
|
||||
b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", m.series("vram"), 100)), " ") + "\n\n")
|
||||
b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU work", "", m.series("gpuwork"), 100)), " ") + "\n\n")
|
||||
b.WriteString(indent(side(w, box("GPU memory", "", m.series("vram"), 100), " "+sDim.Render(gpuWorkNote)), " ") + "\n\n")
|
||||
}
|
||||
if m.mon.res.Configured {
|
||||
rps := m.series("rps")
|
||||
@@ -356,10 +362,12 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) {
|
||||
// Each card: busy, memory, heat and power.
|
||||
if gpus := m.series("gpu"); len(gpus) > 0 {
|
||||
vram, temp, pow := byName(m.series("vram")), byName(m.series("gputemp")), byName(m.series("gpupow"))
|
||||
work := byName(m.series("gpuwork"))
|
||||
b.WriteString("\n " + mdBold.Render("GPUs") + "\n")
|
||||
for _, g := range gpus {
|
||||
fmt.Fprintf(&b, " %-16s busy %s %4.0f%% memory %s %4.0f%% %3.0f°C %s %4.0f W\n", truncate(g.name, 16),
|
||||
sparkSeries(g, sw/2, 100), last(g.values), sparkSeries(vram[g.name], sw/2, 100), last(vram[g.name].values),
|
||||
fmt.Fprintf(&b, " %-16s busy %s %4.0f%% work %s %4.0f%% memory %s %4.0f%% %3.0f°C %s %4.0f W\n", truncate(g.name, 16),
|
||||
sparkSeries(g, sw/3, 100), last(g.values), sparkSeries(work[g.name], sw/3, 100), last(work[g.name].values),
|
||||
sparkSeries(vram[g.name], sw/3, 100), last(vram[g.name].values),
|
||||
last(temp[g.name].values), sparkSeries(temp[g.name], 8, 100), last(pow[g.name].values))
|
||||
}
|
||||
}
|
||||
@@ -373,6 +381,29 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) {
|
||||
return "Monitor", head.String() + b.String(), keys
|
||||
}
|
||||
|
||||
// GPU busy is the driver's figure: the share of time any kernel ran, which
|
||||
// a kernel on one SM of a hundred fills. GPU work is how much of the chip:
|
||||
// the SMs' share of time with work where the card's profiling counters are
|
||||
// read (GPM, DCGM), and its draw against its power limit elsewhere.
|
||||
const gpuWorkNote = "busy: some kernel ran. work: how much of the chip —\nSMs active where the counters are read, power/limit elsewhere."
|
||||
|
||||
// gpuWorkExpr is GPU work in PromQL, for the series matching sel.
|
||||
func gpuWorkExpr(sel string) string {
|
||||
return `llmgw_gpu_sm_active_percent` + sel + ` or (100 * llmgw_gpu_power_watts` + sel +
|
||||
` / llmgw_gpu_power_limit_watts` + sel + `)`
|
||||
}
|
||||
|
||||
// gpuWork is GPU work from a sample, or -1 without the figures for it.
|
||||
func gpuWork(g inventory.GPUSensor) float64 {
|
||||
if p := g.Profile; p != nil && p.SMActive >= 0 {
|
||||
return p.SMActive
|
||||
}
|
||||
if l := g.PowerLoad(); l >= 0 {
|
||||
return 100 * l
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func named(s []chartSeries, name string) []chartSeries {
|
||||
for i := range s {
|
||||
s[i].name = name
|
||||
@@ -410,6 +441,7 @@ func detailQueries(kind, id string, window time.Duration, points int) (time.Dura
|
||||
{ID: "cpu", Expr: `llmgw_node_cpu_utilization_percent{node=` + q + `}`},
|
||||
{ID: "mem", Expr: `100 * llmgw_node_memory_used_bytes{node=` + q + `} / llmgw_node_memory_bytes{node=` + q + `}`},
|
||||
{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent{node=` + q + `}`), By: "card"},
|
||||
{ID: "gpuwork", Expr: card(gpuWorkExpr(`{node=` + q + `}`)), By: "card"},
|
||||
{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes{node=` + q + `} / llmgw_gpu_memory_total_bytes{node=` + q + `}`), By: "card"},
|
||||
{ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
|
||||
{ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
|
||||
@@ -520,7 +552,8 @@ func (m consoleModel) detailCharts(pw int) string {
|
||||
b.WriteString(indent(side(w, box("CPU", fmt.Sprintf("%.0f%%", lastOf(cpu)), cpu, 100),
|
||||
box("Memory", fmt.Sprintf("%.0f%%", lastOf(mem)), mem, 100)), " "))
|
||||
if g := get("gpu"); len(g) > 0 {
|
||||
b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", get("vram"), 100)), " "))
|
||||
b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU work", "", get("gpuwork"), 100)), " "))
|
||||
b.WriteString("\n\n" + indent(side(w, box("GPU memory", "", get("vram"), 100), " "+sDim.Render(gpuWorkNote)), " "))
|
||||
}
|
||||
if m.det.res.Configured {
|
||||
net := append(named(get("netin"), "in"), named(get("netout"), "out")...)
|
||||
|
||||
Reference in new issue
Block a user