How much of each GPU works, beside whether anything does
CI / test (push) Failing after 6m42s

The busy figure is nvidia-smi's utilization.gpu: the share of time any
kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy
while the SMs had work 69% of the time and issued on 18% of their cycles.

Busy is now said to be that, and beside it: each card's draw against its
power limit (efficiency table, fleet and node charts, the console's GPU
work chart), and where the profiling counters can be read, SMs active, SM
occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's
GPM on Hopper and later or a dcgm-exporter on the node
(discovery.dcgm_exporter). The GPU map colours by work by default and keeps
busy as a mode. A MIG instance is never published with its card's profile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-31 20:11:26 +01:00
1 parent 7bfe70c8df
commit c866b3a8ee
34 files changed
+954 -52

No files matched your search

+14 -1
View File
@@ -772,12 +772,25 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(),
MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(),
SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(),
Wear: wearFrom(g),
Wear: wearFrom(g), Profile: profileFrom(g),
})
}
return out
}
// profileFrom is a card's profiling counters, or nil from an agent that
// sent none.
func profileFrom(g *llmgwv1.GPUSensor) *inventory.GPUProfile {
if !g.GetHasProfile() {
return nil
}
return &inventory.GPUProfile{
SMActive: g.GetSmActivePercent(), SMOccupancy: g.GetSmOccupancyPercent(),
TensorActive: g.GetTensorActivePercent(), DRAMActive: g.GetDramActivePercent(),
Source: g.GetProfileSource(),
}
}
func (s *Service) attach(ctx context.Context, nodeID string) *conn {
c := &conn{
nodeID: nodeID,
+35 -4
View File
@@ -84,6 +84,10 @@ type Collector struct {
gpuXID *prometheus.Desc
gpuLastXID *prometheus.Desc
gpuMemUtil *prometheus.Desc
gpuSMActive *prometheus.Desc
gpuSMOccupancy *prometheus.Desc
gpuTensor *prometheus.Desc
gpuDRAMActive *prometheus.Desc
gpuSMClock *prometheus.Desc
gpuTemp *prometheus.Desc
gpuMemTemp *prometheus.Desc
@@ -158,9 +162,22 @@ func New(src Source) *Collector {
gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes",
"A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil),
gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent",
"How busy a GPU was over the last sample period. Absent when the driver does not "+
"report it, which is not the same as idle. Memory used says a model is loaded; "+
"this says whether it is doing anything.", gpu, nil),
"The share of the last sample period in which any kernel ran on a GPU (nvidia-smi's "+
"utilization.gpu). Not how much of the chip: a kernel on one SM reads 100%. Absent "+
"when the driver does not report it, which is not the same as idle. Memory used says "+
"a model is loaded; this says whether it is doing anything, and "+
"llmgw_gpu_sm_active_percent how much.", gpu, nil),
gpuSMActive: prometheus.NewDesc("llmgw_gpu_sm_active_percent",
"The share of time a GPU's SMs had work, averaged over all of them: how much of the chip "+
"the busy figure is using. From NVML's GPM (Hopper and later) or a dcgm-exporter on the node; "+
"absent elsewhere.", gpu, nil),
gpuSMOccupancy: prometheus.NewDesc("llmgw_gpu_sm_occupancy_percent",
"How full a GPU's active SMs were with warps, of the most they can hold.", gpu, nil),
gpuTensor: prometheus.NewDesc("llmgw_gpu_tensor_active_percent",
"The share of time a GPU's tensor cores were working. Low in decode at batch one is normal; "+
"prefill and batches are where it should rise.", gpu, nil),
gpuDRAMActive: prometheus.NewDesc("llmgw_gpu_dram_active_percent",
"The share of time a GPU's memory interface was moving data, from its profiling counters.", gpu, nil),
gpuPCIeReplay: prometheus.NewDesc("llmgw_gpu_pcie_replays",
"PCIe link retries on a GPU's link since the driver loaded; one that keeps climbing is a riser, slot or cable.", gpu, nil),
gpuRetired: prometheus.NewDesc("llmgw_gpu_retired_pages",
@@ -310,7 +327,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuNVLJoined, c.imexUp,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock,
c.gpuSMActive, c.gpuSMOccupancy, c.gpuTensor, c.gpuDRAMActive, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
c.gpuPCIeReplay, c.gpuRetired, c.gpuRetirePend, c.gpuRemapped, c.gpuRemapPend, c.gpuRemapFail,
c.gpuNVLinkErr, c.gpuXID, c.gpuLastXID,
c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
@@ -653,6 +671,19 @@ func (c *Collector) gpuBusy(ch chan<- prometheus.Metric, s inventory.GPUSensor,
if s.SMClockMHz > 0 {
ch <- prometheus.MustNewConstMetric(c.gpuSMClock, prometheus.GaugeValue, s.SMClockMHz, labels...)
}
// The card's profile, here rather than with the sensors because it is
// the whole card's: a MIG instance must not be published as having it.
if p := s.Profile; p != nil {
for _, v := range []struct {
d *prometheus.Desc
val float64
}{{c.gpuSMActive, p.SMActive}, {c.gpuSMOccupancy, p.SMOccupancy}, {c.gpuTensor, p.TensorActive}, {c.gpuDRAMActive, p.DRAMActive}} {
// Zero is an idle chip; only -1, not reported, is left out.
if v.val >= 0 {
ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
}
}
}
}
func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {
+6 -1
View File
@@ -16,7 +16,8 @@ func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}}
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst},
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250,
UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920}}}}
UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920,
Profile: &inventory.GPUProfile{SMActive: 50, SMOccupancy: 30, TensorActive: 10, DRAMActive: 20}}}}}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1"
for _, want := range []string{
@@ -33,4 +34,8 @@ func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) {
t.Errorf("a utilisation was published for a MIG instance:\n%s", out)
}
// The card's profile is the whole card's, not this instance's.
if strings.Contains(out, "llmgw_gpu_sm_active_percent "+labels) {
t.Errorf("the card's SM activity was published for a MIG instance:\n%s", out)
}
}
@@ -127,3 +127,27 @@ func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) {
t.Errorf("a count the card does not keep was published:\n%s", out)
}
}
func TestHowMuchOfTheChipIsPublishedBesideHowBusy(t *testing.T) {
// The Quadro RTX 6000 in decode: 92% busy, its SMs at work 69% of the
// time. The DRAM figure was not read, and is left out rather than zero.
n := inventory.Node{ID: "g1", State: inventory.Ready,
GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}},
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, UtilPercent: 92, MemoryTotalMB: 81559,
Profile: &inventory.GPUProfile{SMActive: 69, SMOccupancy: 45, TensorActive: 0, DRAMActive: -1, Source: "gpm"}}}}}
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
for _, want := range []string{
"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 92\n",
"llmgw_gpu_sm_active_percent class=h100-80g gpu=0 node=g1 -> 69\n",
"llmgw_gpu_sm_occupancy_percent class=h100-80g gpu=0 node=g1 -> 45\n",
// Idle tensor cores are a reading.
"llmgw_gpu_tensor_active_percent class=h100-80g gpu=0 node=g1 -> 0\n",
} {
if !strings.Contains(out, want) {
t.Errorf("missing %s in:\n%s", want, out)
}
}
if strings.Contains(out, "llmgw_gpu_dram_active_percent") {
t.Errorf("a figure not read was published:\n%s", out)
}
}
+30
View File
@@ -340,6 +340,36 @@ type GPUSensor struct {
// Wear is what wears the card out, read every minute; nil from an agent
// that does not read it.
Wear *GPUWear `json:"wear,omitempty"`
// Profile is how much of the chip the work uses, which UtilPercent does
// not say; nil where nothing on the node can read it.
Profile *GPUProfile `json:"profile,omitempty"`
}
// GPUProfile is a card's profiling counters, in percent; -1 is not reported.
// UtilPercent is the share of time any kernel ran, and a kernel on one SM of
// a hundred counts in full; these say how much of the chip it ran on.
type GPUProfile struct {
// SMActive is the share of time the SMs had work, averaged over all of
// them; SMOccupancy how full those SMs were with warps.
SMActive float64 `json:"sm_active"`
SMOccupancy float64 `json:"sm_occupancy"`
// TensorActive is the tensor cores, DRAMActive the memory interface.
TensorActive float64 `json:"tensor_active"`
DRAMActive float64 `json:"dram_active"`
// Source is what read them: "gpm" (NVML, Hopper and later) or "dcgm".
Source string `json:"source,omitempty"`
}
// PowerLoad is the card's draw as a share of its power limit, from 0 to 1,
// or -1 without both figures. Unlike UtilPercent it rises with how much of
// the chip is working, not only with whether anything is: a card reading
// 92% busy drew 239 W of 260 W in decode and 112 W when the busy figure was
// a kernel or two between requests.
func (g GPUSensor) PowerLoad() float64 {
if g.PowerW <= 0 || g.PowerLimitW <= 0 {
return -1
}
return min(g.PowerW/g.PowerLimitW, 1)
}
// GPUWear is a card's wear and failures. Counts of -1 are not reported for