How much of each GPU works, beside whether anything does

The busy figure is nvidia-smi's utilization.gpu: the share of time any
kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy
while the SMs had work 69% of the time and issued on 18% of their cycles.

Busy is now said to be that, and beside it: each card's draw against its
power limit (efficiency table, fleet and node charts, the console's GPU
work chart), and where the profiling counters can be read, SMs active, SM
occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's
GPM on Hopper and later or a dcgm-exporter on the node
(discovery.dcgm_exporter). The GPU map colours by work by default and keeps
busy as a mode. A MIG instance is never published with its card's profile.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -28,6 +28,24 @@ APIs change between minors without a deprecation period.
 
 ## Unreleased
 
+### Added
+
+- **How much of each GPU works, beside whether anything does.** The busy
+  figure is nvidia-smi's `utilization.gpu` — the share of time any kernel
+  ran, which a kernel on one SM fills — and it is now said to be that: on
+  the dashboard, the node page, the metric's help and Grafana. Beside it,
+  each card's draw against its power limit (the efficiency table, the fleet
+  and node charts, the console's GPU work chart), and where the card's
+  profiling counters can be read, its SMs, SM occupancy, tensor cores and
+  memory interface: `llmgw_gpu_sm_active_percent`,
+  `llmgw_gpu_sm_occupancy_percent`, `llmgw_gpu_tensor_active_percent`,
+  `llmgw_gpu_dram_active_percent`. The agent reads them from NVML's GPM on
+  Hopper and later, and from a dcgm-exporter on the node elsewhere
+  (`discovery.dcgm_exporter`, default `http://127.0.0.1:9400/metrics`, `off`
+  for none). The GPU map colours by this "work" by default — SMs active
+  where read, power against the limit otherwise — and keeps busy as a mode.
+  On cuda01 decode read 92% busy with the SMs at work 69% of the time.
+
 ### Fixed
 
 - **A model of several files showed one file's download as the whole.** A node fetching a set reported the part it

api/gen/llmgw/v1/agent.pb.go

@@ -2219,12 +2219,27 @@ type GPUSensor struct {
 	NvlinkReplayErrors        int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"`
 	NvlinkRecoveryErrors      int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"`
 	// xid_errors are the driver's error events since the agent started.
-	XidErrors     int64  `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
+	XidErrors   int64  `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"`
-	LastXid       int32  `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
+	LastXid     int32  `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"`
-	LastXidText   string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
+	LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"`
-	LastXidUnix   int64  `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
+	LastXidUnix int64  `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"`
-	unknownFields protoimpl.UnknownFields
+	// How much of the chip the work on it uses, which utilization_percent
-	sizeCache     protoimpl.SizeCache
+	// does not say: that is the share of time any kernel ran, and a kernel on
+	// one SM of a hundred counts in full. has_profile says these are present;
+	// -1 is not reported. Read from NVML's GPM on Hopper and later, or from a
+	// dcgm-exporter on the node; profile_source says which ("gpm", "dcgm").
+	HasProfile bool `protobuf:"varint,29,opt,name=has_profile,json=hasProfile,proto3" json:"has_profile,omitempty"`
+	// sm_active_percent is the share of time the SMs had work, averaged over
+	// all of them; sm_occupancy_percent how full those SMs were with warps.
+	SmActivePercent    float64 `protobuf:"fixed64,30,opt,name=sm_active_percent,json=smActivePercent,proto3" json:"sm_active_percent,omitempty"`
+	SmOccupancyPercent float64 `protobuf:"fixed64,31,opt,name=sm_occupancy_percent,json=smOccupancyPercent,proto3" json:"sm_occupancy_percent,omitempty"`
+	// tensor_active_percent is the tensor cores; dram_active_percent the
+	// memory interface, which decode is bound by.
+	TensorActivePercent float64 `protobuf:"fixed64,32,opt,name=tensor_active_percent,json=tensorActivePercent,proto3" json:"tensor_active_percent,omitempty"`
+	DramActivePercent   float64 `protobuf:"fixed64,33,opt,name=dram_active_percent,json=dramActivePercent,proto3" json:"dram_active_percent,omitempty"`
+	ProfileSource       string  `protobuf:"bytes,34,opt,name=profile_source,json=profileSource,proto3" json:"profile_source,omitempty"`
+	unknownFields       protoimpl.UnknownFields
+	sizeCache           protoimpl.SizeCache
 }
 
 func (x *GPUSensor) Reset() {
@@ -2453,6 +2468,48 @@ func (x *GPUSensor) GetLastXidUnix() int64 {
 	return 0
 }
 
+func (x *GPUSensor) GetHasProfile() bool {
+	if x != nil {
+		return x.HasProfile
+	}
+	return false
+}
+
+func (x *GPUSensor) GetSmActivePercent() float64 {
+	if x != nil {
+		return x.SmActivePercent
+	}
+	return 0
+}
+
+func (x *GPUSensor) GetSmOccupancyPercent() float64 {
+	if x != nil {
+		return x.SmOccupancyPercent
+	}
+	return 0
+}
+
+func (x *GPUSensor) GetTensorActivePercent() float64 {
+	if x != nil {
+		return x.TensorActivePercent
+	}
+	return 0
+}
+
+func (x *GPUSensor) GetDramActivePercent() float64 {
+	if x != nil {
+		return x.DramActivePercent
+	}
+	return 0
+}
+
+func (x *GPUSensor) GetProfileSource() string {
+	if x != nil {
+		return x.ProfileSource
+	}
+	return ""
+}
+
 // ShardLoad is one engine process group's use.
 type ShardLoad struct {
 	state   protoimpl.MessageState `protogen:"open.v1"`
@@ -5377,7 +5434,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	" \x01(\x01R\tcpuPowerW\x12 \n" +
 	"\finlet_temp_c\x18\v \x01(\x01R\n" +
 	"inletTempC\x12'\n" +
-	"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" +
+	"\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xe8\n" +
+	"\n" +
 	"\tGPUSensor\x12\x14\n" +
 	"\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" +
 	"\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" +
@@ -5409,7 +5467,14 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" +
 	"\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" +
 	"\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" +
-	"\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" +
+	"\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\x12\x1f\n" +
+	"\vhas_profile\x18\x1d \x01(\bR\n" +
+	"hasProfile\x12*\n" +
+	"\x11sm_active_percent\x18\x1e \x01(\x01R\x0fsmActivePercent\x120\n" +
+	"\x14sm_occupancy_percent\x18\x1f \x01(\x01R\x12smOccupancyPercent\x122\n" +
+	"\x15tensor_active_percent\x18  \x01(\x01R\x13tensorActivePercent\x12.\n" +
+	"\x13dram_active_percent\x18! \x01(\x01R\x11dramActivePercent\x12%\n" +
+	"\x0eprofile_source\x18\" \x01(\tR\rprofileSource\"m\n" +
 	"\tShardLoad\x12\x19\n" +
 	"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
 	"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +

api/proto/llmgw/v1/agent.proto

@@ -448,6 +448,21 @@ message GPUSensor {
   int32 last_xid = 26;
   string last_xid_text = 27;
   int64 last_xid_unix = 28;
+  // How much of the chip the work on it uses, which utilization_percent
+  // does not say: that is the share of time any kernel ran, and a kernel on
+  // one SM of a hundred counts in full. has_profile says these are present;
+  // -1 is not reported. Read from NVML's GPM on Hopper and later, or from a
+  // dcgm-exporter on the node; profile_source says which ("gpm", "dcgm").
+  bool has_profile = 29;
+  // sm_active_percent is the share of time the SMs had work, averaged over
+  // all of them; sm_occupancy_percent how full those SMs were with warps.
+  double sm_active_percent = 30;
+  double sm_occupancy_percent = 31;
+  // tensor_active_percent is the tensor cores; dram_active_percent the
+  // memory interface, which decode is bound by.
+  double tensor_active_percent = 32;
+  double dram_active_percent = 33;
+  string profile_source = 34;
 }
 
 // ShardLoad is one engine process group's use.

deploy/monitoring/grafana/llmgw-overview.json

@@ -504,7 +504,7 @@
     {
       "type": "timeseries",
       "title": "GPU busy, by card",
-      "description": "Share of each sample period a kernel was running, sampled every heartbeat.",
+      "description": "Share of each sample period in which any kernel ran (nvidia-smi's utilization.gpu), sampled every heartbeat. Not how much of the chip: one SM busy reads 100%. The SMs panel and power against its limit say how much.",
       "datasource": {
         "type": "prometheus",
         "uid": "llmgw-prom"
@@ -908,6 +908,51 @@
           "mode": "multi"
         }
       }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU SMs and tensor cores active, by card",
+      "description": "From the cards' profiling counters: NVML's GPM on Hopper and later, or a dcgm-exporter on the node. SMs active is the share of time the SMs had work, averaged over all of them; the gap to busy is chip a running kernel leaves idle. Tensor cores stay low in decode at batch one; prefill and batches are where they rise. Empty for cards nothing can profile.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 73,
+        "w": 24,
+        "h": 7
+      },
+      "id": 25,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_sm_active_percent",
+          "legendFormat": "{{node}} gpu {{gpu}} SMs",
+          "refId": "A"
+        },
+        {
+          "expr": "llmgw_gpu_tensor_active_percent",
+          "legendFormat": "{{node}} gpu {{gpu}} tensor",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
     }
   ]
 }

docs/admin-api.openapi.json

@@ -11019,6 +11019,30 @@
           }
         }
       },
+      "inventory.GPUProfile": {
+        "type": "object",
+        "description": "GPUProfile is a card's profiling counters, in percent; -1 is not reported. UtilPercent is the share of time any kernel ran, and a kernel on one SM of a hundred counts in full; these say how much of the chip it ran on.",
+        "properties": {
+          "sm_active": {
+            "type": "number",
+            "description": "SMActive is the share of time the SMs had work, averaged over all of them; SMOccupancy how full those SMs were with warps."
+          },
+          "sm_occupancy": {
+            "type": "number"
+          },
+          "tensor_active": {
+            "type": "number",
+            "description": "TensorActive is the tensor cores, DRAMActive the memory interface."
+          },
+          "dram_active": {
+            "type": "number"
+          },
+          "source": {
+            "type": "string",
+            "description": "Source is what read them: \"gpm\" (NVML, Hopper and later) or \"dcgm\"."
+          }
+        }
+      },
       "inventory.GPUSensor": {
         "type": "object",
         "description": "GPUSensor is one card's heat and power, and what is throttling it.",
@@ -11066,6 +11090,10 @@
           "wear": {
             "$ref": "#/components/schemas/inventory.GPUWear",
             "description": "Wear is what wears the card out, read every minute; nil from an agent that does not read it."
+          },
+          "profile": {
+            "$ref": "#/components/schemas/inventory.GPUProfile",
+            "description": "Profile is how much of the chip the work uses, which UtilPercent does not say; nil where nothing on the node can read it."
           }
         }
       },

docs/install.md

@@ -2019,12 +2019,31 @@ dashboard, is in [deploy/monitoring/](../deploy/monitoring/) — including where
 to run it, which is not the control plane: monitoring that dies with the thing
 it monitors tells you nothing at the moment you need it.
 
-Two figures repay knowing about. `llmgw_gpu_utilization_percent` is absent
+Three figures repay knowing about. `llmgw_gpu_utilization_percent` is absent
 rather than zero when the driver does not report it — a flat zero across a
 fleet means the measurement is missing, not that the fleet is idle. And
 `llmgw_gpu_nvlink_peers` at zero everywhere means the topology is unknown,
 which is what a binary built without the nvml tag reports.
 
+The third is what "busy" means. `llmgw_gpu_utilization_percent` is
+nvidia-smi's `utilization.gpu`: the share of time *any* kernel ran, however
+little of the chip it ran on — a kernel on one SM of 142 reads 100%. A Quadro
+RTX 6000 decoding read 92% busy while its SMs had work 69% of the time and
+issued an instruction on 18% of their cycles. How much of the chip is at work
+is in `llmgw_gpu_sm_active_percent`, `llmgw_gpu_sm_occupancy_percent`,
+`llmgw_gpu_tensor_active_percent` and `llmgw_gpu_dram_active_percent`, from
+the card's profiling counters. The agent reads them from NVML's GPM on
+Hopper and later, with nothing to install; on older cards it reads them from
+a [dcgm-exporter](https://github.com/NVIDIA/dcgm-exporter) on the node, at
+`http://127.0.0.1:9400/metrics` unless `discovery.dcgm_exporter` in
+agent.yaml says otherwise (`off` reads none). Where neither is there, the
+dashboard's GPU map and the console colour cards by their draw against the
+power limit (`llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts`), which
+rises with how much of the chip works rather than with whether anything
+does. Low tensor-core activity in decode is not waste: a quantised
+matrix-vector product at batch one runs on the integer units, and it is
+prefill and batches that use the tensor cores.
+
 ### Token by token, and SLOs
 
 Beside time to the first token, every streamed answer is timed per token:

docs/reference.md

@@ -234,6 +234,7 @@ rather than repeated.
 | `ib_root` | `string` | [Options](#options) |
 | `prefer_nvml` | `bool` | [Options](#options) |
 | `skip_gpus` | `bool` | [Options](#options) |
+| `dcgm_exporter` | `string` | [Options](#options) |
 | `repo` | `string` | [Origin](#origin) |
 | `revision` | `string` | [Origin](#origin) |
 | `files` | `[]OriginFile` | [Origin](#origin) |
@@ -2182,6 +2183,15 @@ is used, is the more exact source of the peers.
 SkipGPUs omits GPU discovery. A node without nvidia-smi is already
 found to have none, so this is for one whose driver is broken.
 
+### `dcgm_exporter`
+
+`string`
+
+DCGMExporter is the dcgm-exporter metrics URL read for how much of
+each card's chip is in use, on cards NVML's GPM does not cover (before
+Hopper). Empty is http://127.0.0.1:9400/metrics; "off" reads none. A
+node without one costs a refused connection every five minutes.
+
 ## Origin
 
 ### `repo`

internal/agent/agent.go

@@ -187,7 +187,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
 		logRing:      ring,
 		provKick:     make(chan struct{}, 1),
 		load:         discovery.NewLoadSampler(),
-		sensors:      discovery.NewSensorSampler(),
+		sensors:      discovery.NewSensorSampler().WithProfile(discovery.NewProfSampler(cfg.Discovery.DCGMExporter)),
 		wear:         discovery.NewHealthSampler(),
 		serverList:   loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
 		kv:           kvevents.NewMirror(),
@@ -798,6 +798,11 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
 				MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB,
 				SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected,
 			}
+			if p := g.Profile; p != nil {
+				gs.HasProfile, gs.ProfileSource = true, p.Source
+				gs.SmActivePercent, gs.SmOccupancyPercent = p.SMActive, p.SMOccupancy
+				gs.TensorActivePercent, gs.DramActivePercent = p.TensorActive, p.DRAMActive
+			}
 			if w, ok := wear[g.Index]; ok {
 				gs.HasWear = true
 				gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending =

internal/agent/discovery/discover.go

@@ -28,6 +28,11 @@ type Options struct {
 	// SkipGPUs omits GPU discovery. A node without nvidia-smi is already
 	// found to have none, so this is for one whose driver is broken.
 	SkipGPUs bool `koanf:"skip_gpus"`
+	// DCGMExporter is the dcgm-exporter metrics URL read for how much of
+	// each card's chip is in use, on cards NVML's GPM does not cover (before
+	// Hopper). Empty is http://127.0.0.1:9400/metrics; "off" reads none. A
+	// node without one costs a refused connection every five minutes.
+	DCGMExporter string `koanf:"dcgm_exporter"`
 	// ModelDirs are scanned for weights. Empty means do not look.
 	ModelDirs []string `koanf:"-"`
 }

internal/agent/discovery/gpm_nonvml.go

@@ -0,0 +1,6 @@
+//go:build !nvml
+
+package discovery
+
+// newGPM has nothing to read GPM with in a build without the nvml tag.
+func newGPM() gpmSource { return nil }

internal/agent/discovery/gpm_nvml.go

@@ -0,0 +1,124 @@
+//go:build nvml
+
+package discovery
+
+import (
+	"sync"
+
+	"github.com/NVIDIA/go-nvml/pkg/nvml"
+)
+
+// nvmlGPM reads GPM, NVML's own profiling counters, on the cards that have
+// them: Hopper and later. Each read takes a sample and works out the metrics
+// against the one before, so NVML stays initialised between reads.
+type nvmlGPM struct {
+	mu     sync.Mutex
+	opened bool
+	failed bool
+	cards  map[int]*gpmCard
+}
+
+type gpmCard struct {
+	dev          nvml.Device
+	prev, cur    nvml.GpmSample
+	havePrevious bool
+}
+
+func newGPM() gpmSource { return &nvmlGPM{} }
+
+// The GPM metrics read, in the order they are asked for.
+var gpmMetrics = []struct {
+	id  nvml.GpmMetricId
+	set func(*GPUProfile, float64)
+}{
+	{nvml.GPM_METRIC_SM_UTIL, func(p *GPUProfile, v float64) { p.SMActive = v }},
+	{nvml.GPM_METRIC_SM_OCCUPANCY, func(p *GPUProfile, v float64) { p.SMOccupancy = v }},
+	{nvml.GPM_METRIC_ANY_TENSOR_UTIL, func(p *GPUProfile, v float64) { p.TensorActive = v }},
+	{nvml.GPM_METRIC_DRAM_BW_UTIL, func(p *GPUProfile, v float64) { p.DRAMActive = v }},
+}
+
+func (g *nvmlGPM) read() map[int]GPUProfile {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.failed {
+		return nil
+	}
+	if !g.opened && !g.open() {
+		g.failed = true
+		return nil
+	}
+	out := map[int]GPUProfile{}
+	for idx, c := range g.cards {
+		if c.cur.Get(c.dev) != nvml.SUCCESS {
+			// A card in MIG mode, or one that has gone: no sample, and the
+			// next must not be measured against an old one.
+			c.havePrevious = false
+			continue
+		}
+		if c.havePrevious {
+			if p, ok := gpmBetween(c.prev, c.cur); ok {
+				out[idx] = p
+			}
+		}
+		c.prev, c.cur = c.cur, c.prev
+		c.havePrevious = true
+	}
+	return out
+}
+
+// open finds the cards with GPM. False is NVML not starting, or no card
+// having it, and the reader then never tries again.
+func (g *nvmlGPM) open() bool {
+	if nvml.Init() != nvml.SUCCESS {
+		return false
+	}
+	g.cards = map[int]*gpmCard{}
+	n, ret := nvml.DeviceGetCount()
+	if ret != nvml.SUCCESS {
+		n = 0
+	}
+	for i := range n {
+		dev, ret := nvml.DeviceGetHandleByIndex(i)
+		if ret != nvml.SUCCESS {
+			continue
+		}
+		sup, ret := dev.GpmQueryDeviceSupport()
+		if ret != nvml.SUCCESS || sup.IsSupportedDevice == 0 {
+			continue
+		}
+		a, ret := nvml.GpmSampleAlloc()
+		if ret != nvml.SUCCESS {
+			continue
+		}
+		b, ret := nvml.GpmSampleAlloc()
+		if ret != nvml.SUCCESS {
+			_ = a.Free()
+			continue
+		}
+		g.cards[i] = &gpmCard{dev: dev, prev: a, cur: b}
+	}
+	if len(g.cards) == 0 {
+		_ = nvml.Shutdown()
+		return false
+	}
+	g.opened = true
+	return true
+}
+
+// gpmBetween is the profile over the time between two samples.
+func gpmBetween(earlier, later nvml.GpmSample) (GPUProfile, bool) {
+	q := nvml.GpmMetricsGetType{NumMetrics: uint32(len(gpmMetrics)), Sample1: earlier, Sample2: later}
+	for i, m := range gpmMetrics {
+		q.Metrics[i].MetricId = uint32(m.id)
+	}
+	if nvml.GpmMetricsGet(&q) != nvml.SUCCESS {
+		return GPUProfile{}, false
+	}
+	p := GPUProfile{SMActive: -1, SMOccupancy: -1, TensorActive: -1, DRAMActive: -1, Source: "gpm"}
+	for i, m := range gpmMetrics {
+		if r := q.Metrics[i]; r.NvmlReturn == uint32(nvml.SUCCESS) && r.Value >= 0 {
+			m.set(&p, r.Value)
+		}
+	}
+	return p, true
+}

internal/agent/discovery/prof.go

@@ -0,0 +1,176 @@
+package discovery
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// GPUProfile is how much of a card's chip the work on it uses. nvidia-smi's
+// utilization.gpu cannot say: it is the share of time any kernel ran, and a
+// kernel keeping one SM of a hundred busy reads 100%. Measured on a Quadro
+// RTX 6000 under decode, it read 92% while the SMs had work 69% of the time
+// and issued an instruction on 18% of their cycles.
+//
+// These come from the card's profiling counters, which only two things read
+// without stopping the work: NVML's GPM, on Hopper and later, and DCGM, which
+// a site may run as dcgm-exporter. Percentages; -1 is not reported.
+type GPUProfile struct {
+	// SMActive is the share of time the SMs had work, averaged over all of
+	// them; SMOccupancy how full those SMs were with warps.
+	SMActive, SMOccupancy float64
+	// TensorActive is the tensor cores. Decode at batch one hardly uses them
+	// even when nothing is wrong: a quantised matrix-vector product runs on
+	// the integer units. Prefill and batches are where they show.
+	TensorActive float64
+	// DRAMActive is the memory interface, which decode is bound by.
+	DRAMActive float64
+	// Source is "gpm" or "dcgm".
+	Source string
+}
+
+// DefaultDCGMExporter is where dcgm-exporter listens unless told otherwise.
+const DefaultDCGMExporter = "http://127.0.0.1:9400/metrics"
+
+// dcgmRetry is how long a dcgm-exporter that did not answer is left alone.
+// Most nodes run none, and asking every ten seconds would be asking nobody.
+const dcgmRetry = 5 * time.Minute
+
+// gpmSource reads GPM, where this build and the card have it.
+type gpmSource interface {
+	read() map[int]GPUProfile
+}
+
+// ProfSampler reads each card's GPUProfile: from GPM where the card has it,
+// otherwise from a dcgm-exporter on the node.
+type ProfSampler struct {
+	gpm  gpmSource
+	dcgm string
+	get  func(ctx context.Context, url string) ([]byte, error)
+	now  func() time.Time
+
+	dcgmAfter time.Time
+}
+
+// NewProfSampler reads this machine. dcgmURL is the dcgm-exporter to ask:
+// empty is DefaultDCGMExporter, "off" asks none.
+func NewProfSampler(dcgmURL string) *ProfSampler {
+	switch dcgmURL {
+	case "":
+		dcgmURL = DefaultDCGMExporter
+	case "off":
+		dcgmURL = ""
+	}
+	return &ProfSampler{gpm: newGPM(), dcgm: dcgmURL, get: httpGet, now: time.Now}
+}
+
+// Sample is each card's profile by index; a card missing has none. GPM needs
+// two samples, so its cards appear from the second call.
+func (p *ProfSampler) Sample(ctx context.Context) map[int]GPUProfile {
+	out := map[int]GPUProfile{}
+	if p.gpm != nil {
+		for i, v := range p.gpm.read() {
+			out[i] = v
+		}
+	}
+	if p.dcgm == "" || p.now().Before(p.dcgmAfter) {
+		return out
+	}
+	body, err := p.get(ctx, p.dcgm)
+	if err != nil {
+		p.dcgmAfter = p.now().Add(dcgmRetry)
+		return out
+	}
+	for i, v := range parseDCGM(body) {
+		if _, ok := out[i]; !ok {
+			out[i] = v
+		}
+	}
+	return out
+}
+
+func httpGet(ctx context.Context, url string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return nil, io.ErrUnexpectedEOF
+	}
+	return io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+}
+
+// The dcgm-exporter fields read, as ratios of one. Its default field list
+// has them on every card DCGM can profile.
+var dcgmFields = map[string]func(*GPUProfile, float64){
+	"DCGM_FI_PROF_SM_ACTIVE":          func(p *GPUProfile, v float64) { p.SMActive = v },
+	"DCGM_FI_PROF_SM_OCCUPANCY":       func(p *GPUProfile, v float64) { p.SMOccupancy = v },
+	"DCGM_FI_PROF_PIPE_TENSOR_ACTIVE": func(p *GPUProfile, v float64) { p.TensorActive = v },
+	"DCGM_FI_PROF_DRAM_ACTIVE":        func(p *GPUProfile, v float64) { p.DRAMActive = v },
+}
+
+// parseDCGM reads dcgm-exporter's text exposition: a line per field and
+// card, the card in the gpu label. A card with none of the fields — DCGM
+// cannot profile it — is left out. A MIG instance's lines carry a
+// GPU_I_ID and are left out too: these are whole cards.
+func parseDCGM(body []byte) map[int]GPUProfile {
+	out := map[int]GPUProfile{}
+	for _, line := range strings.Split(string(body), "\n") {
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		open := strings.IndexByte(line, '{')
+		end := strings.LastIndexByte(line, '}')
+		if open < 0 || end < open {
+			continue
+		}
+		set, ok := dcgmFields[line[:open]]
+		if !ok {
+			continue
+		}
+		labels := line[open+1 : end]
+		if strings.Contains(labels, `GPU_I_ID="`) && !strings.Contains(labels, `GPU_I_ID=""`) {
+			continue
+		}
+		idx, err := strconv.Atoi(labelValue(labels, "gpu"))
+		if err != nil {
+			continue
+		}
+		val := strings.Fields(line[end+1:])
+		if len(val) == 0 {
+			continue
+		}
+		v, err := strconv.ParseFloat(val[0], 64)
+		if err != nil || v < 0 {
+			continue
+		}
+		p, seen := out[idx]
+		if !seen {
+			p = GPUProfile{SMActive: -1, SMOccupancy: -1, TensorActive: -1, DRAMActive: -1, Source: "dcgm"}
+		}
+		set(&p, 100*v)
+		out[idx] = p
+	}
+	return out
+}
+
+// labelValue is one label's value from a Prometheus label set.
+func labelValue(labels, name string) string {
+	for _, kv := range strings.Split(labels, ",") {
+		k, v, ok := strings.Cut(strings.TrimSpace(kv), "=")
+		if ok && k == name {
+			return strings.Trim(v, `"`)
+		}
+	}
+	return ""
+}

internal/agent/discovery/prof_test.go

@@ -0,0 +1,107 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+// dcgmBody is dcgm-exporter's output for two cards and a MIG instance:
+// card 0 profiled, card 1 a model DCGM cannot profile, which has the device
+// fields and none of the PROF ones.
+const dcgmBody = `# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
+# TYPE DCGM_FI_DEV_GPU_UTIL gauge
+DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 100
+DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-b",device="nvidia1",modelName="Quadro RTX 6000",Hostname="n1"} 92
+# HELP DCGM_FI_PROF_SM_ACTIVE The ratio of cycles an SM has at least 1 warp assigned.
+# TYPE DCGM_FI_PROF_SM_ACTIVE gauge
+DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.007042
+DCGM_FI_PROF_SM_ACTIVE{gpu="2",UUID="GPU-c",device="nvidia2",modelName="NVIDIA A100",GPU_I_PROFILE="3g.40gb",GPU_I_ID="1",Hostname="n1"} 0.5
+DCGM_FI_PROF_SM_OCCUPANCY{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.25
+DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0
+`
+
+func TestDCGMExporterSaysHowMuchOfEachChipIsInUse(t *testing.T) {
+	got := parseDCGM([]byte(dcgmBody))
+	if len(got) != 1 {
+		t.Fatalf("want card 0 alone (card 1 unprofiled, card 2 a MIG instance): %+v", got)
+	}
+	p := got[0]
+	// The LinkedIn L40S: 100% busy, one SM of 142 at work.
+	if p.SMActive < 0.70 || p.SMActive > 0.71 || p.SMOccupancy != 25 || p.TensorActive != 0 || p.Source != "dcgm" {
+		t.Errorf("card 0 %+v", p)
+	}
+	// A field the exporter does not send is unknown, not zero.
+	if p.DRAMActive != -1 {
+		t.Errorf("DRAM active not sent, read as %v", p.DRAMActive)
+	}
+}
+
+type fakeGPM map[int]GPUProfile
+
+func (f fakeGPM) read() map[int]GPUProfile { return f }
+
+func TestGPMIsPreferredAndDCGMFillsTheCardsItDoesNotCover(t *testing.T) {
+	calls := 0
+	p := &ProfSampler{
+		gpm:  fakeGPM{1: {SMActive: 40, Source: "gpm"}},
+		dcgm: DefaultDCGMExporter,
+		get: func(context.Context, string) ([]byte, error) {
+			calls++
+			return []byte(`DCGM_FI_PROF_SM_ACTIVE{gpu="0"} 0.5` + "\n" + `DCGM_FI_PROF_SM_ACTIVE{gpu="1"} 0.9` + "\n"), nil
+		},
+		now: time.Now,
+	}
+	got := p.Sample(t.Context())
+	if got[0].Source != "dcgm" || got[0].SMActive != 50 {
+		t.Errorf("card 0 %+v", got[0])
+	}
+	if got[1].Source != "gpm" || got[1].SMActive != 40 {
+		t.Errorf("GPM's card was taken from DCGM: %+v", got[1])
+	}
+	if calls != 1 {
+		t.Errorf("%d calls", calls)
+	}
+}
+
+func TestANodeWithoutDCGMIsNotAskedEverySample(t *testing.T) {
+	now := time.Now()
+	calls := 0
+	p := &ProfSampler{dcgm: DefaultDCGMExporter, now: func() time.Time { return now },
+		get: func(context.Context, string) ([]byte, error) { calls++; return nil, errors.New("connection refused") }}
+	for range 5 {
+		if got := p.Sample(t.Context()); len(got) != 0 {
+			t.Fatalf("%+v", got)
+		}
+		now = now.Add(10 * time.Second)
+	}
+	if calls != 1 {
+		t.Errorf("asked %d times in 50 s", calls)
+	}
+	now = now.Add(dcgmRetry)
+	p.Sample(t.Context())
+	if calls != 2 {
+		t.Errorf("not asked again after %v", dcgmRetry)
+	}
+	if off := NewProfSampler("off"); off.dcgm != "" {
+		t.Errorf("off still asks %q", off.dcgm)
+	}
+}
+
+func TestEachSampleCarriesTheProfileOfTheCardsThatHaveOne(t *testing.T) {
+	smi := func(context.Context, string) ([]byte, error) {
+		return []byte("0, 71, 239, 260, 0x0, 92, 30, 23000, 24576, 1878, 0, 0\n" +
+			"1, 30, 20, 260, 0x0, 0, 0, 512, 24576, 300, 0, 0\n"), nil
+	}
+	now := time.Now()
+	s := sampler(t.TempDir(), smi, &now).WithProfile(&ProfSampler{
+		gpm: fakeGPM{0: {SMActive: 69, SMOccupancy: 45, TensorActive: 1, DRAMActive: 33, Source: "gpm"}}, now: time.Now})
+	got := s.Sample(t.Context()).GPUs
+	if got[0].Profile == nil || got[0].Profile.SMActive != 69 || got[0].UtilPercent != 92 {
+		t.Errorf("card 0 %+v", got[0])
+	}
+	if got[1].Profile != nil {
+		t.Errorf("card 1 has a profile nobody read: %+v", got[1].Profile)
+	}
+}

internal/agent/discovery/sensors.go

@@ -49,6 +49,9 @@ type GPUSensor struct {
 	MemoryUsedMB, MemoryTotalMB float64
 	SMClockMHz                  float64
 	ECCUncorrected              float64
+	// Profile is how much of the chip the work uses, where something on the
+	// node can say: nil on most cards. UtilPercent is not that.
+	Profile *GPUProfile
 }
 
 // sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
@@ -61,6 +64,8 @@ type SensorSampler struct {
 	sys string // /sys, or a test's copy of it
 	smi func(ctx context.Context, fields string) ([]byte, error)
 	now func() time.Time
+	// prof reads the cards' profiling counters; nil reads none.
+	prof *ProfSampler
 
 	mu         sync.Mutex
 	last       Sensors
@@ -74,6 +79,12 @@ func NewSensorSampler() *SensorSampler {
 	return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
 }
 
+// WithProfile has each sample carry the cards' profiles as well.
+func (s *SensorSampler) WithProfile(p *ProfSampler) *SensorSampler {
+	s.prof = p
+	return s
+}
+
 // SampleGPUs reads every card's heat, power and how busy it is now, without
 // a sampler's ten seconds of memory: for a screen that asks every few seconds
 // and wants what it asked for. Nil on a machine without nvidia-smi or a card.
@@ -101,6 +112,14 @@ func (s *SensorSampler) Sample(ctx context.Context) Sensors {
 	out.CPUTempC, out.InletTempC = s.hwmon()
 	out.CPUPowerW = s.rapl(now)
 	out.GPUs = s.gpus(ctx)
+	if s.prof != nil {
+		profs := s.prof.Sample(ctx)
+		for i := range out.GPUs {
+			if p, ok := profs[out.GPUs[i].Index]; ok {
+				out.GPUs[i].Profile = &p
+			}
+		}
+	}
 	s.last, s.lastAt = out, now
 	return out
 }

internal/cli/tui_console.go

@@ -1497,6 +1497,10 @@ func (m consoleModel) gpuRows() [][]string {
 					}
 					if !mig {
 						busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
+						if p := s.Profile; p != nil && p.SMActive >= 0 {
+							// How much of the chip, beside whether anything ran.
+							busy += fmt.Sprintf(" · SM %.0f%%", p.SMActive)
+						}
 						if s.MemoryTotalMB > 0 {
 							mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
 						}

internal/cli/tui_monitor.go

@@ -14,6 +14,7 @@ import (
 	"github.com/charmbracelet/lipgloss"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
 )
 
@@ -80,6 +81,9 @@ func (s *monitorState) sample(d consoleData) {
 		for _, g := range l.GPUs {
 			card := nd.ID + ":" + strconv.Itoa(g.Index)
 			add("gpu/"+card, g.UtilPercent)
+			if w := gpuWork(g); w >= 0 {
+				add("gpuwork/"+card, w)
+			}
 			if g.MemoryTotalMB > 0 {
 				add("vram/"+card, 100*g.MemoryUsedMB/g.MemoryTotalMB)
 			}
@@ -142,6 +146,7 @@ func monitorQueries(window time.Duration, points int) (time.Duration, []adminapi
 		{ID: "cpu", Expr: `avg(llmgw_node_cpu_utilization_percent)`},
 		{ID: "mem", Expr: `100 * sum(llmgw_node_memory_used_bytes) / sum(llmgw_node_memory_bytes)`},
 		{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent`), By: "card"},
+		{ID: "gpuwork", Expr: card(gpuWorkExpr("")), By: "card"},
 		{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes`), By: "card"},
 		{ID: "gputemp", Expr: card(`llmgw_gpu_temperature_celsius`), By: "card"},
 		{ID: "gpupow", Expr: card(`llmgw_gpu_power_watts`), By: "card"},
@@ -314,7 +319,8 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) {
 	b.WriteString(indent(side(w, box("CPU", pct("cpu"), m.series("cpu"), 100),
 		box("Memory", pct("mem"), m.series("mem"), 100)), "  ") + "\n\n")
 	if g := m.series("gpu"); len(g) > 0 || len(m.data.sum.GPUs) > 0 {
-		b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", m.series("vram"), 100)), "  ") + "\n\n")
+		b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU work", "", m.series("gpuwork"), 100)), "  ") + "\n\n")
+		b.WriteString(indent(side(w, box("GPU memory", "", m.series("vram"), 100), "  "+sDim.Render(gpuWorkNote)), "  ") + "\n\n")
 	}
 	if m.mon.res.Configured {
 		rps := m.series("rps")
@@ -356,10 +362,12 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) {
 	// Each card: busy, memory, heat and power.
 	if gpus := m.series("gpu"); len(gpus) > 0 {
 		vram, temp, pow := byName(m.series("vram")), byName(m.series("gputemp")), byName(m.series("gpupow"))
+		work := byName(m.series("gpuwork"))
 		b.WriteString("\n  " + mdBold.Render("GPUs") + "\n")
 		for _, g := range gpus {
-			fmt.Fprintf(&b, "  %-16s busy %s %4.0f%%  memory %s %4.0f%%  %3.0f°C %s  %4.0f W\n", truncate(g.name, 16),
+			fmt.Fprintf(&b, "  %-16s busy %s %4.0f%%  work %s %4.0f%%  memory %s %4.0f%%  %3.0f°C %s  %4.0f W\n", truncate(g.name, 16),
-				sparkSeries(g, sw/2, 100), last(g.values), sparkSeries(vram[g.name], sw/2, 100), last(vram[g.name].values),
+				sparkSeries(g, sw/3, 100), last(g.values), sparkSeries(work[g.name], sw/3, 100), last(work[g.name].values),
+				sparkSeries(vram[g.name], sw/3, 100), last(vram[g.name].values),
 				last(temp[g.name].values), sparkSeries(temp[g.name], 8, 100), last(pow[g.name].values))
 		}
 	}
@@ -373,6 +381,29 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) {
 	return "Monitor", head.String() + b.String(), keys
 }
 
+// GPU busy is the driver's figure: the share of time any kernel ran, which
+// a kernel on one SM of a hundred fills. GPU work is how much of the chip:
+// the SMs' share of time with work where the card's profiling counters are
+// read (GPM, DCGM), and its draw against its power limit elsewhere.
+const gpuWorkNote = "busy: some kernel ran. work: how much of the chip —\nSMs active where the counters are read, power/limit elsewhere."
+
+// gpuWorkExpr is GPU work in PromQL, for the series matching sel.
+func gpuWorkExpr(sel string) string {
+	return `llmgw_gpu_sm_active_percent` + sel + ` or (100 * llmgw_gpu_power_watts` + sel +
+		` / llmgw_gpu_power_limit_watts` + sel + `)`
+}
+
+// gpuWork is GPU work from a sample, or -1 without the figures for it.
+func gpuWork(g inventory.GPUSensor) float64 {
+	if p := g.Profile; p != nil && p.SMActive >= 0 {
+		return p.SMActive
+	}
+	if l := g.PowerLoad(); l >= 0 {
+		return 100 * l
+	}
+	return -1
+}
+
 func named(s []chartSeries, name string) []chartSeries {
 	for i := range s {
 		s[i].name = name
@@ -410,6 +441,7 @@ func detailQueries(kind, id string, window time.Duration, points int) (time.Dura
 			{ID: "cpu", Expr: `llmgw_node_cpu_utilization_percent{node=` + q + `}`},
 			{ID: "mem", Expr: `100 * llmgw_node_memory_used_bytes{node=` + q + `} / llmgw_node_memory_bytes{node=` + q + `}`},
 			{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent{node=` + q + `}`), By: "card"},
+			{ID: "gpuwork", Expr: card(gpuWorkExpr(`{node=` + q + `}`)), By: "card"},
 			{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes{node=` + q + `} / llmgw_gpu_memory_total_bytes{node=` + q + `}`), By: "card"},
 			{ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
 			{ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
@@ -520,7 +552,8 @@ func (m consoleModel) detailCharts(pw int) string {
 		b.WriteString(indent(side(w, box("CPU", fmt.Sprintf("%.0f%%", lastOf(cpu)), cpu, 100),
 			box("Memory", fmt.Sprintf("%.0f%%", lastOf(mem)), mem, 100)), "  "))
 		if g := get("gpu"); len(g) > 0 {
-			b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", get("vram"), 100)), "  "))
+			b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU work", "", get("gpuwork"), 100)), "  "))
+			b.WriteString("\n\n" + indent(side(w, box("GPU memory", "", get("vram"), 100), "  "+sDim.Render(gpuWorkNote)), "  "))
 		}
 		if m.det.res.Configured {
 			net := append(named(get("netin"), "in"), named(get("netout"), "out")...)

internal/control/agentsvc/service.go

@@ -772,12 +772,25 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load {
 			UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(),
 			MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(),
 			SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(),
-			Wear: wearFrom(g),
+			Wear: wearFrom(g), Profile: profileFrom(g),
 		})
 	}
 	return out
 }
 
+// profileFrom is a card's profiling counters, or nil from an agent that
+// sent none.
+func profileFrom(g *llmgwv1.GPUSensor) *inventory.GPUProfile {
+	if !g.GetHasProfile() {
+		return nil
+	}
+	return &inventory.GPUProfile{
+		SMActive: g.GetSmActivePercent(), SMOccupancy: g.GetSmOccupancyPercent(),
+		TensorActive: g.GetTensorActivePercent(), DRAMActive: g.GetDramActivePercent(),
+		Source: g.GetProfileSource(),
+	}
+}
+
 func (s *Service) attach(ctx context.Context, nodeID string) *conn {
 	c := &conn{
 		nodeID: nodeID,

internal/control/fleetmetrics/fleetmetrics.go

@@ -84,6 +84,10 @@ type Collector struct {
 	gpuXID         *prometheus.Desc
 	gpuLastXID     *prometheus.Desc
 	gpuMemUtil     *prometheus.Desc
+	gpuSMActive    *prometheus.Desc
+	gpuSMOccupancy *prometheus.Desc
+	gpuTensor      *prometheus.Desc
+	gpuDRAMActive  *prometheus.Desc
 	gpuSMClock     *prometheus.Desc
 	gpuTemp        *prometheus.Desc
 	gpuMemTemp     *prometheus.Desc
@@ -158,9 +162,22 @@ func New(src Source) *Collector {
 		gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes",
 			"A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil),
 		gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent",
-			"How busy a GPU was over the last sample period. Absent when the driver does not "+
+			"The share of the last sample period in which any kernel ran on a GPU (nvidia-smi's "+
-				"report it, which is not the same as idle. Memory used says a model is loaded; "+
+				"utilization.gpu). Not how much of the chip: a kernel on one SM reads 100%. Absent "+
-				"this says whether it is doing anything.", gpu, nil),
+				"when the driver does not report it, which is not the same as idle. Memory used says "+
+				"a model is loaded; this says whether it is doing anything, and "+
+				"llmgw_gpu_sm_active_percent how much.", gpu, nil),
+		gpuSMActive: prometheus.NewDesc("llmgw_gpu_sm_active_percent",
+			"The share of time a GPU's SMs had work, averaged over all of them: how much of the chip "+
+				"the busy figure is using. From NVML's GPM (Hopper and later) or a dcgm-exporter on the node; "+
+				"absent elsewhere.", gpu, nil),
+		gpuSMOccupancy: prometheus.NewDesc("llmgw_gpu_sm_occupancy_percent",
+			"How full a GPU's active SMs were with warps, of the most they can hold.", gpu, nil),
+		gpuTensor: prometheus.NewDesc("llmgw_gpu_tensor_active_percent",
+			"The share of time a GPU's tensor cores were working. Low in decode at batch one is normal; "+
+				"prefill and batches are where it should rise.", gpu, nil),
+		gpuDRAMActive: prometheus.NewDesc("llmgw_gpu_dram_active_percent",
+			"The share of time a GPU's memory interface was moving data, from its profiling counters.", gpu, nil),
 		gpuPCIeReplay: prometheus.NewDesc("llmgw_gpu_pcie_replays",
 			"PCIe link retries on a GPU's link since the driver loaded; one that keeps climbing is a riser, slot or cable.", gpu, nil),
 		gpuRetired: prometheus.NewDesc("llmgw_gpu_retired_pages",
@@ -310,7 +327,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
 		c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
 		c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
 		c.gpuNVLJoined, c.imexUp,
-		c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
+		c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock,
+		c.gpuSMActive, c.gpuSMOccupancy, c.gpuTensor, c.gpuDRAMActive, c.gpuAssigned, c.gpuTemp, c.gpuHealthy,
 		c.gpuPCIeReplay, c.gpuRetired, c.gpuRetirePend, c.gpuRemapped, c.gpuRemapPend, c.gpuRemapFail,
 		c.gpuNVLinkErr, c.gpuXID, c.gpuLastXID,
 		c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp,
@@ -653,6 +671,19 @@ func (c *Collector) gpuBusy(ch chan<- prometheus.Metric, s inventory.GPUSensor,
 	if s.SMClockMHz > 0 {
 		ch <- prometheus.MustNewConstMetric(c.gpuSMClock, prometheus.GaugeValue, s.SMClockMHz, labels...)
 	}
+	// The card's profile, here rather than with the sensors because it is
+	// the whole card's: a MIG instance must not be published as having it.
+	if p := s.Profile; p != nil {
+		for _, v := range []struct {
+			d   *prometheus.Desc
+			val float64
+		}{{c.gpuSMActive, p.SMActive}, {c.gpuSMOccupancy, p.SMOccupancy}, {c.gpuTensor, p.TensorActive}, {c.gpuDRAMActive, p.DRAMActive}} {
+			// Zero is an idle chip; only -1, not reported, is left out.
+			if v.val >= 0 {
+				ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...)
+			}
+		}
+	}
 }
 
 func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) {

internal/control/fleetmetrics/mig_test.go

@@ -16,7 +16,8 @@ func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
 		MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}}
 	n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst},
 		Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250,
-			UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920}}}}
+			UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920,
+			Profile: &inventory.GPUProfile{SMActive: 50, SMOccupancy: 30, TensorActive: 10, DRAMActive: 20}}}}}
 	out := gather(t, fakeSource{nodes: []inventory.Node{n}})
 	labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1"
 	for _, want := range []string{
@@ -33,4 +34,8 @@ func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
 	if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) {
 		t.Errorf("a utilisation was published for a MIG instance:\n%s", out)
 	}
+	// The card's profile is the whole card's, not this instance's.
+	if strings.Contains(out, "llmgw_gpu_sm_active_percent "+labels) {
+		t.Errorf("the card's SM activity was published for a MIG instance:\n%s", out)
+	}
 }

internal/control/fleetmetrics/sensors_test.go

@@ -127,3 +127,27 @@ func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) {
 		t.Errorf("a count the card does not keep was published:\n%s", out)
 	}
 }
+
+func TestHowMuchOfTheChipIsPublishedBesideHowBusy(t *testing.T) {
+	// The Quadro RTX 6000 in decode: 92% busy, its SMs at work 69% of the
+	// time. The DRAM figure was not read, and is left out rather than zero.
+	n := inventory.Node{ID: "g1", State: inventory.Ready,
+		GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}},
+		Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, UtilPercent: 92, MemoryTotalMB: 81559,
+			Profile: &inventory.GPUProfile{SMActive: 69, SMOccupancy: 45, TensorActive: 0, DRAMActive: -1, Source: "gpm"}}}}}
+	out := gather(t, fakeSource{nodes: []inventory.Node{n}})
+	for _, want := range []string{
+		"llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 92\n",
+		"llmgw_gpu_sm_active_percent class=h100-80g gpu=0 node=g1 -> 69\n",
+		"llmgw_gpu_sm_occupancy_percent class=h100-80g gpu=0 node=g1 -> 45\n",
+		// Idle tensor cores are a reading.
+		"llmgw_gpu_tensor_active_percent class=h100-80g gpu=0 node=g1 -> 0\n",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("missing %s in:\n%s", want, out)
+		}
+	}
+	if strings.Contains(out, "llmgw_gpu_dram_active_percent") {
+		t.Errorf("a figure not read was published:\n%s", out)
+	}
+}

internal/control/inventory/inventory.go

@@ -340,6 +340,36 @@ type GPUSensor struct {
 	// Wear is what wears the card out, read every minute; nil from an agent
 	// that does not read it.
 	Wear *GPUWear `json:"wear,omitempty"`
+	// Profile is how much of the chip the work uses, which UtilPercent does
+	// not say; nil where nothing on the node can read it.
+	Profile *GPUProfile `json:"profile,omitempty"`
+}
+
+// GPUProfile is a card's profiling counters, in percent; -1 is not reported.
+// UtilPercent is the share of time any kernel ran, and a kernel on one SM of
+// a hundred counts in full; these say how much of the chip it ran on.
+type GPUProfile struct {
+	// SMActive is the share of time the SMs had work, averaged over all of
+	// them; SMOccupancy how full those SMs were with warps.
+	SMActive    float64 `json:"sm_active"`
+	SMOccupancy float64 `json:"sm_occupancy"`
+	// TensorActive is the tensor cores, DRAMActive the memory interface.
+	TensorActive float64 `json:"tensor_active"`
+	DRAMActive   float64 `json:"dram_active"`
+	// Source is what read them: "gpm" (NVML, Hopper and later) or "dcgm".
+	Source string `json:"source,omitempty"`
+}
+
+// PowerLoad is the card's draw as a share of its power limit, from 0 to 1,
+// or -1 without both figures. Unlike UtilPercent it rises with how much of
+// the chip is working, not only with whether anything is: a card reading
+// 92% busy drew 239 W of 260 W in decode and 112 W when the busy figure was
+// a kernel or two between requests.
+func (g GPUSensor) PowerLoad() float64 {
+	if g.PowerW <= 0 || g.PowerLimitW <= 0 {
+		return -1
+	}
+	return min(g.PowerW/g.PowerLimitW, 1)
 }
 
 // GPUWear is a card's wear and failures. Counts of -1 are not reported for

internal/docgen/reference_gen.go

@@ -296,6 +296,7 @@ var Reference = []Block{
 		{Key: "ib_root", Type: "string", Doc: "IBRoot is the sysfs InfiniBand directory.", Nested: ""},
 		{Key: "prefer_nvml", Type: "bool", Doc: "PreferNVML uses the NVML backend when this binary has one. NVLink\ntopology is read from nvidia-smi topo -m on every build; NVML, where it\nis used, is the more exact source of the peers.", Nested: ""},
 		{Key: "skip_gpus", Type: "bool", Doc: "SkipGPUs omits GPU discovery. A node without nvidia-smi is already\nfound to have none, so this is for one whose driver is broken.", Nested: ""},
+		{Key: "dcgm_exporter", Type: "string", Doc: "DCGMExporter is the dcgm-exporter metrics URL read for how much of\neach card's chip is in use, on cards NVML's GPM does not cover (before\nHopper). Empty is http://127.0.0.1:9400/metrics; \"off\" reads none. A\nnode without one costs a refused connection every five minutes.", Nested: ""},
 	}},
 	{Package: "catalog", Name: "Origin", Title: "Origin", Doc: "", Fields: []Field{
 		{Key: "repo", Type: "string", Doc: "Repo is the repository, owner/name.", Nested: ""},

internal/web/dashcharts.go

@@ -27,7 +27,11 @@ type FleetCharts struct {
 	// GPUBusy and GPUMemory are the fleet's cards by class, for a fleet that
 	// has them.
 	GPUBusy, GPUMemory TimeSeries
-	Window             string
+	// GPUPowerLoad is the cards' draw against their power limit, and
+	// GPUSMActive their SMs' share of time with work, where something reads
+	// it: how much of each chip the busy figure is using.
+	GPUPowerLoad, GPUSMActive TimeSeries
+	Window                    string
 	// Err is set when the metrics store could not be read. Shown as a line of
 	// text where the charts would be: an empty panel reads as a healthy cluster
 	// with nothing happening, which is the opposite of not knowing. The
@@ -144,4 +148,12 @@ func (f *FleetCharts) LoadGPU(ctx context.Context, h History, window time.Durati
 		"class", window, step); err == nil {
 		f.GPUMemory = l.LineChart(s, "%", 6)
 	}
+	if s, err := h.RangeQuery(ctx,
+		`sum by (class) (llmgw_gpu_power_watts) / sum by (class) (llmgw_gpu_power_limit_watts)`,
+		"class", window, step); err == nil {
+		f.GPUPowerLoad = l.LineChart(s, "%", 6)
+	}
+	if s, err := h.RangeQuery(ctx, `avg by (class) (llmgw_gpu_sm_active_percent) / 100`, "class", window, step); err == nil {
+		f.GPUSMActive = l.LineChart(s, "%", 6)
+	}
 }

internal/web/efficiency.go

@@ -13,8 +13,13 @@ type modelEfficiency struct {
 	// TokensPerSec is output tokens a second; GPUs the cards it holds now.
 	TokensPerSec float64
 	GPUs         float64
-	// Watts is what those cards draw; Util their average utilisation.
+	// Watts is what those cards draw; Util the average share of time a
-	Watts, Util float64
+	// kernel ran on them, which says nothing of how much of each chip it ran
+	// on. PowerLoad (draw against the power limit) and SMActive (the SMs'
+	// share of time with work, where GPM or DCGM reads it) are how much, in
+	// percent; -1 is not known.
+	Watts, Util         float64
+	PowerLoad, SMActive float64
 	// PerGPU and PerWh are the two ways of saying how much work the cards
 	// do: output tokens a second per card, and per watt-hour drawn.
 	PerGPU, PerWh float64
@@ -34,6 +39,10 @@ var efficiencyQueries = []struct {
 		func(e *modelEfficiency, v float64) { e.Watts = v }},
 	{`avg by (model) (llmgw_gpu_utilization_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
 		func(e *modelEfficiency, v float64) { e.Util = v }},
+	{`avg by (model) ((llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts) * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
+		func(e *modelEfficiency, v float64) { e.PowerLoad = 100 * v }},
+	{`avg by (model) (llmgw_gpu_sm_active_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`,
+		func(e *modelEfficiency, v float64) { e.SMActive = v }},
 }
 
 // loadEfficiency is the table, best per card first; nil when the metrics
@@ -56,7 +65,7 @@ func loadEfficiency(ctx context.Context, h History) []modelEfficiency {
 			}
 			e := byModel[s.Name]
 			if e == nil {
-				e = &modelEfficiency{Model: s.Name}
+				e = &modelEfficiency{Model: s.Name, PowerLoad: -1, SMActive: -1}
 				byModel[s.Name] = e
 			}
 			q.into(e, s.Last())

internal/web/gpumap.go

@@ -8,16 +8,25 @@ import (
 	"time"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
 )
 
 // The GPU map is every card in the fleet, a square each, a row per node,
-// coloured by one figure: how busy, how hot, or how full its memory is. A
+// coloured by one figure: how much of the chip is at work, how busy, how
+// hot, or how full its memory is. A
 // hundred cards in a table are a hundred rows to read; on the map the one
 // that is not like the others is seen before anything is read.
 
 // gpuMapModes are what the map can be coloured by; each is named by the key
 // dash.gpumap.mode.<mode>.
-var gpuMapModes = []string{"util", "memory", "temp"}
+//
+// work is the default. util, the driver's busy figure, is the share of time
+// any kernel ran, and a kernel on one SM of a hundred reads 100%: a map of
+// it is green wherever a model is loaded and asked anything. work is the
+// SMs' share of time with work where the card's counters can be read, and
+// its power draw against its limit elsewhere, which rises with how much of
+// the chip runs rather than with whether anything does.
+var gpuMapModes = []string{"work", "util", "memory", "temp"}
 
 func init() {
 	for _, m := range gpuMapModes {
@@ -72,17 +81,16 @@ func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView
 	for _, k := range gpuMapModes {
 		m.Modes = append(m.Modes, struct{ Key, Label string }{k, l.T("dash.gpumap.mode." + k)})
 	}
-	if m.By == "" {
-		m.By = "util"
-	}
 	switch m.By {
+	case "util":
+		m.Legend = l.T("dash.gpumap.legend.util")
 	case "temp":
 		m.Legend = l.T("dash.gpumap.legend.temp")
 	case "memory":
 		m.Legend = l.T("dash.gpumap.legend.memory")
 	default:
-		m.By = "util"
+		m.By = "work"
-		m.Legend = l.T("dash.gpumap.legend.util")
+		m.Legend = l.T("dash.gpumap.legend.work")
 	}
 	for _, n := range nodes {
 		if n.GPUTotal == 0 {
@@ -106,13 +114,21 @@ func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView
 				if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 {
 					title = append(title, l.T("dash.gpumap.busy", l.Num(s.UtilPercent, 0)))
 				}
+				if p := s.Profile; p != nil && p.SMActive >= 0 {
+					title = append(title, l.T("dash.gpumap.sm_active", l.Num(p.SMActive, 0)))
+					if p.TensorActive >= 0 {
+						title = append(title, l.T("dash.gpumap.tensor_active", l.Num(p.TensorActive, 0)))
+					}
+				}
 				if s.MemoryTotalMB > 0 {
 					title = append(title, l.T("dash.gpumap.memory", l.Num(s.MemoryUsedMB/1024, 1), l.Num(s.MemoryTotalMB/1024, 0)))
 				}
 				if s.TempC > 0 {
 					title = append(title, l.Num(s.TempC, 0)+" °C")
 				}
-				if s.PowerW > 0 {
+				if s.PowerW > 0 && s.PowerLimitW > 0 {
+					title = append(title, l.T("dash.gpumap.power_of", l.Num(s.PowerW, 0), l.Num(s.PowerLimitW, 0)))
+				} else if s.PowerW > 0 {
 					title = append(title, l.Num(s.PowerW, 0)+" W")
 				}
 				if len(s.Throttle) > 0 {
@@ -162,6 +178,20 @@ func cellColour(by string, s sensorView, sampled bool) (string, string) {
 		}
 		f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB)
 		return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f)
+	case "work":
+		if !sampled {
+			return unknown, ""
+		}
+		var f float64
+		switch {
+		case s.Profile != nil && s.Profile.SMActive >= 0:
+			f = clamp01(s.Profile.SMActive / 100)
+		case s.PowerW > 0 && s.PowerLimitW > 0:
+			f = clamp01(s.PowerW / s.PowerLimitW)
+		default:
+			return unknown, ""
+		}
+		return fmt.Sprintf("background:hsl(200,%.0f%%,%.0f%%)", 15+55*f, 88-50*f), fmt.Sprintf("%.0f", 100*f)
 	}
 	if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 {
 		return unknown, ""
@@ -286,9 +316,10 @@ func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu
 
 // sensorView is a card's last sample.
 type sensorView = struct {
-	TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
+	TempC, PowerW, PowerLimitW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64
-	Throttle                                                []string
+	Throttle                                                             []string
-	Problems                                                []string
+	Problems                                                             []string
+	Profile                                                              *inventory.GPUProfile
 }
 
 func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
@@ -297,9 +328,9 @@ func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) {
 	}
 	for _, s := range n.Load.GPUs {
 		if s.Index == gpu {
-			return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent,
+			return sensorView{TempC: s.TempC, PowerW: s.PowerW, PowerLimitW: s.PowerLimitW, UtilPercent: s.UtilPercent,
 				MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle,
-				Problems: s.Wear.Problems()}, true
+				Problems: s.Wear.Problems(), Profile: s.Profile}, true
 		}
 	}
 	return sensorView{}, false

internal/web/gpumap_test.go

@@ -17,10 +17,14 @@ type mapFleet struct{ fakeSource }
 
 func (mapFleet) Nodes() []adminapi.NodeView {
 	busy := func(i int, util, temp float64) inventory.GPUSensor {
-		return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
+		return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300, PowerLimitW: 700}
+	}
+	profiled := func(s inventory.GPUSensor) inventory.GPUSensor {
+		s.Profile = &inventory.GPUProfile{SMActive: 0, SMOccupancy: 0, TensorActive: 0, DRAMActive: -1, Source: "dcgm"}
+		return s
 	}
 	return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
-		Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
+		Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{profiled(busy(0, 1, 35)), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
 }
 
 func worn(s inventory.GPUSensor) inventory.GPUSensor {
@@ -60,6 +64,20 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
 	if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
 		t.Error("the map coloured by temperature does not say so, or show it")
 	}
+	// The default colours by how much of the chip works: card 2's draw of
+	// 300 W of 700, where nothing reads its SMs; card 0's SMs where
+	// something does. Busy stays a mode of its own.
+	if !strings.Contains(body, "<strong>work</strong>") || !strings.Contains(body, ">43</span>") ||
+		!strings.Contains(body, "300 of 700 W") {
+		t.Error("the map does not colour by work, or say the draw against the limit")
+	}
+	if !strings.Contains(body, "gpu07 GPU 0 · chat (chat-1) · 1% busy · SMs 0% active · tensor cores 0%") {
+		t.Error("a profiled card does not say how much of its chip is at work")
+	}
+	_, util := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=util", nil)
+	if !strings.Contains(util, "<strong>busy</strong>") || !strings.Contains(util, ">93</span>") {
+		t.Error("the map coloured by busy does not say so, or show it")
+	}
 	_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
 	if strings.Contains(cpu, "gpu-map") {
 		t.Error("a fleet without GPUs has a GPU map")
@@ -68,7 +86,7 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
 
 func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
 	h := &fakeHistory{byExpr: map[string][]promq.Series{}}
-	for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}} {
+	for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}, {0.6, 0.2}, {55, 0}} {
 		h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])}
 	}
 	got := loadEfficiency(context.Background(), h)
@@ -76,7 +94,7 @@ func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
 		t.Fatalf("%+v", got)
 	}
 	c := got[0]
-	if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 {
+	if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 || c.PowerLoad != 60 || c.SMActive != 55 {
 		t.Errorf("chat: %+v", c)
 	}
 	if got[1].PerGPU != 15 {

internal/web/i18n_cluster_test.go

@@ -37,7 +37,7 @@ func TestTheDashboardSpeaksTurkishWhenAsked(t *testing.T) {
 		// A state is still its class, and a Turkish word.
 		`<td class="state state-ready">hazır</td>`,
 		// The GPU map: its modes, and what a card says under the pointer.
-		"<strong>meşguliyet</strong>", "gpu07 GPU 2 · code (code-1) · %93 meşgul · 39,1 / 80 GB",
+		"<strong>iş</strong>", "gpu07 GPU 2 · code (code-1) · %93 meşgul · 39,1 / 80 GB",
 		// How long a replica has sat idle, in Turkish units.
 		"30dk0sn önce",
 		// The traffic chart's peak, with Turkish thousands.

internal/web/locales/en/dashboard.json

@@ -26,11 +26,16 @@
   "dash.gpumap.mode.util": "busy",
   "dash.gpumap.mode.memory": "memory",
   "dash.gpumap.mode.temp": "temperature",
+  "dash.gpumap.mode.work": "work",
   "dash.gpumap.legend.temp": "Blue is cool, red is 90 °C and over.",
   "dash.gpumap.legend.memory": "Pale is empty, dark is full. Engines reserve most of a card's memory when they start.",
-  "dash.gpumap.legend.util": "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds.",
+  "dash.gpumap.legend.util": "Pale is idle, dark green is busy: the share of time any kernel ran, however little of the chip it used. A dashed border is a card no replica holds.",
+  "dash.gpumap.legend.work": "Pale is idle, dark blue is the whole chip at work: the SMs' share of time with work where the card's counters can be read (GPM, DCGM), its power draw against its limit elsewhere. A dashed border is a card no replica holds.",
   "dash.gpumap.free": "free",
   "dash.gpumap.busy": "%s%% busy",
+  "dash.gpumap.sm_active": "SMs %s%% active",
+  "dash.gpumap.tensor_active": "tensor cores %s%%",
+  "dash.gpumap.power_of": "%s of %s W",
   "dash.gpumap.memory": "%s of %s GB",
   "dash.gpumap.held_back": "held back by %s",
   "dash.gpumap.note": "A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.",
@@ -41,8 +46,11 @@
   "dash.eff.tps": "Output tokens/s",
   "dash.eff.per_gpu": "Per GPU",
   "dash.eff.busy": "Busy",
+  "dash.eff.power_load": "Power load",
+  "dash.eff.sm_active": "SM active",
   "dash.eff.draw": "Draw",
   "dash.eff.per_wh": "Tokens per Wh",
+  "dash.eff.note": "Busy is the share of time any kernel ran on the cards, not how much of each chip it ran on: a kernel on one SM reads 100%. Power load (draw against the power limit) and SM active (the SMs' share of time with work, where the cards' counters can be read) say how much.",
 
   "dash.idle.title": "Cards held and doing nothing",
   "dash.col.replica": "Replica",
@@ -150,7 +158,11 @@
   "dash.fleet.ready": "Replicas ready, by model",
   "dash.fleet.ready_note": "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here.",
   "dash.fleet.gpu_busy": "GPUs busy, by class",
-  "dash.fleet.gpu_busy_note": "The average card of each class. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.",
+  "dash.fleet.gpu_busy_note": "The average card of each class: the share of time any kernel ran, not how much of the chip it used. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.",
+  "dash.fleet.gpu_power_load": "GPU power load, by class",
+  "dash.fleet.gpu_power_load_note": "The cards' draw against their power limit. It rises with how much of each chip is working, where busy only says that something is; a class busy above and low here runs small kernels or waits between them.",
+  "dash.fleet.gpu_sm_active": "SMs active, by class",
+  "dash.fleet.gpu_sm_active_note": "The share of time the cards' SMs had work, from their profiling counters (NVML GPM on Hopper and later, or a dcgm-exporter on the node). The gap to busy is the part of the chip a running kernel leaves idle.",
   "dash.fleet.gpu_memory": "GPU memory in use, by class",
   "dash.fleet.gpu_memory_note": "Engines reserve most of a card's memory when they start, so this follows replicas placed more than requests served.",
   "dash.fleet.queue": "Engine queue depth, by replica",

internal/web/locales/en/node.json

@@ -77,7 +77,11 @@
   "node.chart.engine_memory": "Engine memory, by model",
   "node.chart.engine_memory_note": "Resident memory, including the weights mapped from disk. The kernel can take those pages back, so this can exceed RAM in use.",
   "node.chart.gpu_busy": "GPU busy, by card",
-  "node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running. A card held by a replica and idle here is one nobody is asking.",
+  "node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running — not how much of the chip it used: one SM busy reads 100%. A card held by a replica and idle here is one nobody is asking.",
+  "node.chart.gpu_power_load": "GPU power load, by card",
+  "node.chart.gpu_power_load_note": "Draw against the card's power limit. Busy says a kernel ran; this rises with how much of the chip it ran on.",
+  "node.chart.gpu_sm_active": "SMs active, by card",
+  "node.chart.gpu_sm_active_note": "The share of time the card's SMs had work, from its profiling counters. When busy reads 90 and this 60, a third of the chip sits idle while a kernel runs.",
   "node.chart.gpu_memory": "GPU memory used, by card",
   "node.chart.gpu_memory_note": "Framebuffer in use: the weights and the KV cache an engine reserved, and anything not llmgw's.",
   "node.chart.gpu_temp": "GPU temperature, °C",

internal/web/locales/tr/dashboard.json

@@ -26,11 +26,16 @@
   "dash.gpumap.mode.util": "meşguliyet",
   "dash.gpumap.mode.memory": "bellek",
   "dash.gpumap.mode.temp": "sıcaklık",
+  "dash.gpumap.mode.work": "iş",
   "dash.gpumap.legend.temp": "Mavi serin, kırmızı 90 °C ve üstü.",
   "dash.gpumap.legend.memory": "Açık renk boş, koyu renk dolu. Motorlar başlarken bir kartın belleğinin çoğunu ayırır.",
-  "dash.gpumap.legend.util": "Açık renk boşta, koyu yeşil tamamen meşgul. Kesikli kenarlık, hiçbir replikanın tutmadığı bir karttır.",
+  "dash.gpumap.legend.util": "Açık renk boşta, koyu yeşil meşgul demektir: yonganın ne kadarını kullanırsa kullansın, herhangi bir çekirdek işlevinin çalıştığı zaman payı. Kesik kenarlık hiçbir replikanın tutmadığı karttır.",
+  "dash.gpumap.legend.work": "Açık renk boşta, koyu mavi tüm yongası çalışan kart demektir: kartın sayaçları okunabiliyorsa (GPM, DCGM) SM'lerin iş yaptığı zaman payı, okunamıyorsa çektiği gücün güç sınırına oranı. Kesik kenarlık hiçbir replikanın tutmadığı karttır.",
   "dash.gpumap.free": "boş",
   "dash.gpumap.busy": "%%%s meşgul",
+  "dash.gpumap.sm_active": "SM'ler %%%s etkin",
+  "dash.gpumap.tensor_active": "tensör çekirdekleri %%%s",
+  "dash.gpumap.power_of": "%s / %s W",
   "dash.gpumap.memory": "%s / %s GB",
   "dash.gpumap.held_back": "kısıtlayan: %s",
   "dash.gpumap.note": "Kırmızı çerçeve, yıpranmakta olan bir karttır: XID hataları, yeniden eşlenmiş ya da sıfırlama bekleyen bellek satırları, NVLink hataları. Karttaki rakam, renklendirmede kullanılan değerdir; kartın geri kalanı imlecin altındadır.",
@@ -41,8 +46,11 @@
   "dash.eff.tps": "Çıkış token/sn",
   "dash.eff.per_gpu": "GPU başına",
   "dash.eff.busy": "Meşgul",
+  "dash.eff.power_load": "Güç yükü",
+  "dash.eff.sm_active": "Etkin SM",
   "dash.eff.draw": "Çekilen güç",
   "dash.eff.per_wh": "Wh başına token",
+  "dash.eff.note": "Meşgul, kartlarda herhangi bir çekirdek işlevinin çalıştığı zaman payıdır; her yonganın ne kadarında çalıştığı değil: tek bir SM'de çalışan işlev %100 okunur. Ne kadarını kullandığını güç yükü (çekilen gücün güç sınırına oranı) ve etkin SM (kartların sayaçları okunabildiğinde SM'lerin iş yaptığı zaman payı) söyler.",
 
   "dash.idle.title": "Tutulan ama hiçbir şey yapmayan kartlar",
   "dash.col.replica": "Replika",
@@ -148,7 +156,11 @@
   "dash.fleet.ready": "Hazır replikalar, modele göre",
   "dash.fleet.ready_note": "Bilerek durdurulmuş bir modelin hiç çizgisi olmaz; isteğe bağlı çalışan bir katalogun burada sessiz görünmesinin nedeni budur.",
   "dash.fleet.gpu_busy": "Meşgul GPU'lar, sınıfa göre",
-  "dash.fleet.gpu_busy_note": "Her sınıfın ortalama kartı. Replikalar yerleştirilmişken burada düşük olması, tutulan ama istenmeyen kapasite demektir; her düğümün sayfası kartlarını tek tek gösterir.",
+  "dash.fleet.gpu_busy_note": "Her sınıfın ortalama kartı: herhangi bir çekirdek işlevinin çalıştığı zaman payı, yonganın ne kadarını kullandığı değil. Replikalar yerleştirilmişken burada düşük olması, tutulan ama istenmeyen kapasite demektir; her düğümün sayfası kartlarını tek tek gösterir.",
+  "dash.fleet.gpu_power_load": "GPU güç yükü, sınıfa göre",
+  "dash.fleet.gpu_power_load_note": "Kartların çektiği gücün güç sınırına oranı. Meşgul yalnızca bir şeyin çalıştığını söylerken bu, her yonganın ne kadarının çalıştığıyla artar; yukarıda meşgul ve burada düşük görünen bir sınıf küçük çekirdek işlevleri çalıştırıyor ya da aralarında bekliyordur.",
+  "dash.fleet.gpu_sm_active": "Etkin SM'ler, sınıfa göre",
+  "dash.fleet.gpu_sm_active_note": "Kartların SM'lerinin iş yaptığı zaman payı, profil sayaçlarından (Hopper ve sonrasında NVML GPM ya da düğümdeki bir dcgm-exporter). Meşgul ile arasındaki fark, çalışan bir işlevin yongada boş bıraktığı kısımdır.",
   "dash.fleet.gpu_memory": "Kullanılan GPU belleği, sınıfa göre",
   "dash.fleet.gpu_memory_note": "Motorlar başlarken bir kartın belleğinin çoğunu ayırır; bu yüzden bu grafik sunulan isteklerden çok yerleştirilen replikaları izler.",
   "dash.fleet.queue": "Motor kuyruk derinliği, replikaya göre",

internal/web/locales/tr/node.json

@@ -76,7 +76,11 @@
   "node.chart.engine_memory": "Motor belleği, modele göre",
   "node.chart.engine_memory_note": "Diskten eşlenen ağırlıklar dâhil yerleşik bellek. Çekirdek o sayfaları geri alabilir, bu yüzden bu değer kullanılan RAM'i aşabilir.",
   "node.chart.gpu_busy": "Meşgul GPU, karta göre",
-  "node.chart.gpu_busy_note": "Her örnekleme süresinde bir çekirdek işlevinin çalıştığı pay. Bir replikanın tuttuğu ve burada boşta görünen kart, kimsenin istemediği karttır.",
+  "node.chart.gpu_busy_note": "Her örnekleme süresinde bir çekirdek işlevinin çalıştığı pay — yonganın ne kadarını kullandığı değil: tek bir SM'nin meşgul olması %100 okunur. Bir replikanın tuttuğu ve burada boşta görünen kart, kimsenin istemediği karttır.",
+  "node.chart.gpu_power_load": "GPU güç yükü, karta göre",
+  "node.chart.gpu_power_load_note": "Çekilen gücün kartın güç sınırına oranı. Meşgul bir işlevin çalıştığını söyler; bu, yonganın ne kadarında çalıştığıyla artar.",
+  "node.chart.gpu_sm_active": "Etkin SM'ler, karta göre",
+  "node.chart.gpu_sm_active_note": "Kartın SM'lerinin iş yaptığı zaman payı, profil sayaçlarından. Meşgul %90 iken bunun %60 olması, bir işlev çalışırken yonganın üçte birinin boşta olduğu demektir.",
   "node.chart.gpu_memory": "Kullanılan GPU belleği, karta göre",
   "node.chart.gpu_memory_note": "Kullanımdaki çerçeve arabelleği: bir motorun ayırdığı ağırlıklar ve KV önbelleği, ayrıca llmgw'ye ait olmayan her şey.",
   "node.chart.gpu_temp": "GPU sıcaklığı, °C",

internal/web/node_page.go

@@ -51,8 +51,11 @@ type nodeCharts struct {
 	CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries
 	// The GPUs', one line per card; empty on a node without them.
 	GPUUtil, GPUMemory, GPUTemp, GPUPower TimeSeries
-	Window                                string
+	// GPUPowerLoad is draw against the power limit; GPUSMActive the SMs'
-	Err                                   string
+	// share of time with work, empty on a card nothing can profile.
+	GPUPowerLoad, GPUSMActive TimeSeries
+	Window                    string
+	Err                       string
 }
 
 // nodePanels are the charts, each a query scoped to this node. %s is the
@@ -89,6 +92,10 @@ var gpuPanels = []struct {
 		func(c *nodeCharts, t TimeSeries) { c.GPUTemp = t }},
 	{`llmgw_gpu_power_watts{node=%s}`, "",
 		func(c *nodeCharts, t TimeSeries) { c.GPUPower = t }},
+	{`llmgw_gpu_power_watts{node=%[1]s} / llmgw_gpu_power_limit_watts{node=%[1]s}`, "%",
+		func(c *nodeCharts, t TimeSeries) { c.GPUPowerLoad = t }},
+	{`llmgw_gpu_sm_active_percent{node=%s} / 100`, "%",
+		func(c *nodeCharts, t TimeSeries) { c.GPUSMActive = t }},
 }
 
 // NodePage shows one node. Its charts come from the metrics store when one

internal/web/templates/dashboard.html

@@ -64,11 +64,13 @@
 {{/* Per card, because a model on eight cards making twice the tokens of one
      on two is the worse of the two; per watt-hour, because that is what the
      room is paying. */}}
-<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.eff.tps"}}</th><th class="num">{{t "dash.eff.per_gpu"}}</th><th class="num">{{t "dash.eff.busy"}}</th><th class="num">{{t "dash.eff.draw"}}</th><th class="num">{{t "dash.eff.per_wh"}}</th></tr>
+<table class="rows"><tr><th>{{t "dash.col.model"}}</th><th class="num">GPUs</th><th class="num">{{t "dash.eff.tps"}}</th><th class="num">{{t "dash.eff.per_gpu"}}</th><th class="num">{{t "dash.eff.busy"}}</th><th class="num">{{t "dash.eff.power_load"}}</th><th class="num">{{t "dash.eff.sm_active"}}</th><th class="num">{{t "dash.eff.draw"}}</th><th class="num">{{t "dash.eff.per_wh"}}</th></tr>
 {{range .}}<tr><td>{{.Model}}</td><td class="num">{{num .GPUs 0}}</td><td class="num">{{num .TokensPerSec 1}}</td>
   <td class="num">{{num .PerGPU 1}}</td><td class="num">{{pc .Util 0}}</td>
+  <td class="num">{{if ge .PowerLoad 0.0}}{{pc .PowerLoad 0}}{{else}}—{{end}}</td><td class="num">{{if ge .SMActive 0.0}}{{pc .SMActive 0}}{{else}}—{{end}}</td>
   <td class="num">{{if .Watts}}{{num .Watts 0}} W{{else}}—{{end}}</td><td class="num">{{if .PerWh}}{{num .PerWh 0}}{{else}}—{{end}}</td></tr>
 {{end}}</table>
+<p class="note">{{t "dash.eff.note"}}</p>
 {{end}}
 {{with .IdleGPUs}}
 <h3>{{t "dash.idle.title"}}</h3>
@@ -282,6 +284,10 @@ document.querySelectorAll("form.build-choice select").forEach(function (s) {
   {{template "tschart" dict "t" .GPUBusy "title" (t "dash.fleet.gpu_busy") "note" (t "dash.fleet.gpu_busy_note")}}
   {{template "tschart" dict "t" .GPUMemory "title" (t "dash.fleet.gpu_memory") "note" (t "dash.fleet.gpu_memory_note")}}
 </div>
+<div class="chart-row">
+  {{template "tschart" dict "t" .GPUPowerLoad "title" (t "dash.fleet.gpu_power_load") "note" (t "dash.fleet.gpu_power_load_note")}}
+  {{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "dash.fleet.gpu_sm_active") "note" (t "dash.fleet.gpu_sm_active_note")}}{{end}}
+</div>
 {{end}}
 <div class="chart-row">
   {{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "dash.fleet.queue_note")}}

internal/web/templates/node.html

@@ -141,6 +141,10 @@
   {{template "tschart" dict "t" .GPUTemp "title" (t "node.chart.gpu_temp") "note" (t "node.chart.gpu_temp_note")}}
   {{template "tschart" dict "t" .GPUPower "title" (t "node.chart.gpu_power") "note" (t "node.chart.gpu_power_note")}}
 </div>
+<div class="chart-row">
+  {{template "tschart" dict "t" .GPUPowerLoad "title" (t "node.chart.gpu_power_load") "note" (t "node.chart.gpu_power_load_note")}}
+  {{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "node.chart.gpu_sm_active") "note" (t "node.chart.gpu_sm_active_note")}}{{end}}
+</div>
 {{end}}
 <div class="chart-row">
   {{template "tschart" dict "t" .Load "title" (t "node.chart.load") "note" (t "node.chart.load_note")}}