diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d5a40..e43a186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,24 @@ APIs change between minors without a deprecation period. ## Unreleased +### Added + +- **How much of each GPU works, beside whether anything does.** The busy + figure is nvidia-smi's `utilization.gpu` — the share of time any kernel + ran, which a kernel on one SM fills — and it is now said to be that: on + the dashboard, the node page, the metric's help and Grafana. Beside it, + each card's draw against its power limit (the efficiency table, the fleet + and node charts, the console's GPU work chart), and where the card's + profiling counters can be read, its SMs, SM occupancy, tensor cores and + memory interface: `llmgw_gpu_sm_active_percent`, + `llmgw_gpu_sm_occupancy_percent`, `llmgw_gpu_tensor_active_percent`, + `llmgw_gpu_dram_active_percent`. The agent reads them from NVML's GPM on + Hopper and later, and from a dcgm-exporter on the node elsewhere + (`discovery.dcgm_exporter`, default `http://127.0.0.1:9400/metrics`, `off` + for none). The GPU map colours by this "work" by default — SMs active + where read, power against the limit otherwise — and keeps busy as a mode. + On cuda01 decode read 92% busy with the SMs at work 69% of the time. + ### Fixed - **A model of several files showed one file's download as the whole.** A node fetching a set reported the part it diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 084897b..9c2814a 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -2219,12 +2219,27 @@ type GPUSensor struct { NvlinkReplayErrors int64 `protobuf:"varint,23,opt,name=nvlink_replay_errors,json=nvlinkReplayErrors,proto3" json:"nvlink_replay_errors,omitempty"` NvlinkRecoveryErrors int64 `protobuf:"varint,24,opt,name=nvlink_recovery_errors,json=nvlinkRecoveryErrors,proto3" json:"nvlink_recovery_errors,omitempty"` // xid_errors are the driver's error events since the agent started. - XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"` - LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"` - LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"` - LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + XidErrors int64 `protobuf:"varint,25,opt,name=xid_errors,json=xidErrors,proto3" json:"xid_errors,omitempty"` + LastXid int32 `protobuf:"varint,26,opt,name=last_xid,json=lastXid,proto3" json:"last_xid,omitempty"` + LastXidText string `protobuf:"bytes,27,opt,name=last_xid_text,json=lastXidText,proto3" json:"last_xid_text,omitempty"` + LastXidUnix int64 `protobuf:"varint,28,opt,name=last_xid_unix,json=lastXidUnix,proto3" json:"last_xid_unix,omitempty"` + // How much of the chip the work on it uses, which utilization_percent + // does not say: that is the share of time any kernel ran, and a kernel on + // one SM of a hundred counts in full. has_profile says these are present; + // -1 is not reported. Read from NVML's GPM on Hopper and later, or from a + // dcgm-exporter on the node; profile_source says which ("gpm", "dcgm"). + HasProfile bool `protobuf:"varint,29,opt,name=has_profile,json=hasProfile,proto3" json:"has_profile,omitempty"` + // sm_active_percent is the share of time the SMs had work, averaged over + // all of them; sm_occupancy_percent how full those SMs were with warps. + SmActivePercent float64 `protobuf:"fixed64,30,opt,name=sm_active_percent,json=smActivePercent,proto3" json:"sm_active_percent,omitempty"` + SmOccupancyPercent float64 `protobuf:"fixed64,31,opt,name=sm_occupancy_percent,json=smOccupancyPercent,proto3" json:"sm_occupancy_percent,omitempty"` + // tensor_active_percent is the tensor cores; dram_active_percent the + // memory interface, which decode is bound by. + TensorActivePercent float64 `protobuf:"fixed64,32,opt,name=tensor_active_percent,json=tensorActivePercent,proto3" json:"tensor_active_percent,omitempty"` + DramActivePercent float64 `protobuf:"fixed64,33,opt,name=dram_active_percent,json=dramActivePercent,proto3" json:"dram_active_percent,omitempty"` + ProfileSource string `protobuf:"bytes,34,opt,name=profile_source,json=profileSource,proto3" json:"profile_source,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GPUSensor) Reset() { @@ -2453,6 +2468,48 @@ func (x *GPUSensor) GetLastXidUnix() int64 { return 0 } +func (x *GPUSensor) GetHasProfile() bool { + if x != nil { + return x.HasProfile + } + return false +} + +func (x *GPUSensor) GetSmActivePercent() float64 { + if x != nil { + return x.SmActivePercent + } + return 0 +} + +func (x *GPUSensor) GetSmOccupancyPercent() float64 { + if x != nil { + return x.SmOccupancyPercent + } + return 0 +} + +func (x *GPUSensor) GetTensorActivePercent() float64 { + if x != nil { + return x.TensorActivePercent + } + return 0 +} + +func (x *GPUSensor) GetDramActivePercent() float64 { + if x != nil { + return x.DramActivePercent + } + return 0 +} + +func (x *GPUSensor) GetProfileSource() string { + if x != nil { + return x.ProfileSource + } + return "" +} + // ShardLoad is one engine process group's use. type ShardLoad struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -5377,7 +5434,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + " \x01(\x01R\tcpuPowerW\x12 \n" + "\finlet_temp_c\x18\v \x01(\x01R\n" + "inletTempC\x12'\n" + - "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xde\b\n" + + "\x04gpus\x18\f \x03(\v2\x13.llmgw.v1.GPUSensorR\x04gpus\"\xe8\n" + + "\n" + "\tGPUSensor\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x15\n" + "\x06temp_c\x18\x02 \x01(\x01R\x05tempC\x12\"\n" + @@ -5409,7 +5467,14 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "xid_errors\x18\x19 \x01(\x03R\txidErrors\x12\x19\n" + "\blast_xid\x18\x1a \x01(\x05R\alastXid\x12\"\n" + "\rlast_xid_text\x18\x1b \x01(\tR\vlastXidText\x12\"\n" + - "\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\"m\n" + + "\rlast_xid_unix\x18\x1c \x01(\x03R\vlastXidUnix\x12\x1f\n" + + "\vhas_profile\x18\x1d \x01(\bR\n" + + "hasProfile\x12*\n" + + "\x11sm_active_percent\x18\x1e \x01(\x01R\x0fsmActivePercent\x120\n" + + "\x14sm_occupancy_percent\x18\x1f \x01(\x01R\x12smOccupancyPercent\x122\n" + + "\x15tensor_active_percent\x18 \x01(\x01R\x13tensorActivePercent\x12.\n" + + "\x13dram_active_percent\x18! \x01(\x01R\x11dramActivePercent\x12%\n" + + "\x0eprofile_source\x18\" \x01(\tR\rprofileSource\"m\n" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index a364907..e605e85 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -448,6 +448,21 @@ message GPUSensor { int32 last_xid = 26; string last_xid_text = 27; int64 last_xid_unix = 28; + // How much of the chip the work on it uses, which utilization_percent + // does not say: that is the share of time any kernel ran, and a kernel on + // one SM of a hundred counts in full. has_profile says these are present; + // -1 is not reported. Read from NVML's GPM on Hopper and later, or from a + // dcgm-exporter on the node; profile_source says which ("gpm", "dcgm"). + bool has_profile = 29; + // sm_active_percent is the share of time the SMs had work, averaged over + // all of them; sm_occupancy_percent how full those SMs were with warps. + double sm_active_percent = 30; + double sm_occupancy_percent = 31; + // tensor_active_percent is the tensor cores; dram_active_percent the + // memory interface, which decode is bound by. + double tensor_active_percent = 32; + double dram_active_percent = 33; + string profile_source = 34; } // ShardLoad is one engine process group's use. diff --git a/deploy/monitoring/grafana/llmgw-overview.json b/deploy/monitoring/grafana/llmgw-overview.json index cfe385b..68360ea 100644 --- a/deploy/monitoring/grafana/llmgw-overview.json +++ b/deploy/monitoring/grafana/llmgw-overview.json @@ -504,7 +504,7 @@ { "type": "timeseries", "title": "GPU busy, by card", - "description": "Share of each sample period a kernel was running, sampled every heartbeat.", + "description": "Share of each sample period in which any kernel ran (nvidia-smi's utilization.gpu), sampled every heartbeat. Not how much of the chip: one SM busy reads 100%. The SMs panel and power against its limit say how much.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" @@ -908,6 +908,51 @@ "mode": "multi" } } + }, + { + "type": "timeseries", + "title": "GPU SMs and tensor cores active, by card", + "description": "From the cards' profiling counters: NVML's GPM on Hopper and later, or a dcgm-exporter on the node. SMs active is the share of time the SMs had work, averaged over all of them; the gap to busy is chip a running kernel leaves idle. Tensor cores stay low in decode at batch one; prefill and batches are where they rise. Empty for cards nothing can profile.", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "gridPos": { + "x": 0, + "y": 73, + "w": 24, + "h": 7 + }, + "id": 25, + "targets": [ + { + "expr": "llmgw_gpu_sm_active_percent", + "legendFormat": "{{node}} gpu {{gpu}} SMs", + "refId": "A" + }, + { + "expr": "llmgw_gpu_tensor_active_percent", + "legendFormat": "{{node}} gpu {{gpu}} tensor", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } } ] } \ No newline at end of file diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index f4a04d9..92fed22 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -11019,6 +11019,30 @@ } } }, + "inventory.GPUProfile": { + "type": "object", + "description": "GPUProfile is a card's profiling counters, in percent; -1 is not reported. UtilPercent is the share of time any kernel ran, and a kernel on one SM of a hundred counts in full; these say how much of the chip it ran on.", + "properties": { + "sm_active": { + "type": "number", + "description": "SMActive is the share of time the SMs had work, averaged over all of them; SMOccupancy how full those SMs were with warps." + }, + "sm_occupancy": { + "type": "number" + }, + "tensor_active": { + "type": "number", + "description": "TensorActive is the tensor cores, DRAMActive the memory interface." + }, + "dram_active": { + "type": "number" + }, + "source": { + "type": "string", + "description": "Source is what read them: \"gpm\" (NVML, Hopper and later) or \"dcgm\"." + } + } + }, "inventory.GPUSensor": { "type": "object", "description": "GPUSensor is one card's heat and power, and what is throttling it.", @@ -11066,6 +11090,10 @@ "wear": { "$ref": "#/components/schemas/inventory.GPUWear", "description": "Wear is what wears the card out, read every minute; nil from an agent that does not read it." + }, + "profile": { + "$ref": "#/components/schemas/inventory.GPUProfile", + "description": "Profile is how much of the chip the work uses, which UtilPercent does not say; nil where nothing on the node can read it." } } }, diff --git a/docs/install.md b/docs/install.md index 9d16227..fa49961 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2019,12 +2019,31 @@ dashboard, is in [deploy/monitoring/](../deploy/monitoring/) — including where to run it, which is not the control plane: monitoring that dies with the thing it monitors tells you nothing at the moment you need it. -Two figures repay knowing about. `llmgw_gpu_utilization_percent` is absent +Three figures repay knowing about. `llmgw_gpu_utilization_percent` is absent rather than zero when the driver does not report it — a flat zero across a fleet means the measurement is missing, not that the fleet is idle. And `llmgw_gpu_nvlink_peers` at zero everywhere means the topology is unknown, which is what a binary built without the nvml tag reports. +The third is what "busy" means. `llmgw_gpu_utilization_percent` is +nvidia-smi's `utilization.gpu`: the share of time *any* kernel ran, however +little of the chip it ran on — a kernel on one SM of 142 reads 100%. A Quadro +RTX 6000 decoding read 92% busy while its SMs had work 69% of the time and +issued an instruction on 18% of their cycles. How much of the chip is at work +is in `llmgw_gpu_sm_active_percent`, `llmgw_gpu_sm_occupancy_percent`, +`llmgw_gpu_tensor_active_percent` and `llmgw_gpu_dram_active_percent`, from +the card's profiling counters. The agent reads them from NVML's GPM on +Hopper and later, with nothing to install; on older cards it reads them from +a [dcgm-exporter](https://github.com/NVIDIA/dcgm-exporter) on the node, at +`http://127.0.0.1:9400/metrics` unless `discovery.dcgm_exporter` in +agent.yaml says otherwise (`off` reads none). Where neither is there, the +dashboard's GPU map and the console colour cards by their draw against the +power limit (`llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts`), which +rises with how much of the chip works rather than with whether anything +does. Low tensor-core activity in decode is not waste: a quantised +matrix-vector product at batch one runs on the integer units, and it is +prefill and batches that use the tensor cores. + ### Token by token, and SLOs Beside time to the first token, every streamed answer is timed per token: diff --git a/docs/reference.md b/docs/reference.md index fa3470d..5ffe50d 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -234,6 +234,7 @@ rather than repeated. | `ib_root` | `string` | [Options](#options) | | `prefer_nvml` | `bool` | [Options](#options) | | `skip_gpus` | `bool` | [Options](#options) | +| `dcgm_exporter` | `string` | [Options](#options) | | `repo` | `string` | [Origin](#origin) | | `revision` | `string` | [Origin](#origin) | | `files` | `[]OriginFile` | [Origin](#origin) | @@ -2182,6 +2183,15 @@ is used, is the more exact source of the peers. SkipGPUs omits GPU discovery. A node without nvidia-smi is already found to have none, so this is for one whose driver is broken. +### `dcgm_exporter` + +`string` + +DCGMExporter is the dcgm-exporter metrics URL read for how much of +each card's chip is in use, on cards NVML's GPM does not cover (before +Hopper). Empty is http://127.0.0.1:9400/metrics; "off" reads none. A +node without one costs a refused connection every five minutes. + ## Origin ### `repo` diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 22c177b..1248328 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -187,7 +187,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er logRing: ring, provKick: make(chan struct{}, 1), load: discovery.NewLoadSampler(), - sensors: discovery.NewSensorSampler(), + sensors: discovery.NewSensorSampler().WithProfile(discovery.NewProfSampler(cfg.Discovery.DCGMExporter)), wear: discovery.NewHealthSampler(), serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()), kv: kvevents.NewMirror(), @@ -798,6 +798,11 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad { MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB, SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected, } + if p := g.Profile; p != nil { + gs.HasProfile, gs.ProfileSource = true, p.Source + gs.SmActivePercent, gs.SmOccupancyPercent = p.SMActive, p.SMOccupancy + gs.TensorActivePercent, gs.DramActivePercent = p.TensorActive, p.DRAMActive + } if w, ok := wear[g.Index]; ok { gs.HasWear = true gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending = diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index e8081b7..b7c0527 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -28,6 +28,11 @@ type Options struct { // SkipGPUs omits GPU discovery. A node without nvidia-smi is already // found to have none, so this is for one whose driver is broken. SkipGPUs bool `koanf:"skip_gpus"` + // DCGMExporter is the dcgm-exporter metrics URL read for how much of + // each card's chip is in use, on cards NVML's GPM does not cover (before + // Hopper). Empty is http://127.0.0.1:9400/metrics; "off" reads none. A + // node without one costs a refused connection every five minutes. + DCGMExporter string `koanf:"dcgm_exporter"` // ModelDirs are scanned for weights. Empty means do not look. ModelDirs []string `koanf:"-"` } diff --git a/internal/agent/discovery/gpm_nonvml.go b/internal/agent/discovery/gpm_nonvml.go new file mode 100644 index 0000000..986992d --- /dev/null +++ b/internal/agent/discovery/gpm_nonvml.go @@ -0,0 +1,6 @@ +//go:build !nvml + +package discovery + +// newGPM has nothing to read GPM with in a build without the nvml tag. +func newGPM() gpmSource { return nil } diff --git a/internal/agent/discovery/gpm_nvml.go b/internal/agent/discovery/gpm_nvml.go new file mode 100644 index 0000000..f08ca7b --- /dev/null +++ b/internal/agent/discovery/gpm_nvml.go @@ -0,0 +1,124 @@ +//go:build nvml + +package discovery + +import ( + "sync" + + "github.com/NVIDIA/go-nvml/pkg/nvml" +) + +// nvmlGPM reads GPM, NVML's own profiling counters, on the cards that have +// them: Hopper and later. Each read takes a sample and works out the metrics +// against the one before, so NVML stays initialised between reads. +type nvmlGPM struct { + mu sync.Mutex + opened bool + failed bool + cards map[int]*gpmCard +} + +type gpmCard struct { + dev nvml.Device + prev, cur nvml.GpmSample + havePrevious bool +} + +func newGPM() gpmSource { return &nvmlGPM{} } + +// The GPM metrics read, in the order they are asked for. +var gpmMetrics = []struct { + id nvml.GpmMetricId + set func(*GPUProfile, float64) +}{ + {nvml.GPM_METRIC_SM_UTIL, func(p *GPUProfile, v float64) { p.SMActive = v }}, + {nvml.GPM_METRIC_SM_OCCUPANCY, func(p *GPUProfile, v float64) { p.SMOccupancy = v }}, + {nvml.GPM_METRIC_ANY_TENSOR_UTIL, func(p *GPUProfile, v float64) { p.TensorActive = v }}, + {nvml.GPM_METRIC_DRAM_BW_UTIL, func(p *GPUProfile, v float64) { p.DRAMActive = v }}, +} + +func (g *nvmlGPM) read() map[int]GPUProfile { + g.mu.Lock() + defer g.mu.Unlock() + if g.failed { + return nil + } + if !g.opened && !g.open() { + g.failed = true + return nil + } + out := map[int]GPUProfile{} + for idx, c := range g.cards { + if c.cur.Get(c.dev) != nvml.SUCCESS { + // A card in MIG mode, or one that has gone: no sample, and the + // next must not be measured against an old one. + c.havePrevious = false + continue + } + if c.havePrevious { + if p, ok := gpmBetween(c.prev, c.cur); ok { + out[idx] = p + } + } + c.prev, c.cur = c.cur, c.prev + c.havePrevious = true + } + return out +} + +// open finds the cards with GPM. False is NVML not starting, or no card +// having it, and the reader then never tries again. +func (g *nvmlGPM) open() bool { + if nvml.Init() != nvml.SUCCESS { + return false + } + g.cards = map[int]*gpmCard{} + n, ret := nvml.DeviceGetCount() + if ret != nvml.SUCCESS { + n = 0 + } + for i := range n { + dev, ret := nvml.DeviceGetHandleByIndex(i) + if ret != nvml.SUCCESS { + continue + } + sup, ret := dev.GpmQueryDeviceSupport() + if ret != nvml.SUCCESS || sup.IsSupportedDevice == 0 { + continue + } + a, ret := nvml.GpmSampleAlloc() + if ret != nvml.SUCCESS { + continue + } + b, ret := nvml.GpmSampleAlloc() + if ret != nvml.SUCCESS { + _ = a.Free() + continue + } + g.cards[i] = &gpmCard{dev: dev, prev: a, cur: b} + } + if len(g.cards) == 0 { + _ = nvml.Shutdown() + return false + } + g.opened = true + return true +} + +// gpmBetween is the profile over the time between two samples. +func gpmBetween(earlier, later nvml.GpmSample) (GPUProfile, bool) { + q := nvml.GpmMetricsGetType{NumMetrics: uint32(len(gpmMetrics)), Sample1: earlier, Sample2: later} + for i, m := range gpmMetrics { + q.Metrics[i].MetricId = uint32(m.id) + } + if nvml.GpmMetricsGet(&q) != nvml.SUCCESS { + return GPUProfile{}, false + } + p := GPUProfile{SMActive: -1, SMOccupancy: -1, TensorActive: -1, DRAMActive: -1, Source: "gpm"} + for i, m := range gpmMetrics { + if r := q.Metrics[i]; r.NvmlReturn == uint32(nvml.SUCCESS) && r.Value >= 0 { + m.set(&p, r.Value) + } + } + return p, true +} diff --git a/internal/agent/discovery/prof.go b/internal/agent/discovery/prof.go new file mode 100644 index 0000000..aa323c0 --- /dev/null +++ b/internal/agent/discovery/prof.go @@ -0,0 +1,176 @@ +package discovery + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// GPUProfile is how much of a card's chip the work on it uses. nvidia-smi's +// utilization.gpu cannot say: it is the share of time any kernel ran, and a +// kernel keeping one SM of a hundred busy reads 100%. Measured on a Quadro +// RTX 6000 under decode, it read 92% while the SMs had work 69% of the time +// and issued an instruction on 18% of their cycles. +// +// These come from the card's profiling counters, which only two things read +// without stopping the work: NVML's GPM, on Hopper and later, and DCGM, which +// a site may run as dcgm-exporter. Percentages; -1 is not reported. +type GPUProfile struct { + // SMActive is the share of time the SMs had work, averaged over all of + // them; SMOccupancy how full those SMs were with warps. + SMActive, SMOccupancy float64 + // TensorActive is the tensor cores. Decode at batch one hardly uses them + // even when nothing is wrong: a quantised matrix-vector product runs on + // the integer units. Prefill and batches are where they show. + TensorActive float64 + // DRAMActive is the memory interface, which decode is bound by. + DRAMActive float64 + // Source is "gpm" or "dcgm". + Source string +} + +// DefaultDCGMExporter is where dcgm-exporter listens unless told otherwise. +const DefaultDCGMExporter = "http://127.0.0.1:9400/metrics" + +// dcgmRetry is how long a dcgm-exporter that did not answer is left alone. +// Most nodes run none, and asking every ten seconds would be asking nobody. +const dcgmRetry = 5 * time.Minute + +// gpmSource reads GPM, where this build and the card have it. +type gpmSource interface { + read() map[int]GPUProfile +} + +// ProfSampler reads each card's GPUProfile: from GPM where the card has it, +// otherwise from a dcgm-exporter on the node. +type ProfSampler struct { + gpm gpmSource + dcgm string + get func(ctx context.Context, url string) ([]byte, error) + now func() time.Time + + dcgmAfter time.Time +} + +// NewProfSampler reads this machine. dcgmURL is the dcgm-exporter to ask: +// empty is DefaultDCGMExporter, "off" asks none. +func NewProfSampler(dcgmURL string) *ProfSampler { + switch dcgmURL { + case "": + dcgmURL = DefaultDCGMExporter + case "off": + dcgmURL = "" + } + return &ProfSampler{gpm: newGPM(), dcgm: dcgmURL, get: httpGet, now: time.Now} +} + +// Sample is each card's profile by index; a card missing has none. GPM needs +// two samples, so its cards appear from the second call. +func (p *ProfSampler) Sample(ctx context.Context) map[int]GPUProfile { + out := map[int]GPUProfile{} + if p.gpm != nil { + for i, v := range p.gpm.read() { + out[i] = v + } + } + if p.dcgm == "" || p.now().Before(p.dcgmAfter) { + return out + } + body, err := p.get(ctx, p.dcgm) + if err != nil { + p.dcgmAfter = p.now().Add(dcgmRetry) + return out + } + for i, v := range parseDCGM(body) { + if _, ok := out[i]; !ok { + out[i] = v + } + } + return out +} + +func httpGet(ctx context.Context, url string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, io.ErrUnexpectedEOF + } + return io.ReadAll(io.LimitReader(resp.Body, 4<<20)) +} + +// The dcgm-exporter fields read, as ratios of one. Its default field list +// has them on every card DCGM can profile. +var dcgmFields = map[string]func(*GPUProfile, float64){ + "DCGM_FI_PROF_SM_ACTIVE": func(p *GPUProfile, v float64) { p.SMActive = v }, + "DCGM_FI_PROF_SM_OCCUPANCY": func(p *GPUProfile, v float64) { p.SMOccupancy = v }, + "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE": func(p *GPUProfile, v float64) { p.TensorActive = v }, + "DCGM_FI_PROF_DRAM_ACTIVE": func(p *GPUProfile, v float64) { p.DRAMActive = v }, +} + +// parseDCGM reads dcgm-exporter's text exposition: a line per field and +// card, the card in the gpu label. A card with none of the fields — DCGM +// cannot profile it — is left out. A MIG instance's lines carry a +// GPU_I_ID and are left out too: these are whole cards. +func parseDCGM(body []byte) map[int]GPUProfile { + out := map[int]GPUProfile{} + for _, line := range strings.Split(string(body), "\n") { + if line == "" || line[0] == '#' { + continue + } + open := strings.IndexByte(line, '{') + end := strings.LastIndexByte(line, '}') + if open < 0 || end < open { + continue + } + set, ok := dcgmFields[line[:open]] + if !ok { + continue + } + labels := line[open+1 : end] + if strings.Contains(labels, `GPU_I_ID="`) && !strings.Contains(labels, `GPU_I_ID=""`) { + continue + } + idx, err := strconv.Atoi(labelValue(labels, "gpu")) + if err != nil { + continue + } + val := strings.Fields(line[end+1:]) + if len(val) == 0 { + continue + } + v, err := strconv.ParseFloat(val[0], 64) + if err != nil || v < 0 { + continue + } + p, seen := out[idx] + if !seen { + p = GPUProfile{SMActive: -1, SMOccupancy: -1, TensorActive: -1, DRAMActive: -1, Source: "dcgm"} + } + set(&p, 100*v) + out[idx] = p + } + return out +} + +// labelValue is one label's value from a Prometheus label set. +func labelValue(labels, name string) string { + for _, kv := range strings.Split(labels, ",") { + k, v, ok := strings.Cut(strings.TrimSpace(kv), "=") + if ok && k == name { + return strings.Trim(v, `"`) + } + } + return "" +} diff --git a/internal/agent/discovery/prof_test.go b/internal/agent/discovery/prof_test.go new file mode 100644 index 0000000..490687c --- /dev/null +++ b/internal/agent/discovery/prof_test.go @@ -0,0 +1,107 @@ +package discovery + +import ( + "context" + "errors" + "testing" + "time" +) + +// dcgmBody is dcgm-exporter's output for two cards and a MIG instance: +// card 0 profiled, card 1 a model DCGM cannot profile, which has the device +// fields and none of the PROF ones. +const dcgmBody = `# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %). +# TYPE DCGM_FI_DEV_GPU_UTIL gauge +DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 100 +DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-b",device="nvidia1",modelName="Quadro RTX 6000",Hostname="n1"} 92 +# HELP DCGM_FI_PROF_SM_ACTIVE The ratio of cycles an SM has at least 1 warp assigned. +# TYPE DCGM_FI_PROF_SM_ACTIVE gauge +DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.007042 +DCGM_FI_PROF_SM_ACTIVE{gpu="2",UUID="GPU-c",device="nvidia2",modelName="NVIDIA A100",GPU_I_PROFILE="3g.40gb",GPU_I_ID="1",Hostname="n1"} 0.5 +DCGM_FI_PROF_SM_OCCUPANCY{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.25 +DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0 +` + +func TestDCGMExporterSaysHowMuchOfEachChipIsInUse(t *testing.T) { + got := parseDCGM([]byte(dcgmBody)) + if len(got) != 1 { + t.Fatalf("want card 0 alone (card 1 unprofiled, card 2 a MIG instance): %+v", got) + } + p := got[0] + // The LinkedIn L40S: 100% busy, one SM of 142 at work. + if p.SMActive < 0.70 || p.SMActive > 0.71 || p.SMOccupancy != 25 || p.TensorActive != 0 || p.Source != "dcgm" { + t.Errorf("card 0 %+v", p) + } + // A field the exporter does not send is unknown, not zero. + if p.DRAMActive != -1 { + t.Errorf("DRAM active not sent, read as %v", p.DRAMActive) + } +} + +type fakeGPM map[int]GPUProfile + +func (f fakeGPM) read() map[int]GPUProfile { return f } + +func TestGPMIsPreferredAndDCGMFillsTheCardsItDoesNotCover(t *testing.T) { + calls := 0 + p := &ProfSampler{ + gpm: fakeGPM{1: {SMActive: 40, Source: "gpm"}}, + dcgm: DefaultDCGMExporter, + get: func(context.Context, string) ([]byte, error) { + calls++ + return []byte(`DCGM_FI_PROF_SM_ACTIVE{gpu="0"} 0.5` + "\n" + `DCGM_FI_PROF_SM_ACTIVE{gpu="1"} 0.9` + "\n"), nil + }, + now: time.Now, + } + got := p.Sample(t.Context()) + if got[0].Source != "dcgm" || got[0].SMActive != 50 { + t.Errorf("card 0 %+v", got[0]) + } + if got[1].Source != "gpm" || got[1].SMActive != 40 { + t.Errorf("GPM's card was taken from DCGM: %+v", got[1]) + } + if calls != 1 { + t.Errorf("%d calls", calls) + } +} + +func TestANodeWithoutDCGMIsNotAskedEverySample(t *testing.T) { + now := time.Now() + calls := 0 + p := &ProfSampler{dcgm: DefaultDCGMExporter, now: func() time.Time { return now }, + get: func(context.Context, string) ([]byte, error) { calls++; return nil, errors.New("connection refused") }} + for range 5 { + if got := p.Sample(t.Context()); len(got) != 0 { + t.Fatalf("%+v", got) + } + now = now.Add(10 * time.Second) + } + if calls != 1 { + t.Errorf("asked %d times in 50 s", calls) + } + now = now.Add(dcgmRetry) + p.Sample(t.Context()) + if calls != 2 { + t.Errorf("not asked again after %v", dcgmRetry) + } + if off := NewProfSampler("off"); off.dcgm != "" { + t.Errorf("off still asks %q", off.dcgm) + } +} + +func TestEachSampleCarriesTheProfileOfTheCardsThatHaveOne(t *testing.T) { + smi := func(context.Context, string) ([]byte, error) { + return []byte("0, 71, 239, 260, 0x0, 92, 30, 23000, 24576, 1878, 0, 0\n" + + "1, 30, 20, 260, 0x0, 0, 0, 512, 24576, 300, 0, 0\n"), nil + } + now := time.Now() + s := sampler(t.TempDir(), smi, &now).WithProfile(&ProfSampler{ + gpm: fakeGPM{0: {SMActive: 69, SMOccupancy: 45, TensorActive: 1, DRAMActive: 33, Source: "gpm"}}, now: time.Now}) + got := s.Sample(t.Context()).GPUs + if got[0].Profile == nil || got[0].Profile.SMActive != 69 || got[0].UtilPercent != 92 { + t.Errorf("card 0 %+v", got[0]) + } + if got[1].Profile != nil { + t.Errorf("card 1 has a profile nobody read: %+v", got[1].Profile) + } +} diff --git a/internal/agent/discovery/sensors.go b/internal/agent/discovery/sensors.go index f5e5d66..4b3d3c3 100644 --- a/internal/agent/discovery/sensors.go +++ b/internal/agent/discovery/sensors.go @@ -49,6 +49,9 @@ type GPUSensor struct { MemoryUsedMB, MemoryTotalMB float64 SMClockMHz float64 ECCUncorrected float64 + // Profile is how much of the chip the work uses, where something on the + // node can say: nil on most cards. UtilPercent is not that. + Profile *GPUProfile } // sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often @@ -61,6 +64,8 @@ type SensorSampler struct { sys string // /sys, or a test's copy of it smi func(ctx context.Context, fields string) ([]byte, error) now func() time.Time + // prof reads the cards' profiling counters; nil reads none. + prof *ProfSampler mu sync.Mutex last Sensors @@ -74,6 +79,12 @@ func NewSensorSampler() *SensorSampler { return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now} } +// WithProfile has each sample carry the cards' profiles as well. +func (s *SensorSampler) WithProfile(p *ProfSampler) *SensorSampler { + s.prof = p + return s +} + // SampleGPUs reads every card's heat, power and how busy it is now, without // a sampler's ten seconds of memory: for a screen that asks every few seconds // and wants what it asked for. Nil on a machine without nvidia-smi or a card. @@ -101,6 +112,14 @@ func (s *SensorSampler) Sample(ctx context.Context) Sensors { out.CPUTempC, out.InletTempC = s.hwmon() out.CPUPowerW = s.rapl(now) out.GPUs = s.gpus(ctx) + if s.prof != nil { + profs := s.prof.Sample(ctx) + for i := range out.GPUs { + if p, ok := profs[out.GPUs[i].Index]; ok { + out.GPUs[i].Profile = &p + } + } + } s.last, s.lastAt = out, now return out } diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 03bddbc..a53f1d6 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -1497,6 +1497,10 @@ func (m consoleModel) gpuRows() [][]string { } if !mig { busy = fmt.Sprintf("%.0f%%", s.UtilPercent) + if p := s.Profile; p != nil && p.SMActive >= 0 { + // How much of the chip, beside whether anything ran. + busy += fmt.Sprintf(" · SM %.0f%%", p.SMActive) + } if s.MemoryTotalMB > 0 { mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024) } diff --git a/internal/cli/tui_monitor.go b/internal/cli/tui_monitor.go index 1e8d8ea..954515c 100644 --- a/internal/cli/tui_monitor.go +++ b/internal/cli/tui_monitor.go @@ -14,6 +14,7 @@ import ( "github.com/charmbracelet/lipgloss" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" ) @@ -80,6 +81,9 @@ func (s *monitorState) sample(d consoleData) { for _, g := range l.GPUs { card := nd.ID + ":" + strconv.Itoa(g.Index) add("gpu/"+card, g.UtilPercent) + if w := gpuWork(g); w >= 0 { + add("gpuwork/"+card, w) + } if g.MemoryTotalMB > 0 { add("vram/"+card, 100*g.MemoryUsedMB/g.MemoryTotalMB) } @@ -142,6 +146,7 @@ func monitorQueries(window time.Duration, points int) (time.Duration, []adminapi {ID: "cpu", Expr: `avg(llmgw_node_cpu_utilization_percent)`}, {ID: "mem", Expr: `100 * sum(llmgw_node_memory_used_bytes) / sum(llmgw_node_memory_bytes)`}, {ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent`), By: "card"}, + {ID: "gpuwork", Expr: card(gpuWorkExpr("")), By: "card"}, {ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes`), By: "card"}, {ID: "gputemp", Expr: card(`llmgw_gpu_temperature_celsius`), By: "card"}, {ID: "gpupow", Expr: card(`llmgw_gpu_power_watts`), By: "card"}, @@ -314,7 +319,8 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) { b.WriteString(indent(side(w, box("CPU", pct("cpu"), m.series("cpu"), 100), box("Memory", pct("mem"), m.series("mem"), 100)), " ") + "\n\n") if g := m.series("gpu"); len(g) > 0 || len(m.data.sum.GPUs) > 0 { - b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", m.series("vram"), 100)), " ") + "\n\n") + b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU work", "", m.series("gpuwork"), 100)), " ") + "\n\n") + b.WriteString(indent(side(w, box("GPU memory", "", m.series("vram"), 100), " "+sDim.Render(gpuWorkNote)), " ") + "\n\n") } if m.mon.res.Configured { rps := m.series("rps") @@ -356,10 +362,12 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) { // Each card: busy, memory, heat and power. if gpus := m.series("gpu"); len(gpus) > 0 { vram, temp, pow := byName(m.series("vram")), byName(m.series("gputemp")), byName(m.series("gpupow")) + work := byName(m.series("gpuwork")) b.WriteString("\n " + mdBold.Render("GPUs") + "\n") for _, g := range gpus { - fmt.Fprintf(&b, " %-16s busy %s %4.0f%% memory %s %4.0f%% %3.0f°C %s %4.0f W\n", truncate(g.name, 16), - sparkSeries(g, sw/2, 100), last(g.values), sparkSeries(vram[g.name], sw/2, 100), last(vram[g.name].values), + fmt.Fprintf(&b, " %-16s busy %s %4.0f%% work %s %4.0f%% memory %s %4.0f%% %3.0f°C %s %4.0f W\n", truncate(g.name, 16), + sparkSeries(g, sw/3, 100), last(g.values), sparkSeries(work[g.name], sw/3, 100), last(work[g.name].values), + sparkSeries(vram[g.name], sw/3, 100), last(vram[g.name].values), last(temp[g.name].values), sparkSeries(temp[g.name], 8, 100), last(pow[g.name].values)) } } @@ -373,6 +381,29 @@ func (m consoleModel) monitorPanel(pw int) (string, string, string) { return "Monitor", head.String() + b.String(), keys } +// GPU busy is the driver's figure: the share of time any kernel ran, which +// a kernel on one SM of a hundred fills. GPU work is how much of the chip: +// the SMs' share of time with work where the card's profiling counters are +// read (GPM, DCGM), and its draw against its power limit elsewhere. +const gpuWorkNote = "busy: some kernel ran. work: how much of the chip —\nSMs active where the counters are read, power/limit elsewhere." + +// gpuWorkExpr is GPU work in PromQL, for the series matching sel. +func gpuWorkExpr(sel string) string { + return `llmgw_gpu_sm_active_percent` + sel + ` or (100 * llmgw_gpu_power_watts` + sel + + ` / llmgw_gpu_power_limit_watts` + sel + `)` +} + +// gpuWork is GPU work from a sample, or -1 without the figures for it. +func gpuWork(g inventory.GPUSensor) float64 { + if p := g.Profile; p != nil && p.SMActive >= 0 { + return p.SMActive + } + if l := g.PowerLoad(); l >= 0 { + return 100 * l + } + return -1 +} + func named(s []chartSeries, name string) []chartSeries { for i := range s { s[i].name = name @@ -410,6 +441,7 @@ func detailQueries(kind, id string, window time.Duration, points int) (time.Dura {ID: "cpu", Expr: `llmgw_node_cpu_utilization_percent{node=` + q + `}`}, {ID: "mem", Expr: `100 * llmgw_node_memory_used_bytes{node=` + q + `} / llmgw_node_memory_bytes{node=` + q + `}`}, {ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent{node=` + q + `}`), By: "card"}, + {ID: "gpuwork", Expr: card(gpuWorkExpr(`{node=` + q + `}`)), By: "card"}, {ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes{node=` + q + `} / llmgw_gpu_memory_total_bytes{node=` + q + `}`), By: "card"}, {ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`}, {ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`}, @@ -520,7 +552,8 @@ func (m consoleModel) detailCharts(pw int) string { b.WriteString(indent(side(w, box("CPU", fmt.Sprintf("%.0f%%", lastOf(cpu)), cpu, 100), box("Memory", fmt.Sprintf("%.0f%%", lastOf(mem)), mem, 100)), " ")) if g := get("gpu"); len(g) > 0 { - b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", get("vram"), 100)), " ")) + b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU work", "", get("gpuwork"), 100)), " ")) + b.WriteString("\n\n" + indent(side(w, box("GPU memory", "", get("vram"), 100), " "+sDim.Render(gpuWorkNote)), " ")) } if m.det.res.Configured { net := append(named(get("netin"), "in"), named(get("netout"), "out")...) diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index a20bc3e..d741ad4 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -772,12 +772,25 @@ func loadFrom(l *llmgwv1.NodeLoad, at time.Time) *inventory.Load { UtilPercent: g.GetUtilizationPercent(), MemUtilPercent: g.GetMemoryUtilizationPercent(), MemoryUsedMB: g.GetMemoryUsedMb(), MemoryTotalMB: g.GetMemoryTotalMb(), SMClockMHz: g.GetSmClockMhz(), ECCUncorrected: g.GetEccUncorrected(), - Wear: wearFrom(g), + Wear: wearFrom(g), Profile: profileFrom(g), }) } return out } +// profileFrom is a card's profiling counters, or nil from an agent that +// sent none. +func profileFrom(g *llmgwv1.GPUSensor) *inventory.GPUProfile { + if !g.GetHasProfile() { + return nil + } + return &inventory.GPUProfile{ + SMActive: g.GetSmActivePercent(), SMOccupancy: g.GetSmOccupancyPercent(), + TensorActive: g.GetTensorActivePercent(), DRAMActive: g.GetDramActivePercent(), + Source: g.GetProfileSource(), + } +} + func (s *Service) attach(ctx context.Context, nodeID string) *conn { c := &conn{ nodeID: nodeID, diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 16b9b5b..2a8441f 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -84,6 +84,10 @@ type Collector struct { gpuXID *prometheus.Desc gpuLastXID *prometheus.Desc gpuMemUtil *prometheus.Desc + gpuSMActive *prometheus.Desc + gpuSMOccupancy *prometheus.Desc + gpuTensor *prometheus.Desc + gpuDRAMActive *prometheus.Desc gpuSMClock *prometheus.Desc gpuTemp *prometheus.Desc gpuMemTemp *prometheus.Desc @@ -158,9 +162,22 @@ func New(src Source) *Collector { gpuMemoryUsed: prometheus.NewDesc("llmgw_gpu_memory_used_bytes", "A GPU's framebuffer in use, including processes llmgw did not start.", gpu, nil), gpuUtil: prometheus.NewDesc("llmgw_gpu_utilization_percent", - "How busy a GPU was over the last sample period. Absent when the driver does not "+ - "report it, which is not the same as idle. Memory used says a model is loaded; "+ - "this says whether it is doing anything.", gpu, nil), + "The share of the last sample period in which any kernel ran on a GPU (nvidia-smi's "+ + "utilization.gpu). Not how much of the chip: a kernel on one SM reads 100%. Absent "+ + "when the driver does not report it, which is not the same as idle. Memory used says "+ + "a model is loaded; this says whether it is doing anything, and "+ + "llmgw_gpu_sm_active_percent how much.", gpu, nil), + gpuSMActive: prometheus.NewDesc("llmgw_gpu_sm_active_percent", + "The share of time a GPU's SMs had work, averaged over all of them: how much of the chip "+ + "the busy figure is using. From NVML's GPM (Hopper and later) or a dcgm-exporter on the node; "+ + "absent elsewhere.", gpu, nil), + gpuSMOccupancy: prometheus.NewDesc("llmgw_gpu_sm_occupancy_percent", + "How full a GPU's active SMs were with warps, of the most they can hold.", gpu, nil), + gpuTensor: prometheus.NewDesc("llmgw_gpu_tensor_active_percent", + "The share of time a GPU's tensor cores were working. Low in decode at batch one is normal; "+ + "prefill and batches are where it should rise.", gpu, nil), + gpuDRAMActive: prometheus.NewDesc("llmgw_gpu_dram_active_percent", + "The share of time a GPU's memory interface was moving data, from its profiling counters.", gpu, nil), gpuPCIeReplay: prometheus.NewDesc("llmgw_gpu_pcie_replays", "PCIe link retries on a GPU's link since the driver loaded; one that keeps climbing is a riser, slot or cable.", gpu, nil), gpuRetired: prometheus.NewDesc("llmgw_gpu_retired_pages", @@ -310,7 +327,8 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuNVLJoined, c.imexUp, - c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, c.gpuAssigned, c.gpuTemp, c.gpuHealthy, + c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuMemUtil, c.gpuSMClock, + c.gpuSMActive, c.gpuSMOccupancy, c.gpuTensor, c.gpuDRAMActive, c.gpuAssigned, c.gpuTemp, c.gpuHealthy, c.gpuPCIeReplay, c.gpuRetired, c.gpuRetirePend, c.gpuRemapped, c.gpuRemapPend, c.gpuRemapFail, c.gpuNVLinkErr, c.gpuXID, c.gpuLastXID, c.gpuMemTemp, c.gpuPower, c.gpuPowerLimit, c.gpuThrottled, c.nodeCPUTemp, c.nodeCPUPower, c.nodeInletTemp, @@ -653,6 +671,19 @@ func (c *Collector) gpuBusy(ch chan<- prometheus.Metric, s inventory.GPUSensor, if s.SMClockMHz > 0 { ch <- prometheus.MustNewConstMetric(c.gpuSMClock, prometheus.GaugeValue, s.SMClockMHz, labels...) } + // The card's profile, here rather than with the sensors because it is + // the whole card's: a MIG instance must not be published as having it. + if p := s.Profile; p != nil { + for _, v := range []struct { + d *prometheus.Desc + val float64 + }{{c.gpuSMActive, p.SMActive}, {c.gpuSMOccupancy, p.SMOccupancy}, {c.gpuTensor, p.TensorActive}, {c.gpuDRAMActive, p.DRAMActive}} { + // Zero is an idle chip; only -1, not reported, is left out. + if v.val >= 0 { + ch <- prometheus.MustNewConstMetric(v.d, prometheus.GaugeValue, v.val, labels...) + } + } + } } func (c *Collector) gpuSensor(ch chan<- prometheus.Metric, s inventory.GPUSensor, labels []string) { diff --git a/internal/control/fleetmetrics/mig_test.go b/internal/control/fleetmetrics/mig_test.go index bfcabb3..e98802c 100644 --- a/internal/control/fleetmetrics/mig_test.go +++ b/internal/control/fleetmetrics/mig_test.go @@ -16,7 +16,8 @@ func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) { MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}} n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst}, Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250, - UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920}}}} + UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920, + Profile: &inventory.GPUProfile{SMActive: 50, SMOccupancy: 30, TensorActive: 10, DRAMActive: 20}}}}} out := gather(t, fakeSource{nodes: []inventory.Node{n}}) labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1" for _, want := range []string{ @@ -33,4 +34,8 @@ func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) { if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) { t.Errorf("a utilisation was published for a MIG instance:\n%s", out) } + // The card's profile is the whole card's, not this instance's. + if strings.Contains(out, "llmgw_gpu_sm_active_percent "+labels) { + t.Errorf("the card's SM activity was published for a MIG instance:\n%s", out) + } } diff --git a/internal/control/fleetmetrics/sensors_test.go b/internal/control/fleetmetrics/sensors_test.go index 8204fcb..7878f9e 100644 --- a/internal/control/fleetmetrics/sensors_test.go +++ b/internal/control/fleetmetrics/sensors_test.go @@ -127,3 +127,27 @@ func TestAGPUsWearIsPublishedAndWhatItDoesNotKeepIsLeftOut(t *testing.T) { t.Errorf("a count the card does not keep was published:\n%s", out) } } + +func TestHowMuchOfTheChipIsPublishedBesideHowBusy(t *testing.T) { + // The Quadro RTX 6000 in decode: 92% busy, its SMs at work 69% of the + // time. The DRAM figure was not read, and is left out rather than zero. + n := inventory.Node{ID: "g1", State: inventory.Ready, + GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559}}, + Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, UtilPercent: 92, MemoryTotalMB: 81559, + Profile: &inventory.GPUProfile{SMActive: 69, SMOccupancy: 45, TensorActive: 0, DRAMActive: -1, Source: "gpm"}}}}} + out := gather(t, fakeSource{nodes: []inventory.Node{n}}) + for _, want := range []string{ + "llmgw_gpu_utilization_percent class=h100-80g gpu=0 node=g1 -> 92\n", + "llmgw_gpu_sm_active_percent class=h100-80g gpu=0 node=g1 -> 69\n", + "llmgw_gpu_sm_occupancy_percent class=h100-80g gpu=0 node=g1 -> 45\n", + // Idle tensor cores are a reading. + "llmgw_gpu_tensor_active_percent class=h100-80g gpu=0 node=g1 -> 0\n", + } { + if !strings.Contains(out, want) { + t.Errorf("missing %s in:\n%s", want, out) + } + } + if strings.Contains(out, "llmgw_gpu_dram_active_percent") { + t.Errorf("a figure not read was published:\n%s", out) + } +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index ebb41f9..3fa3fc7 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -340,6 +340,36 @@ type GPUSensor struct { // Wear is what wears the card out, read every minute; nil from an agent // that does not read it. Wear *GPUWear `json:"wear,omitempty"` + // Profile is how much of the chip the work uses, which UtilPercent does + // not say; nil where nothing on the node can read it. + Profile *GPUProfile `json:"profile,omitempty"` +} + +// GPUProfile is a card's profiling counters, in percent; -1 is not reported. +// UtilPercent is the share of time any kernel ran, and a kernel on one SM of +// a hundred counts in full; these say how much of the chip it ran on. +type GPUProfile struct { + // SMActive is the share of time the SMs had work, averaged over all of + // them; SMOccupancy how full those SMs were with warps. + SMActive float64 `json:"sm_active"` + SMOccupancy float64 `json:"sm_occupancy"` + // TensorActive is the tensor cores, DRAMActive the memory interface. + TensorActive float64 `json:"tensor_active"` + DRAMActive float64 `json:"dram_active"` + // Source is what read them: "gpm" (NVML, Hopper and later) or "dcgm". + Source string `json:"source,omitempty"` +} + +// PowerLoad is the card's draw as a share of its power limit, from 0 to 1, +// or -1 without both figures. Unlike UtilPercent it rises with how much of +// the chip is working, not only with whether anything is: a card reading +// 92% busy drew 239 W of 260 W in decode and 112 W when the busy figure was +// a kernel or two between requests. +func (g GPUSensor) PowerLoad() float64 { + if g.PowerW <= 0 || g.PowerLimitW <= 0 { + return -1 + } + return min(g.PowerW/g.PowerLimitW, 1) } // GPUWear is a card's wear and failures. Counts of -1 are not reported for diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index f6fa203..93e20db 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -296,6 +296,7 @@ var Reference = []Block{ {Key: "ib_root", Type: "string", Doc: "IBRoot is the sysfs InfiniBand directory.", Nested: ""}, {Key: "prefer_nvml", Type: "bool", Doc: "PreferNVML uses the NVML backend when this binary has one. NVLink\ntopology is read from nvidia-smi topo -m on every build; NVML, where it\nis used, is the more exact source of the peers.", Nested: ""}, {Key: "skip_gpus", Type: "bool", Doc: "SkipGPUs omits GPU discovery. A node without nvidia-smi is already\nfound to have none, so this is for one whose driver is broken.", Nested: ""}, + {Key: "dcgm_exporter", Type: "string", Doc: "DCGMExporter is the dcgm-exporter metrics URL read for how much of\neach card's chip is in use, on cards NVML's GPM does not cover (before\nHopper). Empty is http://127.0.0.1:9400/metrics; \"off\" reads none. A\nnode without one costs a refused connection every five minutes.", Nested: ""}, }}, {Package: "catalog", Name: "Origin", Title: "Origin", Doc: "", Fields: []Field{ {Key: "repo", Type: "string", Doc: "Repo is the repository, owner/name.", Nested: ""}, diff --git a/internal/web/dashcharts.go b/internal/web/dashcharts.go index acb1aa4..4b57e56 100644 --- a/internal/web/dashcharts.go +++ b/internal/web/dashcharts.go @@ -27,7 +27,11 @@ type FleetCharts struct { // GPUBusy and GPUMemory are the fleet's cards by class, for a fleet that // has them. GPUBusy, GPUMemory TimeSeries - Window string + // GPUPowerLoad is the cards' draw against their power limit, and + // GPUSMActive their SMs' share of time with work, where something reads + // it: how much of each chip the busy figure is using. + GPUPowerLoad, GPUSMActive TimeSeries + Window string // Err is set when the metrics store could not be read. Shown as a line of // text where the charts would be: an empty panel reads as a healthy cluster // with nothing happening, which is the opposite of not knowing. The @@ -144,4 +148,12 @@ func (f *FleetCharts) LoadGPU(ctx context.Context, h History, window time.Durati "class", window, step); err == nil { f.GPUMemory = l.LineChart(s, "%", 6) } + if s, err := h.RangeQuery(ctx, + `sum by (class) (llmgw_gpu_power_watts) / sum by (class) (llmgw_gpu_power_limit_watts)`, + "class", window, step); err == nil { + f.GPUPowerLoad = l.LineChart(s, "%", 6) + } + if s, err := h.RangeQuery(ctx, `avg by (class) (llmgw_gpu_sm_active_percent) / 100`, "class", window, step); err == nil { + f.GPUSMActive = l.LineChart(s, "%", 6) + } } diff --git a/internal/web/efficiency.go b/internal/web/efficiency.go index 4a86638..cead838 100644 --- a/internal/web/efficiency.go +++ b/internal/web/efficiency.go @@ -13,8 +13,13 @@ type modelEfficiency struct { // TokensPerSec is output tokens a second; GPUs the cards it holds now. TokensPerSec float64 GPUs float64 - // Watts is what those cards draw; Util their average utilisation. - Watts, Util float64 + // Watts is what those cards draw; Util the average share of time a + // kernel ran on them, which says nothing of how much of each chip it ran + // on. PowerLoad (draw against the power limit) and SMActive (the SMs' + // share of time with work, where GPM or DCGM reads it) are how much, in + // percent; -1 is not known. + Watts, Util float64 + PowerLoad, SMActive float64 // PerGPU and PerWh are the two ways of saying how much work the cards // do: output tokens a second per card, and per watt-hour drawn. PerGPU, PerWh float64 @@ -34,6 +39,10 @@ var efficiencyQueries = []struct { func(e *modelEfficiency, v float64) { e.Watts = v }}, {`avg by (model) (llmgw_gpu_utilization_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`, func(e *modelEfficiency, v float64) { e.Util = v }}, + {`avg by (model) ((llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts) * on (node, gpu) group_left (model) llmgw_gpu_assigned)`, + func(e *modelEfficiency, v float64) { e.PowerLoad = 100 * v }}, + {`avg by (model) (llmgw_gpu_sm_active_percent * on (node, gpu) group_left (model) llmgw_gpu_assigned)`, + func(e *modelEfficiency, v float64) { e.SMActive = v }}, } // loadEfficiency is the table, best per card first; nil when the metrics @@ -56,7 +65,7 @@ func loadEfficiency(ctx context.Context, h History) []modelEfficiency { } e := byModel[s.Name] if e == nil { - e = &modelEfficiency{Model: s.Name} + e = &modelEfficiency{Model: s.Name, PowerLoad: -1, SMActive: -1} byModel[s.Name] = e } q.into(e, s.Last()) diff --git a/internal/web/gpumap.go b/internal/web/gpumap.go index 5cbe476..f065132 100644 --- a/internal/web/gpumap.go +++ b/internal/web/gpumap.go @@ -8,16 +8,25 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) // The GPU map is every card in the fleet, a square each, a row per node, -// coloured by one figure: how busy, how hot, or how full its memory is. A +// coloured by one figure: how much of the chip is at work, how busy, how +// hot, or how full its memory is. A // hundred cards in a table are a hundred rows to read; on the map the one // that is not like the others is seen before anything is read. // gpuMapModes are what the map can be coloured by; each is named by the key // dash.gpumap.mode.. -var gpuMapModes = []string{"util", "memory", "temp"} +// +// work is the default. util, the driver's busy figure, is the share of time +// any kernel ran, and a kernel on one SM of a hundred reads 100%: a map of +// it is green wherever a model is loaded and asked anything. work is the +// SMs' share of time with work where the card's counters can be read, and +// its power draw against its limit elsewhere, which rises with how much of +// the chip runs rather than with whether anything does. +var gpuMapModes = []string{"work", "util", "memory", "temp"} func init() { for _, m := range gpuMapModes { @@ -72,17 +81,16 @@ func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView for _, k := range gpuMapModes { m.Modes = append(m.Modes, struct{ Key, Label string }{k, l.T("dash.gpumap.mode." + k)}) } - if m.By == "" { - m.By = "util" - } switch m.By { + case "util": + m.Legend = l.T("dash.gpumap.legend.util") case "temp": m.Legend = l.T("dash.gpumap.legend.temp") case "memory": m.Legend = l.T("dash.gpumap.legend.memory") default: - m.By = "util" - m.Legend = l.T("dash.gpumap.legend.util") + m.By = "work" + m.Legend = l.T("dash.gpumap.legend.work") } for _, n := range nodes { if n.GPUTotal == 0 { @@ -106,13 +114,21 @@ func buildGPUMap(l *Lang, nodes []adminapi.NodeView, reps []adminapi.ReplicaView if s.MemoryTotalMB > 0 && s.UtilPercent >= 0 { title = append(title, l.T("dash.gpumap.busy", l.Num(s.UtilPercent, 0))) } + if p := s.Profile; p != nil && p.SMActive >= 0 { + title = append(title, l.T("dash.gpumap.sm_active", l.Num(p.SMActive, 0))) + if p.TensorActive >= 0 { + title = append(title, l.T("dash.gpumap.tensor_active", l.Num(p.TensorActive, 0))) + } + } if s.MemoryTotalMB > 0 { title = append(title, l.T("dash.gpumap.memory", l.Num(s.MemoryUsedMB/1024, 1), l.Num(s.MemoryTotalMB/1024, 0))) } if s.TempC > 0 { title = append(title, l.Num(s.TempC, 0)+" °C") } - if s.PowerW > 0 { + if s.PowerW > 0 && s.PowerLimitW > 0 { + title = append(title, l.T("dash.gpumap.power_of", l.Num(s.PowerW, 0), l.Num(s.PowerLimitW, 0))) + } else if s.PowerW > 0 { title = append(title, l.Num(s.PowerW, 0)+" W") } if len(s.Throttle) > 0 { @@ -162,6 +178,20 @@ func cellColour(by string, s sensorView, sampled bool) (string, string) { } f := clamp01(s.MemoryUsedMB / s.MemoryTotalMB) return fmt.Sprintf("background:hsl(265,45%%,%.0f%%)", 90-55*f), fmt.Sprintf("%.0f", 100*f) + case "work": + if !sampled { + return unknown, "" + } + var f float64 + switch { + case s.Profile != nil && s.Profile.SMActive >= 0: + f = clamp01(s.Profile.SMActive / 100) + case s.PowerW > 0 && s.PowerLimitW > 0: + f = clamp01(s.PowerW / s.PowerLimitW) + default: + return unknown, "" + } + return fmt.Sprintf("background:hsl(200,%.0f%%,%.0f%%)", 15+55*f, 88-50*f), fmt.Sprintf("%.0f", 100*f) } if !sampled || s.MemoryTotalMB <= 0 || s.UtilPercent < 0 { return unknown, "" @@ -286,9 +316,10 @@ func gpuKey(node string, gpu int) string { return fmt.Sprintf("%s/%d", node, gpu // sensorView is a card's last sample. type sensorView = struct { - TempC, PowerW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64 - Throttle []string - Problems []string + TempC, PowerW, PowerLimitW, UtilPercent, MemoryUsedMB, MemoryTotalMB float64 + Throttle []string + Problems []string + Profile *inventory.GPUProfile } func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) { @@ -297,9 +328,9 @@ func sensorOf(n adminapi.NodeView, gpu int) (sensorView, bool) { } for _, s := range n.Load.GPUs { if s.Index == gpu { - return sensorView{TempC: s.TempC, PowerW: s.PowerW, UtilPercent: s.UtilPercent, + return sensorView{TempC: s.TempC, PowerW: s.PowerW, PowerLimitW: s.PowerLimitW, UtilPercent: s.UtilPercent, MemoryUsedMB: s.MemoryUsedMB, MemoryTotalMB: s.MemoryTotalMB, Throttle: s.Throttle, - Problems: s.Wear.Problems()}, true + Problems: s.Wear.Problems(), Profile: s.Profile}, true } } return sensorView{}, false diff --git a/internal/web/gpumap_test.go b/internal/web/gpumap_test.go index ec0fdd7..945b7c9 100644 --- a/internal/web/gpumap_test.go +++ b/internal/web/gpumap_test.go @@ -17,10 +17,14 @@ type mapFleet struct{ fakeSource } func (mapFleet) Nodes() []adminapi.NodeView { busy := func(i int, util, temp float64) inventory.GPUSensor { - return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300} + return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300, PowerLimitW: 700} + } + profiled := func(s inventory.GPUSensor) inventory.GPUSensor { + s.Profile = &inventory.GPUProfile{SMActive: 0, SMOccupancy: 0, TensorActive: 0, DRAMActive: -1, Source: "dcgm"} + return s } return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"}, - Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}} + Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{profiled(busy(0, 1, 35)), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}} } func worn(s inventory.GPUSensor) inventory.GPUSensor { @@ -60,6 +64,20 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) { if !strings.Contains(temp, "temperature") || !strings.Contains(temp, ">81") { t.Error("the map coloured by temperature does not say so, or show it") } + // The default colours by how much of the chip works: card 2's draw of + // 300 W of 700, where nothing reads its SMs; card 0's SMs where + // something does. Busy stays a mode of its own. + if !strings.Contains(body, "work") || !strings.Contains(body, ">43") || + !strings.Contains(body, "300 of 700 W") { + t.Error("the map does not colour by work, or say the draw against the limit") + } + if !strings.Contains(body, "gpu07 GPU 0 · chat (chat-1) · 1% busy · SMs 0% active · tensor cores 0%") { + t.Error("a profiled card does not say how much of its chip is at work") + } + _, util := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=util", nil) + if !strings.Contains(util, "busy") || !strings.Contains(util, ">93") { + t.Error("the map coloured by busy does not say so, or show it") + } _, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) if strings.Contains(cpu, "gpu-map") { t.Error("a fleet without GPUs has a GPU map") @@ -68,7 +86,7 @@ func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) { func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) { h := &fakeHistory{byExpr: map[string][]promq.Series{}} - for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}} { + for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}, {0.6, 0.2}, {55, 0}} { h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])} } got := loadEfficiency(context.Background(), h) @@ -76,7 +94,7 @@ func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) { t.Fatalf("%+v", got) } c := got[0] - if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 { + if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 || c.PowerLoad != 60 || c.SMActive != 55 { t.Errorf("chat: %+v", c) } if got[1].PerGPU != 15 { diff --git a/internal/web/i18n_cluster_test.go b/internal/web/i18n_cluster_test.go index 5411cb9..dbf1a10 100644 --- a/internal/web/i18n_cluster_test.go +++ b/internal/web/i18n_cluster_test.go @@ -37,7 +37,7 @@ func TestTheDashboardSpeaksTurkishWhenAsked(t *testing.T) { // A state is still its class, and a Turkish word. `hazır`, // The GPU map: its modes, and what a card says under the pointer. - "meşguliyet", "gpu07 GPU 2 · code (code-1) · %93 meşgul · 39,1 / 80 GB", + "iş", "gpu07 GPU 2 · code (code-1) · %93 meşgul · 39,1 / 80 GB", // How long a replica has sat idle, in Turkish units. "30dk0sn önce", // The traffic chart's peak, with Turkish thousands. diff --git a/internal/web/locales/en/dashboard.json b/internal/web/locales/en/dashboard.json index 9124e27..29a800b 100644 --- a/internal/web/locales/en/dashboard.json +++ b/internal/web/locales/en/dashboard.json @@ -26,11 +26,16 @@ "dash.gpumap.mode.util": "busy", "dash.gpumap.mode.memory": "memory", "dash.gpumap.mode.temp": "temperature", + "dash.gpumap.mode.work": "work", "dash.gpumap.legend.temp": "Blue is cool, red is 90 °C and over.", "dash.gpumap.legend.memory": "Pale is empty, dark is full. Engines reserve most of a card's memory when they start.", - "dash.gpumap.legend.util": "Pale is idle, dark green is fully busy. A dashed border is a card no replica holds.", + "dash.gpumap.legend.util": "Pale is idle, dark green is busy: the share of time any kernel ran, however little of the chip it used. A dashed border is a card no replica holds.", + "dash.gpumap.legend.work": "Pale is idle, dark blue is the whole chip at work: the SMs' share of time with work where the card's counters can be read (GPM, DCGM), its power draw against its limit elsewhere. A dashed border is a card no replica holds.", "dash.gpumap.free": "free", "dash.gpumap.busy": "%s%% busy", + "dash.gpumap.sm_active": "SMs %s%% active", + "dash.gpumap.tensor_active": "tensor cores %s%%", + "dash.gpumap.power_of": "%s of %s W", "dash.gpumap.memory": "%s of %s GB", "dash.gpumap.held_back": "held back by %s", "dash.gpumap.note": "A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.", @@ -41,8 +46,11 @@ "dash.eff.tps": "Output tokens/s", "dash.eff.per_gpu": "Per GPU", "dash.eff.busy": "Busy", + "dash.eff.power_load": "Power load", + "dash.eff.sm_active": "SM active", "dash.eff.draw": "Draw", "dash.eff.per_wh": "Tokens per Wh", + "dash.eff.note": "Busy is the share of time any kernel ran on the cards, not how much of each chip it ran on: a kernel on one SM reads 100%. Power load (draw against the power limit) and SM active (the SMs' share of time with work, where the cards' counters can be read) say how much.", "dash.idle.title": "Cards held and doing nothing", "dash.col.replica": "Replica", @@ -150,7 +158,11 @@ "dash.fleet.ready": "Replicas ready, by model", "dash.fleet.ready_note": "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here.", "dash.fleet.gpu_busy": "GPUs busy, by class", - "dash.fleet.gpu_busy_note": "The average card of each class. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.", + "dash.fleet.gpu_busy_note": "The average card of each class: the share of time any kernel ran, not how much of the chip it used. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.", + "dash.fleet.gpu_power_load": "GPU power load, by class", + "dash.fleet.gpu_power_load_note": "The cards' draw against their power limit. It rises with how much of each chip is working, where busy only says that something is; a class busy above and low here runs small kernels or waits between them.", + "dash.fleet.gpu_sm_active": "SMs active, by class", + "dash.fleet.gpu_sm_active_note": "The share of time the cards' SMs had work, from their profiling counters (NVML GPM on Hopper and later, or a dcgm-exporter on the node). The gap to busy is the part of the chip a running kernel leaves idle.", "dash.fleet.gpu_memory": "GPU memory in use, by class", "dash.fleet.gpu_memory_note": "Engines reserve most of a card's memory when they start, so this follows replicas placed more than requests served.", "dash.fleet.queue": "Engine queue depth, by replica", diff --git a/internal/web/locales/en/node.json b/internal/web/locales/en/node.json index be76b4f..0454480 100644 --- a/internal/web/locales/en/node.json +++ b/internal/web/locales/en/node.json @@ -77,7 +77,11 @@ "node.chart.engine_memory": "Engine memory, by model", "node.chart.engine_memory_note": "Resident memory, including the weights mapped from disk. The kernel can take those pages back, so this can exceed RAM in use.", "node.chart.gpu_busy": "GPU busy, by card", - "node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running. A card held by a replica and idle here is one nobody is asking.", + "node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running — not how much of the chip it used: one SM busy reads 100%. A card held by a replica and idle here is one nobody is asking.", + "node.chart.gpu_power_load": "GPU power load, by card", + "node.chart.gpu_power_load_note": "Draw against the card's power limit. Busy says a kernel ran; this rises with how much of the chip it ran on.", + "node.chart.gpu_sm_active": "SMs active, by card", + "node.chart.gpu_sm_active_note": "The share of time the card's SMs had work, from its profiling counters. When busy reads 90 and this 60, a third of the chip sits idle while a kernel runs.", "node.chart.gpu_memory": "GPU memory used, by card", "node.chart.gpu_memory_note": "Framebuffer in use: the weights and the KV cache an engine reserved, and anything not llmgw's.", "node.chart.gpu_temp": "GPU temperature, °C", diff --git a/internal/web/locales/tr/dashboard.json b/internal/web/locales/tr/dashboard.json index 346dc89..4e4c6b3 100644 --- a/internal/web/locales/tr/dashboard.json +++ b/internal/web/locales/tr/dashboard.json @@ -26,11 +26,16 @@ "dash.gpumap.mode.util": "meşguliyet", "dash.gpumap.mode.memory": "bellek", "dash.gpumap.mode.temp": "sıcaklık", + "dash.gpumap.mode.work": "iş", "dash.gpumap.legend.temp": "Mavi serin, kırmızı 90 °C ve üstü.", "dash.gpumap.legend.memory": "Açık renk boş, koyu renk dolu. Motorlar başlarken bir kartın belleğinin çoğunu ayırır.", - "dash.gpumap.legend.util": "Açık renk boşta, koyu yeşil tamamen meşgul. Kesikli kenarlık, hiçbir replikanın tutmadığı bir karttır.", + "dash.gpumap.legend.util": "Açık renk boşta, koyu yeşil meşgul demektir: yonganın ne kadarını kullanırsa kullansın, herhangi bir çekirdek işlevinin çalıştığı zaman payı. Kesik kenarlık hiçbir replikanın tutmadığı karttır.", + "dash.gpumap.legend.work": "Açık renk boşta, koyu mavi tüm yongası çalışan kart demektir: kartın sayaçları okunabiliyorsa (GPM, DCGM) SM'lerin iş yaptığı zaman payı, okunamıyorsa çektiği gücün güç sınırına oranı. Kesik kenarlık hiçbir replikanın tutmadığı karttır.", "dash.gpumap.free": "boş", "dash.gpumap.busy": "%%%s meşgul", + "dash.gpumap.sm_active": "SM'ler %%%s etkin", + "dash.gpumap.tensor_active": "tensör çekirdekleri %%%s", + "dash.gpumap.power_of": "%s / %s W", "dash.gpumap.memory": "%s / %s GB", "dash.gpumap.held_back": "kısıtlayan: %s", "dash.gpumap.note": "Kırmızı çerçeve, yıpranmakta olan bir karttır: XID hataları, yeniden eşlenmiş ya da sıfırlama bekleyen bellek satırları, NVLink hataları. Karttaki rakam, renklendirmede kullanılan değerdir; kartın geri kalanı imlecin altındadır.", @@ -41,8 +46,11 @@ "dash.eff.tps": "Çıkış token/sn", "dash.eff.per_gpu": "GPU başına", "dash.eff.busy": "Meşgul", + "dash.eff.power_load": "Güç yükü", + "dash.eff.sm_active": "Etkin SM", "dash.eff.draw": "Çekilen güç", "dash.eff.per_wh": "Wh başına token", + "dash.eff.note": "Meşgul, kartlarda herhangi bir çekirdek işlevinin çalıştığı zaman payıdır; her yonganın ne kadarında çalıştığı değil: tek bir SM'de çalışan işlev %100 okunur. Ne kadarını kullandığını güç yükü (çekilen gücün güç sınırına oranı) ve etkin SM (kartların sayaçları okunabildiğinde SM'lerin iş yaptığı zaman payı) söyler.", "dash.idle.title": "Tutulan ama hiçbir şey yapmayan kartlar", "dash.col.replica": "Replika", @@ -148,7 +156,11 @@ "dash.fleet.ready": "Hazır replikalar, modele göre", "dash.fleet.ready_note": "Bilerek durdurulmuş bir modelin hiç çizgisi olmaz; isteğe bağlı çalışan bir katalogun burada sessiz görünmesinin nedeni budur.", "dash.fleet.gpu_busy": "Meşgul GPU'lar, sınıfa göre", - "dash.fleet.gpu_busy_note": "Her sınıfın ortalama kartı. Replikalar yerleştirilmişken burada düşük olması, tutulan ama istenmeyen kapasite demektir; her düğümün sayfası kartlarını tek tek gösterir.", + "dash.fleet.gpu_busy_note": "Her sınıfın ortalama kartı: herhangi bir çekirdek işlevinin çalıştığı zaman payı, yonganın ne kadarını kullandığı değil. Replikalar yerleştirilmişken burada düşük olması, tutulan ama istenmeyen kapasite demektir; her düğümün sayfası kartlarını tek tek gösterir.", + "dash.fleet.gpu_power_load": "GPU güç yükü, sınıfa göre", + "dash.fleet.gpu_power_load_note": "Kartların çektiği gücün güç sınırına oranı. Meşgul yalnızca bir şeyin çalıştığını söylerken bu, her yonganın ne kadarının çalıştığıyla artar; yukarıda meşgul ve burada düşük görünen bir sınıf küçük çekirdek işlevleri çalıştırıyor ya da aralarında bekliyordur.", + "dash.fleet.gpu_sm_active": "Etkin SM'ler, sınıfa göre", + "dash.fleet.gpu_sm_active_note": "Kartların SM'lerinin iş yaptığı zaman payı, profil sayaçlarından (Hopper ve sonrasında NVML GPM ya da düğümdeki bir dcgm-exporter). Meşgul ile arasındaki fark, çalışan bir işlevin yongada boş bıraktığı kısımdır.", "dash.fleet.gpu_memory": "Kullanılan GPU belleği, sınıfa göre", "dash.fleet.gpu_memory_note": "Motorlar başlarken bir kartın belleğinin çoğunu ayırır; bu yüzden bu grafik sunulan isteklerden çok yerleştirilen replikaları izler.", "dash.fleet.queue": "Motor kuyruk derinliği, replikaya göre", diff --git a/internal/web/locales/tr/node.json b/internal/web/locales/tr/node.json index 404bce3..14dd943 100644 --- a/internal/web/locales/tr/node.json +++ b/internal/web/locales/tr/node.json @@ -76,7 +76,11 @@ "node.chart.engine_memory": "Motor belleği, modele göre", "node.chart.engine_memory_note": "Diskten eşlenen ağırlıklar dâhil yerleşik bellek. Çekirdek o sayfaları geri alabilir, bu yüzden bu değer kullanılan RAM'i aşabilir.", "node.chart.gpu_busy": "Meşgul GPU, karta göre", - "node.chart.gpu_busy_note": "Her örnekleme süresinde bir çekirdek işlevinin çalıştığı pay. Bir replikanın tuttuğu ve burada boşta görünen kart, kimsenin istemediği karttır.", + "node.chart.gpu_busy_note": "Her örnekleme süresinde bir çekirdek işlevinin çalıştığı pay — yonganın ne kadarını kullandığı değil: tek bir SM'nin meşgul olması %100 okunur. Bir replikanın tuttuğu ve burada boşta görünen kart, kimsenin istemediği karttır.", + "node.chart.gpu_power_load": "GPU güç yükü, karta göre", + "node.chart.gpu_power_load_note": "Çekilen gücün kartın güç sınırına oranı. Meşgul bir işlevin çalıştığını söyler; bu, yonganın ne kadarında çalıştığıyla artar.", + "node.chart.gpu_sm_active": "Etkin SM'ler, karta göre", + "node.chart.gpu_sm_active_note": "Kartın SM'lerinin iş yaptığı zaman payı, profil sayaçlarından. Meşgul %90 iken bunun %60 olması, bir işlev çalışırken yonganın üçte birinin boşta olduğu demektir.", "node.chart.gpu_memory": "Kullanılan GPU belleği, karta göre", "node.chart.gpu_memory_note": "Kullanımdaki çerçeve arabelleği: bir motorun ayırdığı ağırlıklar ve KV önbelleği, ayrıca llmgw'ye ait olmayan her şey.", "node.chart.gpu_temp": "GPU sıcaklığı, °C", diff --git a/internal/web/node_page.go b/internal/web/node_page.go index 81bd3cb..d5eb162 100644 --- a/internal/web/node_page.go +++ b/internal/web/node_page.go @@ -51,8 +51,11 @@ type nodeCharts struct { CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries // The GPUs', one line per card; empty on a node without them. GPUUtil, GPUMemory, GPUTemp, GPUPower TimeSeries - Window string - Err string + // GPUPowerLoad is draw against the power limit; GPUSMActive the SMs' + // share of time with work, empty on a card nothing can profile. + GPUPowerLoad, GPUSMActive TimeSeries + Window string + Err string } // nodePanels are the charts, each a query scoped to this node. %s is the @@ -89,6 +92,10 @@ var gpuPanels = []struct { func(c *nodeCharts, t TimeSeries) { c.GPUTemp = t }}, {`llmgw_gpu_power_watts{node=%s}`, "", func(c *nodeCharts, t TimeSeries) { c.GPUPower = t }}, + {`llmgw_gpu_power_watts{node=%[1]s} / llmgw_gpu_power_limit_watts{node=%[1]s}`, "%", + func(c *nodeCharts, t TimeSeries) { c.GPUPowerLoad = t }}, + {`llmgw_gpu_sm_active_percent{node=%s} / 100`, "%", + func(c *nodeCharts, t TimeSeries) { c.GPUSMActive = t }}, } // NodePage shows one node. Its charts come from the metrics store when one diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 4d19e4f..1454c93 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -64,11 +64,13 @@ {{/* Per card, because a model on eight cards making twice the tokens of one on two is the worse of the two; per watt-hour, because that is what the room is paying. */}} - +
{{t "dash.col.model"}}GPUs{{t "dash.eff.tps"}}{{t "dash.eff.per_gpu"}}{{t "dash.eff.busy"}}{{t "dash.eff.draw"}}{{t "dash.eff.per_wh"}}
{{range .}} + {{end}}
{{t "dash.col.model"}}GPUs{{t "dash.eff.tps"}}{{t "dash.eff.per_gpu"}}{{t "dash.eff.busy"}}{{t "dash.eff.power_load"}}{{t "dash.eff.sm_active"}}{{t "dash.eff.draw"}}{{t "dash.eff.per_wh"}}
{{.Model}}{{num .GPUs 0}}{{num .TokensPerSec 1}} {{num .PerGPU 1}}{{pc .Util 0}}{{if ge .PowerLoad 0.0}}{{pc .PowerLoad 0}}{{else}}—{{end}}{{if ge .SMActive 0.0}}{{pc .SMActive 0}}{{else}}—{{end}} {{if .Watts}}{{num .Watts 0}} W{{else}}—{{end}}{{if .PerWh}}{{num .PerWh 0}}{{else}}—{{end}}
+

{{t "dash.eff.note"}}

{{end}} {{with .IdleGPUs}}

{{t "dash.idle.title"}}

@@ -282,6 +284,10 @@ document.querySelectorAll("form.build-choice select").forEach(function (s) { {{template "tschart" dict "t" .GPUBusy "title" (t "dash.fleet.gpu_busy") "note" (t "dash.fleet.gpu_busy_note")}} {{template "tschart" dict "t" .GPUMemory "title" (t "dash.fleet.gpu_memory") "note" (t "dash.fleet.gpu_memory_note")}} +
+ {{template "tschart" dict "t" .GPUPowerLoad "title" (t "dash.fleet.gpu_power_load") "note" (t "dash.fleet.gpu_power_load_note")}} + {{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "dash.fleet.gpu_sm_active") "note" (t "dash.fleet.gpu_sm_active_note")}}{{end}} +
{{end}}
{{template "tschart" dict "t" .Queue "title" (t "dash.fleet.queue") "note" (t "dash.fleet.queue_note")}} diff --git a/internal/web/templates/node.html b/internal/web/templates/node.html index a241f1b..7250ea4 100644 --- a/internal/web/templates/node.html +++ b/internal/web/templates/node.html @@ -141,6 +141,10 @@ {{template "tschart" dict "t" .GPUTemp "title" (t "node.chart.gpu_temp") "note" (t "node.chart.gpu_temp_note")}} {{template "tschart" dict "t" .GPUPower "title" (t "node.chart.gpu_power") "note" (t "node.chart.gpu_power_note")}}
+
+ {{template "tschart" dict "t" .GPUPowerLoad "title" (t "node.chart.gpu_power_load") "note" (t "node.chart.gpu_power_load_note")}} + {{if not .GPUSMActive.Empty}}{{template "tschart" dict "t" .GPUSMActive "title" (t "node.chart.gpu_sm_active") "note" (t "node.chart.gpu_sm_active_note")}}{{end}} +
{{end}}
{{template "tschart" dict "t" .Load "title" (t "node.chart.load") "note" (t "node.chart.load_note")}}