CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
111 lines
5.8 KiB
JSON
111 lines
5.8 KiB
JSON
{
|
|
"node.last_seen": "last seen %s ago",
|
|
"node.never_seen": "never seen",
|
|
"node.nav.cluster": "cluster",
|
|
"node.cores": "%d cores",
|
|
"node.ram_in_use_of": "RAM in use · of %s",
|
|
"node.load": "load · %s 5m · %s 15m",
|
|
"node.inlet": "inlet air",
|
|
"node.not_measured": "CPU and RAM not measured lately",
|
|
"node.disk_free": "disk free · of %dG",
|
|
"node.gpus_free": "GPUs free",
|
|
|
|
"node.gpus": "GPUs",
|
|
"node.gpu.busy": "Busy",
|
|
"node.gpu.memory_used": "Memory used",
|
|
"node.gpu.temp": "Temperature",
|
|
"node.gpu.memory_temp": "Memory temp.",
|
|
"node.gpu.power": "Power",
|
|
"node.gpu.clock": "Clock",
|
|
"node.gpu.held_back": "Held back by, and wear",
|
|
"node.gpu.memory_util": "memory %s%%",
|
|
"node.of_gb": "of %s GB",
|
|
"node.of": "of %s",
|
|
"node.gpu.nothing": "nothing",
|
|
"node.gpu.ecc": "%s uncorrected ECC errors",
|
|
"node.gpu.pcie": "%s PCIe retries",
|
|
|
|
"node.fabric": "Fabric",
|
|
"node.fabric.on": "On %s",
|
|
"node.fabric.map": "the map →",
|
|
"node.fabric.none": "On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.",
|
|
"node.ib.adapter": "Adapter",
|
|
"node.ib.rate": "Rate",
|
|
"node.ib.physically": "Physically on",
|
|
"node.ib.errors": "Errors since boot",
|
|
"node.ib.sent": "Sent",
|
|
"node.ib.received": "Received",
|
|
"node.unknown": "unknown",
|
|
"node.none": "none",
|
|
"node.ib.note": "Re-read on every heartbeat. A port that goes down takes its fabric off this node within seconds; one that comes back returns it. Errors are cumulative since the adapter came up: a number that grows is the one to look at.",
|
|
|
|
"node.nvl.title": "GPUs and NVLink",
|
|
"node.nvl.domain": "In NVLink domain <code>%s</code>: tensor parallelism can span this node and the rest of its domain.",
|
|
"node.nvl.ends_here": "NVLink ends at this machine.",
|
|
"node.imex.active": "active",
|
|
"node.imex.channels.one": "1 channel",
|
|
"node.imex.channels.other": "%d channels",
|
|
"node.imex.not_installed": "not installed",
|
|
"node.nvl.matrix": "NVLink between GPUs",
|
|
"node.nvl.over_nvlink": "GPU %d ↔ GPU %d over NVLink",
|
|
"node.nvl.over_pcie": "GPU %d ↔ GPU %d over PCIe",
|
|
"node.gpu.name": "Name",
|
|
"node.gpu.compute": "Compute",
|
|
"node.gpu.numa": "NUMA",
|
|
"node.gpu.nearest": "Nearest adapter",
|
|
"node.gpu.multinode": "Multi-node NVLink",
|
|
"node.gpu.joined": "joined",
|
|
"node.gpu.bandwidth": "%s bandwidth",
|
|
"node.nvl.note": "NV is a pair of cards joined by NVLink; a tensor-parallel group kept to NV pairs runs its all-reduces at NVLink speed, and placement picks it that way. The nearest adapter is where a card's share of traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS is across the processors.",
|
|
|
|
"node.runs": "What runs here",
|
|
"node.memory": "Memory",
|
|
"node.cores_f": "%s cores",
|
|
"node.no_engines": "No engines are placed on this node.",
|
|
"node.version_differs": "the control plane runs %s",
|
|
"node.gpu.models": "Models",
|
|
"node.engines": "Engines: %s",
|
|
"node.engines_none": "Engines: none installed",
|
|
"node.model_files": "%d model file(s) on disk",
|
|
|
|
"node.last": "The last %s",
|
|
"node.chart.cpu_note": "All cores together, from the node's heartbeat.",
|
|
"node.chart.ram": "RAM in use",
|
|
"node.chart.ram_note": "Total less what the kernel could hand to a new process.",
|
|
"node.chart.cores": "Cores busy, by model",
|
|
"node.chart.cores_note": "Each engine's process group, the way top counts it.",
|
|
"node.chart.engine_memory": "Engine memory, by model",
|
|
"node.chart.engine_memory_note": "Resident memory, including the weights mapped from disk. The kernel can take those pages back, so this can exceed RAM in use.",
|
|
"node.chart.gpu_busy": "GPU busy, by card",
|
|
"node.chart.gpu_busy_note": "Of each sample period, the share a kernel was running — not how much of the chip it used: one SM busy reads 100%. A card held by a replica and idle here is one nobody is asking.",
|
|
"node.chart.gpu_power_load": "GPU power load, by card",
|
|
"node.chart.gpu_power_load_note": "Draw against the card's power limit. Busy says a kernel ran; this rises with how much of the chip it ran on.",
|
|
"node.chart.gpu_sm_active": "SMs active, by card",
|
|
"node.chart.gpu_sm_active_note": "The share of time the card's SMs had work, from its profiling counters. When busy reads 90 and this 60, a third of the chip sits idle while a kernel runs.",
|
|
"node.chart.gpu_memory": "GPU memory used, by card",
|
|
"node.chart.gpu_memory_note": "Framebuffer in use: the weights and the KV cache an engine reserved, and anything not llmgw's.",
|
|
"node.chart.gpu_temp": "GPU temperature, °C",
|
|
"node.chart.gpu_temp_note": "Throttling usually starts in the high 80s; see what holds a card back in the table above.",
|
|
"node.chart.gpu_power": "GPU power, W",
|
|
"node.chart.gpu_power_note": "Draw against the card's enforced limit; at the limit is sw_power_cap.",
|
|
"node.chart.load": "Load average",
|
|
"node.chart.load_note": "Above the core count means work is waiting for a CPU.",
|
|
"node.chart.queue_note": "Requests accepted and not yet started, on the replicas here now.",
|
|
"node.no_store": "No metrics store is configured, so there is no history here; <code>monitoring.prometheus</code> in the server configuration adds it.",
|
|
|
|
"logs.title": "%s logs",
|
|
"logs.agent": "the agent's log",
|
|
"logs.engine": "output of engine <code>%s</code>",
|
|
"logs.last_lines": "last %d lines",
|
|
"logs.lines": "lines",
|
|
"logs.stop": "stop following",
|
|
"logs.follow": "follow",
|
|
"logs.follow_every": "(refresh every 5s)",
|
|
"logs.empty": "Nothing logged yet.",
|
|
"logs.cannot_read": "this control plane cannot read node logs",
|
|
|
|
"nodeengine.notice.auto": "%s installs %s in the build its hardware runs; the node is told now, and installs it in the background",
|
|
"nodeengine.notice.none": "%s installs %s no longer; the node is told now, and installs it in the background",
|
|
"nodeengine.notice.build": "%s installs %s built for %s; the node is told now, and installs it in the background"
|
|
}
|