Ilker ManapandClaude Opus 5.5
c866b3a8ee
CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
239 lines
12 KiB
JSON
239 lines
12 KiB
JSON
{
|
|
"dash.title": "cluster",
|
|
"dash.h1": "Cluster",
|
|
"dash.sub": "%d nodes · %d of %d replicas ready · %d models",
|
|
"dash.nav.metrics": "metrics",
|
|
|
|
"dash.tile.requests": "requests · 24h",
|
|
"dash.tile.tokens": "tokens · 24h",
|
|
"dash.tile.errors": "errors · 24h",
|
|
"dash.tile.spend": "spend · 24h",
|
|
|
|
"dash.capacity": "Capacity",
|
|
"dash.cap.class": "Class",
|
|
"dash.cap.in_use": "In use",
|
|
"dash.cap.free": "Free",
|
|
"dash.cap.faulty": "Faulty",
|
|
"dash.cap.usage": "Usage",
|
|
"dash.cpu.ready": "nodes ready · CPU only",
|
|
"dash.cpu.memory": "memory",
|
|
"dash.cpu.disk_of": "%s of %s GB",
|
|
"dash.cpu.disk_free": "free for weights",
|
|
"dash.no_nodes": "No nodes have registered.",
|
|
|
|
"dash.gpumap.title": "GPUs",
|
|
"dash.gpumap.by": "by",
|
|
"dash.gpumap.mode.util": "busy",
|
|
"dash.gpumap.mode.memory": "memory",
|
|
"dash.gpumap.mode.temp": "temperature",
|
|
"dash.gpumap.mode.work": "work",
|
|
"dash.gpumap.legend.temp": "Blue is cool, red is 90 °C and over.",
|
|
"dash.gpumap.legend.memory": "Pale is empty, dark is full. Engines reserve most of a card's memory when they start.",
|
|
"dash.gpumap.legend.util": "Pale is idle, dark green is busy: the share of time any kernel ran, however little of the chip it used. A dashed border is a card no replica holds.",
|
|
"dash.gpumap.legend.work": "Pale is idle, dark blue is the whole chip at work: the SMs' share of time with work where the card's counters can be read (GPM, DCGM), its power draw against its limit elsewhere. A dashed border is a card no replica holds.",
|
|
"dash.gpumap.free": "free",
|
|
"dash.gpumap.busy": "%s%% busy",
|
|
"dash.gpumap.sm_active": "SMs %s%% active",
|
|
"dash.gpumap.tensor_active": "tensor cores %s%%",
|
|
"dash.gpumap.power_of": "%s of %s W",
|
|
"dash.gpumap.memory": "%s of %s GB",
|
|
"dash.gpumap.held_back": "held back by %s",
|
|
"dash.gpumap.note": "A red outline is a card wearing out — XID errors, memory rows remapped or waiting for a reset, NVLink errors. The figure is the one coloured by; the rest of a card is under the pointer.",
|
|
|
|
"dash.eff.title": "What each model gets from its cards",
|
|
"dash.eff.window": "last 15 minutes",
|
|
"dash.col.model": "Model",
|
|
"dash.eff.tps": "Output tokens/s",
|
|
"dash.eff.per_gpu": "Per GPU",
|
|
"dash.eff.busy": "Busy",
|
|
"dash.eff.power_load": "Power load",
|
|
"dash.eff.sm_active": "SM active",
|
|
"dash.eff.draw": "Draw",
|
|
"dash.eff.per_wh": "Tokens per Wh",
|
|
"dash.eff.note": "Busy is the share of time any kernel ran on the cards, not how much of each chip it ran on: a kernel on one SM reads 100%. Power load (draw against the power limit) and SM active (the SMs' share of time with work, where the cards' counters can be read) say how much.",
|
|
|
|
"dash.idle.title": "Cards held and doing nothing",
|
|
"dash.col.replica": "Replica",
|
|
"dash.col.node": "Node",
|
|
"dash.idle.busiest": "Busiest card",
|
|
"dash.idle.last_request": "Last request",
|
|
"dash.idle.ago": "%s ago",
|
|
"dash.idle.unknown": "not known",
|
|
"dash.clock.h": "h",
|
|
"dash.clock.m": "m",
|
|
"dash.clock.s": "s",
|
|
"dash.clock.ms": "ms",
|
|
|
|
"dash.nodes": "Nodes",
|
|
"dash.col.state": "State",
|
|
"dash.nodes.ram": "RAM",
|
|
"dash.nodes.disk": "Disk",
|
|
"dash.nodes.engines": "Engines",
|
|
"dash.nodes.build": "vLLM build",
|
|
"dash.nodes.classes": "Classes",
|
|
"dash.nodes.fabrics": "Fabrics",
|
|
"dash.nodes.models": "Models",
|
|
"dash.nodes.replicas": "Replicas",
|
|
"dash.nodes.seen": "Seen",
|
|
"dash.nodes.logs": "logs",
|
|
"dash.nodes.cpus_of": "of %d",
|
|
"dash.nodes.mem_of": "of %s",
|
|
"dash.nodes.no_engines": "none yet",
|
|
"dash.nodes.build_hint": "the vLLM build this node installs",
|
|
"dash.no_agents": "No agents have registered.",
|
|
|
|
"dash.state.ready": "ready",
|
|
"dash.state.cordoned": "cordoned",
|
|
"dash.state.draining": "draining",
|
|
"dash.state.unreachable": "unreachable",
|
|
"dash.state.lost": "lost",
|
|
"dash.state.unknown": "unknown",
|
|
"dash.state.placing": "placing",
|
|
"dash.state.pending": "pending",
|
|
"dash.state.starting": "starting",
|
|
"dash.state.loading": "loading",
|
|
"dash.state.stopped": "stopped",
|
|
"dash.state.failed": "failed",
|
|
"dash.state.installing": "installing",
|
|
"dash.state.unsupported": "unsupported",
|
|
"dash.state.skipped": "skipped",
|
|
|
|
"dash.models": "Models",
|
|
"dash.models.aliases": "Aliases",
|
|
"dash.models.ready": "Ready",
|
|
"dash.models.wanted": "Wanted",
|
|
"dash.models.class": "Class",
|
|
"dash.models.reserves": "Reserves",
|
|
"dash.models.profiles": "Profiles",
|
|
"dash.models.fallbacks": "Fallbacks",
|
|
"dash.models.for": "What it is for",
|
|
"dash.models.held_for": "gave room to %s",
|
|
"dash.models.preempted": "replicas lost to a higher class",
|
|
"dash.models.used": "%sG used",
|
|
"dash.models.external_hint": "served outside the cluster; its prompts leave it",
|
|
"dash.models.external": "external",
|
|
"dash.models.empty": "The catalog is empty.",
|
|
"dash.unplaced": "Why nothing was placed",
|
|
|
|
"dash.replicas": "Replicas",
|
|
"dash.replicas.pool": "Pool",
|
|
"dash.replicas.endpoint": "Endpoint",
|
|
"dash.replicas.age": "Age",
|
|
"dash.replicas.none": "No replicas.",
|
|
"dash.holding": "Holding hardware nothing is asking for",
|
|
"dash.holding.unused_for": "Unused for",
|
|
|
|
"dash.fabrics.fabric": "Fabric",
|
|
"dash.fabrics.physically": "Is physically",
|
|
"dash.fabrics.fully_free": "Fully free",
|
|
"dash.fabrics.weights": "Weights",
|
|
"dash.fabrics.weights_on": "%d model(s) on %d/%d nodes",
|
|
"dash.fabrics.not_reported": "not reported",
|
|
"dash.fabrics.see_map": "See each fabric's map →",
|
|
"dash.fabrics.sms": "Subnet managers",
|
|
"dash.fabrics.master": "Master",
|
|
"dash.fabrics.standby": "Standby",
|
|
"dash.fabrics.last_change": "Last change of master",
|
|
"dash.fabrics.roce": "none: RoCE",
|
|
"dash.fabrics.lid_priority": "LID %d, priority %d",
|
|
"dash.fabrics.none": "none",
|
|
"dash.fabrics.not_listed": "not listed",
|
|
"dash.fabrics.in_a_day": "(%d in a day)",
|
|
"dash.fabrics.none_in_a_day": "none in a day",
|
|
"dash.fabrics.note": "A fabric's name is what the nodes were told; what it is physically is the subnet manager its InfiniBand ports see, or the network its RoCE ports are on. A replica spanning nodes is only ever formed on one of those. Fully free counts whole unused nodes, which is what decides whether such a replica can land.",
|
|
|
|
"dash.nvlink.title": "NVLink domains",
|
|
"dash.nvlink.domains.one": "1 multi-node domain",
|
|
"dash.nvlink.domains.other": "%d multi-node domains",
|
|
"dash.nvlink.trays": "%d/%d trays usable",
|
|
"dash.nvlink.problems.one": "1 problem",
|
|
"dash.nvlink.problems.other": "%d problems",
|
|
"dash.nvlink.see_racks": "see the racks →",
|
|
|
|
"dash.fleet.title": "What the cluster is doing",
|
|
"dash.fleet.last": "last %s",
|
|
"dash.no_history": "No history: %s",
|
|
"dash.fleet.saturation": "Pool saturation, by model",
|
|
"dash.fleet.saturation_note": "The worse of queue depth and cache pressure. 1 is the limit measured as healthy, not the point of failure.",
|
|
"dash.fleet.ready": "Replicas ready, by model",
|
|
"dash.fleet.ready_note": "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here.",
|
|
"dash.fleet.gpu_busy": "GPUs busy, by class",
|
|
"dash.fleet.gpu_busy_note": "The average card of each class: the share of time any kernel ran, not how much of the chip it used. Low here with replicas placed is capacity held and not asked for; each node's page has its cards one by one.",
|
|
"dash.fleet.gpu_power_load": "GPU power load, by class",
|
|
"dash.fleet.gpu_power_load_note": "The cards' draw against their power limit. It rises with how much of each chip is working, where busy only says that something is; a class busy above and low here runs small kernels or waits between them.",
|
|
"dash.fleet.gpu_sm_active": "SMs active, by class",
|
|
"dash.fleet.gpu_sm_active_note": "The share of time the cards' SMs had work, from their profiling counters (NVML GPM on Hopper and later, or a dcgm-exporter on the node). The gap to busy is the part of the chip a running kernel leaves idle.",
|
|
"dash.fleet.gpu_memory": "GPU memory in use, by class",
|
|
"dash.fleet.gpu_memory_note": "Engines reserve most of a card's memory when they start, so this follows replicas placed more than requests served.",
|
|
"dash.fleet.queue": "Engine queue depth, by replica",
|
|
"dash.fleet.queue_note": "Requests accepted and not yet started. The first thing to move when a replica is falling behind.",
|
|
"dash.fleet.kv": "KV cache occupancy, by replica",
|
|
"dash.fleet.kv_note": "How full the attention cache is. Long contexts fill it with an empty queue, which is why saturation reads both.",
|
|
|
|
"dash.traffic": "Traffic",
|
|
"dash.traffic.window": "— last 24 hours, hourly",
|
|
"dash.traffic.requests": "Requests",
|
|
"dash.traffic.tokens": "Tokens",
|
|
"dash.traffic.latency": "Mean latency",
|
|
"dash.traffic.none": "No traffic recorded yet.",
|
|
"dash.unit.requests": "requests",
|
|
"dash.unit.tokens": "tokens",
|
|
"dash.unit.ms_mean": "ms mean",
|
|
"dash.unit.requests_h": "requests/h",
|
|
"dash.unit.tokens_h": "tokens/h",
|
|
|
|
"dash.people": "People",
|
|
"dash.col.person": "Person",
|
|
"dash.people.keys": "Keys",
|
|
"dash.col.requests": "Requests",
|
|
"dash.col.tokens": "Tokens",
|
|
"dash.col.errors": "Errors",
|
|
"dash.col.spend": "Spend",
|
|
"dash.people.last_seen": "Last seen",
|
|
"dash.people.operator_keys": "operator-issued keys",
|
|
|
|
"dash.running": "What is being run",
|
|
"dash.running.note": "Counted by the model that actually answered, which differs from the one requested whenever a fallback served it.",
|
|
|
|
"dash.usage": "Usage",
|
|
"dash.usage.window": "— last 24 hours",
|
|
"dash.usage.through": "Through",
|
|
"dash.usage.against": "Spends against",
|
|
"dash.usage.prompt": "Prompt",
|
|
"dash.usage.completion": "Completion",
|
|
"dash.usage.last": "Last",
|
|
"dash.usage.chat_page": "chat page",
|
|
"dash.usage.personal": "personal",
|
|
"dash.usage.note": "Spend is in units, whatever a unit was configured to mean. Usage is written in batches, so the last few seconds may be missing.",
|
|
"dash.usage.none": "No requests recorded in the last 24 hours.",
|
|
"dash.usage.auth_off": "Authentication is off, so there is no identity to account against.",
|
|
|
|
"dash.admin_listener": "Metrics, pprof and the management API are on the admin listener, which is bound to loopback and not reachable from here. pprof can dump the process heap, where API keys sit in plaintext, which is why that port is not exposed:",
|
|
|
|
"dash.presets.title": "Pictures and 3D models",
|
|
"dash.presets.about": "Models that come with llmgw, their workflow, nodes and licence already tried. Adding one also has the GPU nodes install ComfyUI, about 10 GB each, when the cluster has none.",
|
|
"dash.presets.makes": "Makes",
|
|
"dash.presets.makes.pictures": "pictures",
|
|
"dash.presets.makes.meshes": "3D models",
|
|
"dash.presets.card": "Needs",
|
|
"dash.presets.licence": "Licence",
|
|
"dash.presets.not_commercial": "%s, not for commercial use",
|
|
"dash.presets.fits": "Fits",
|
|
"dash.presets.fits_none": "no card here",
|
|
"dash.presets.state": "Here",
|
|
"dash.presets.added_auto": "added automatically, %s",
|
|
"dash.presets.added_by": "added by %s, %s",
|
|
"dash.presets.in_catalog": "in the catalog",
|
|
"dash.presets.declined": "removed, so not added again by itself",
|
|
"dash.presets.add": "Add",
|
|
"dash.presets.add_again": "Add again",
|
|
"dash.presets.auto_on": "Added by themselves: when a card fits one and the catalog has nothing that makes what it makes.",
|
|
"dash.presets.auto_off": "Not added by themselves: only when somebody adds one.",
|
|
"dash.presets.turn_off": "Stop adding them",
|
|
"dash.presets.turn_on": "Add them by themselves",
|
|
"dash.presets.notice.added": "%s added; it is placed on the next pass, and loads when first asked for",
|
|
"dash.presets.notice.auto_on": "Pictures and 3D models are added by themselves where a card fits",
|
|
"dash.presets.notice.auto_off": "Pictures and 3D models are no longer added by themselves",
|
|
"dash.presets.adds_comfyui": "Adding it installs ComfyUI on the GPU nodes, about 10 GB each, and restarts the control planes, standbys first"
|
|
}
|