Ilker ManapandClaude Opus 5.5
c866b3a8ee
CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
958 lines
22 KiB
JSON
958 lines
22 KiB
JSON
{
|
|
"uid": "llmgw-overview",
|
|
"title": "llmgw \u2014 cluster",
|
|
"tags": [
|
|
"llmgw"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Nodes ready",
|
|
"type": "stat",
|
|
"description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_node_up == 1)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Replicas serving",
|
|
"type": "stat",
|
|
"description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
|
|
"gridPos": {
|
|
"x": 4,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(llmgw_model_replicas_ready)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Models with something serving",
|
|
"type": "stat",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_model_replicas_ready > 0)",
|
|
"legendFormat": "models",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Requests per second",
|
|
"type": "timeseries",
|
|
"description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 0,
|
|
"w": 12,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_requests_total[5m]))",
|
|
"legendFormat": "all",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
|
|
"legendFormat": "failing",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Replicas ready, by model",
|
|
"type": "timeseries",
|
|
"description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_model_replicas_ready",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Pool saturation",
|
|
"type": "timeseries",
|
|
"description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_pool_saturation",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"max": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "Time to first token",
|
|
"type": "timeseries",
|
|
"description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p50",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p99",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "Tokens per second",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
|
|
"legendFormat": "{{direction}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "Engine queue depth",
|
|
"type": "timeseries",
|
|
"description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_waiting",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "KV cache occupancy",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_kv_cache_usage",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Model disk free",
|
|
"type": "timeseries",
|
|
"description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
|
|
"gridPos": {
|
|
"x": 16,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_disk_free_bytes",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"title": "Seconds since each node last reported",
|
|
"type": "timeseries",
|
|
"description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_last_seen_seconds",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"title": "Replicas by state",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (state) (llmgw_replicas)",
|
|
"legendFormat": "{{state}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "GPUs",
|
|
"id": 14,
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 30,
|
|
"w": 24,
|
|
"h": 1
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU busy, by card",
|
|
"description": "Share of each sample period in which any kernel ran (nvidia-smi's utilization.gpu), sampled every heartbeat. Not how much of the chip: one SM busy reads 100%. The SMs panel and power against its limit say how much.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 31,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 15,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_utilization_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU memory in use, by card",
|
|
"description": "Framebuffer in use. Engines reserve most of it at start.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 31,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 16,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU memory bandwidth busy, by card",
|
|
"description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 38,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 17,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_memory_utilization_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU temperature",
|
|
"description": "Throttling usually starts in the high 80s.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 38,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 18,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_temperature_celsius",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "celsius"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU power against its limit",
|
|
"description": "1 is the enforced limit; a card held there is power-capped.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 45,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 19,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1.1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU clock",
|
|
"description": "A clock held low under load is the card throttled; see the next panel for why.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 45,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 20,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_sm_clock_mhz",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "rotmhz"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "What holds GPUs back",
|
|
"description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 52,
|
|
"w": 24,
|
|
"h": 7
|
|
},
|
|
"id": 21,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (reason) (llmgw_gpu_throttled)",
|
|
"legendFormat": "throttled: {{reason}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
|
|
"legendFormat": "ECC errors {{node}} gpu {{gpu}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Output tokens/s per GPU, by model",
|
|
"description": "What each model gets out of each card it holds.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 59,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 22,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
|
|
"legendFormat": "{{model}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Output tokens per watt-hour, by model",
|
|
"description": "Tokens for the energy the model's cards draw.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 59,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 23,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
|
|
"legendFormat": "{{model}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU wear",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"id": 24,
|
|
"description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 66,
|
|
"w": 24,
|
|
"h": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "increase(llmgw_gpu_xid_errors[15m]) > 0",
|
|
"legendFormat": "XID {{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0",
|
|
"legendFormat": "rows remapped {{node}} gpu {{gpu}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0",
|
|
"legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0",
|
|
"legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}",
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU SMs and tensor cores active, by card",
|
|
"description": "From the cards' profiling counters: NVML's GPM on Hopper and later, or a dcgm-exporter on the node. SMs active is the share of time the SMs had work, averaged over all of them; the gap to busy is chip a running kernel leaves idle. Tensor cores stay low in decode at batch one; prefill and batches are where they rise. Empty for cards nothing can profile.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 73,
|
|
"w": 24,
|
|
"h": 7
|
|
},
|
|
"id": 25,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_sm_active_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}} SMs",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "llmgw_gpu_tensor_active_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}} tensor",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
} |