Ilker ManapandClaude Opus 5.5
82a63cfc6e
Every minute the agent reads nvidia-smi's full report, its NVLink error counters and the kernel's XID lines. They are published per card, shown on the node page and the GPU map, charted in Grafana, and five alerts watch them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
913 lines
21 KiB
JSON
913 lines
21 KiB
JSON
{
|
|
"uid": "llmgw-overview",
|
|
"title": "llmgw \u2014 cluster",
|
|
"tags": [
|
|
"llmgw"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Nodes ready",
|
|
"type": "stat",
|
|
"description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_node_up == 1)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Replicas serving",
|
|
"type": "stat",
|
|
"description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
|
|
"gridPos": {
|
|
"x": 4,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(llmgw_model_replicas_ready)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Models with something serving",
|
|
"type": "stat",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_model_replicas_ready > 0)",
|
|
"legendFormat": "models",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Requests per second",
|
|
"type": "timeseries",
|
|
"description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 0,
|
|
"w": 12,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_requests_total[5m]))",
|
|
"legendFormat": "all",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
|
|
"legendFormat": "failing",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Replicas ready, by model",
|
|
"type": "timeseries",
|
|
"description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_model_replicas_ready",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Pool saturation",
|
|
"type": "timeseries",
|
|
"description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_pool_saturation",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"max": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "Time to first token",
|
|
"type": "timeseries",
|
|
"description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p50",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p99",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "Tokens per second",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
|
|
"legendFormat": "{{direction}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "Engine queue depth",
|
|
"type": "timeseries",
|
|
"description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_waiting",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "KV cache occupancy",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_kv_cache_usage",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Model disk free",
|
|
"type": "timeseries",
|
|
"description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
|
|
"gridPos": {
|
|
"x": 16,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_disk_free_bytes",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"title": "Seconds since each node last reported",
|
|
"type": "timeseries",
|
|
"description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_last_seen_seconds",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"title": "Replicas by state",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (state) (llmgw_replicas)",
|
|
"legendFormat": "{{state}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "GPUs",
|
|
"id": 14,
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 30,
|
|
"w": 24,
|
|
"h": 1
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU busy, by card",
|
|
"description": "Share of each sample period a kernel was running, sampled every heartbeat.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 31,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 15,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_utilization_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU memory in use, by card",
|
|
"description": "Framebuffer in use. Engines reserve most of it at start.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 31,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 16,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU memory bandwidth busy, by card",
|
|
"description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 38,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 17,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_memory_utilization_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU temperature",
|
|
"description": "Throttling usually starts in the high 80s.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 38,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 18,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_temperature_celsius",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "celsius"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU power against its limit",
|
|
"description": "1 is the enforced limit; a card held there is power-capped.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 45,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 19,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1.1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU clock",
|
|
"description": "A clock held low under load is the card throttled; see the next panel for why.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 45,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 20,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_sm_clock_mhz",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "rotmhz"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "What holds GPUs back",
|
|
"description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 52,
|
|
"w": 24,
|
|
"h": 7
|
|
},
|
|
"id": 21,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (reason) (llmgw_gpu_throttled)",
|
|
"legendFormat": "throttled: {{reason}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
|
|
"legendFormat": "ECC errors {{node}} gpu {{gpu}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Output tokens/s per GPU, by model",
|
|
"description": "What each model gets out of each card it holds.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 59,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 22,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
|
|
"legendFormat": "{{model}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Output tokens per watt-hour, by model",
|
|
"description": "Tokens for the energy the model's cards draw.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 59,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 23,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
|
|
"legendFormat": "{{model}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU wear",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"id": 24,
|
|
"description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 66,
|
|
"w": 24,
|
|
"h": 7
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "increase(llmgw_gpu_xid_errors[15m]) > 0",
|
|
"legendFormat": "XID {{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0",
|
|
"legendFormat": "rows remapped {{node}} gpu {{gpu}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0",
|
|
"legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0",
|
|
"legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}",
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
} |