Files
llmgw/deploy/monitoring/grafana/llmgw-overview.json
T
Ilker ManapandClaude Opus 5.5 bd36798b15
Release / build (push) Successful in 1m12s
Release / notes (push) Successful in 4s
CI / test (push) Successful in 3m52s
Sample GPU utilisation and memory every heartbeat, and chart them
They were read once, at agent start, and published unchanged since.
Each sample now carries utilisation, memory and bandwidth in use, the SM
clock and uncorrected ECC errors. The node page charts each card, the
cluster page the fleet by class, Grafana has a GPU row, and three GPU
alerts are added.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 15:42:17 +02:00

784 lines
17 KiB
JSON

{
"uid": "llmgw-overview",
"title": "llmgw \u2014 cluster",
"tags": [
"llmgw"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-6h",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "Nodes ready",
"type": "stat",
"description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
"gridPos": {
"x": 0,
"y": 0,
"w": 4,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "count(llmgw_node_up == 1)",
"legendFormat": "ready",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"textMode": "auto",
"colorMode": "value"
}
},
{
"id": 2,
"title": "Replicas serving",
"type": "stat",
"description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
"gridPos": {
"x": 4,
"y": 0,
"w": 4,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum(llmgw_model_replicas_ready)",
"legendFormat": "ready",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"textMode": "auto",
"colorMode": "value"
}
},
{
"id": 3,
"title": "Models with something serving",
"type": "stat",
"description": "",
"gridPos": {
"x": 8,
"y": 0,
"w": 4,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "count(llmgw_model_replicas_ready > 0)",
"legendFormat": "models",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"textMode": "auto",
"colorMode": "value"
}
},
{
"id": 4,
"title": "Requests per second",
"type": "timeseries",
"description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(llmgw_requests_total[5m]))",
"legendFormat": "all",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
},
{
"refId": "B",
"expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
"legendFormat": "failing",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 5,
"title": "Replicas ready, by model",
"type": "timeseries",
"description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
"gridPos": {
"x": 0,
"y": 4,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_model_replicas_ready",
"legendFormat": "{{model}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
}
},
{
"id": 6,
"title": "Pool saturation",
"type": "timeseries",
"description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
"gridPos": {
"x": 12,
"y": 4,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_pool_saturation",
"legendFormat": "{{model}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"max": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
}
},
{
"id": 7,
"title": "Time to first token",
"type": "timeseries",
"description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
"gridPos": {
"x": 0,
"y": 11,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
"legendFormat": "{{model}} p50",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
"legendFormat": "{{model}} p99",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 8,
"title": "Tokens per second",
"type": "timeseries",
"description": "",
"gridPos": {
"x": 12,
"y": 11,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
"legendFormat": "{{direction}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 9,
"title": "Engine queue depth",
"type": "timeseries",
"description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
"gridPos": {
"x": 0,
"y": 18,
"w": 8,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_endpoint_waiting",
"legendFormat": "{{replica}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
}
},
{
"id": 10,
"title": "KV cache occupancy",
"type": "timeseries",
"description": "",
"gridPos": {
"x": 8,
"y": 18,
"w": 8,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_endpoint_kv_cache_usage",
"legendFormat": "{{replica}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 11,
"title": "Model disk free",
"type": "timeseries",
"description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
"gridPos": {
"x": 16,
"y": 18,
"w": 8,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_node_disk_free_bytes",
"legendFormat": "{{node}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
},
{
"id": 12,
"title": "Seconds since each node last reported",
"type": "timeseries",
"description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_node_last_seen_seconds",
"legendFormat": "{{node}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 13,
"title": "Replicas by state",
"type": "timeseries",
"description": "",
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum by (state) (llmgw_replicas)",
"legendFormat": "{{state}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
}
},
{
"type": "row",
"title": "GPUs",
"id": 14,
"gridPos": {
"x": 0,
"y": 30,
"w": 24,
"h": 1
},
"collapsed": false,
"panels": []
},
{
"type": "timeseries",
"title": "GPU busy, by card",
"description": "Share of each sample period a kernel was running, sampled every heartbeat.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 31,
"w": 12,
"h": 7
},
"id": 15,
"targets": [
{
"expr": "llmgw_gpu_utilization_percent",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU memory in use, by card",
"description": "Framebuffer in use. Engines reserve most of it at start.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 31,
"w": 12,
"h": 7
},
"id": 16,
"targets": [
{
"expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU memory bandwidth busy, by card",
"description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 38,
"w": 12,
"h": 7
},
"id": 17,
"targets": [
{
"expr": "llmgw_gpu_memory_utilization_percent",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU temperature",
"description": "Throttling usually starts in the high 80s.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 38,
"w": 12,
"h": 7
},
"id": 18,
"targets": [
{
"expr": "llmgw_gpu_temperature_celsius",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "celsius"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU power against its limit",
"description": "1 is the enforced limit; a card held there is power-capped.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 45,
"w": 12,
"h": 7
},
"id": 19,
"targets": [
{
"expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1.1
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU clock",
"description": "A clock held low under load is the card throttled; see the next panel for why.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 45,
"w": 12,
"h": 7
},
"id": 20,
"targets": [
{
"expr": "llmgw_gpu_sm_clock_mhz",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "rotmhz"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "What holds GPUs back",
"description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 52,
"w": 24,
"h": 7
},
"id": 21,
"targets": [
{
"expr": "sum by (reason) (llmgw_gpu_throttled)",
"legendFormat": "throttled: {{reason}}",
"refId": "A"
},
{
"expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
"legendFormat": "ECC errors {{node}} gpu {{gpu}}",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
}
]
}