Files
llmgw/deploy/monitoring/grafana/llmgw-overview.json
T
2026-10-01 15:56:42 +02:00

860 lines
19 KiB
JSON

{
"uid": "llmgw-overview",
"title": "llmgw \u2014 cluster",
"tags": [
"llmgw"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-6h",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "Nodes ready",
"type": "stat",
"description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
"gridPos": {
"x": 0,
"y": 0,
"w": 4,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "count(llmgw_node_up == 1)",
"legendFormat": "ready",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"textMode": "auto",
"colorMode": "value"
}
},
{
"id": 2,
"title": "Replicas serving",
"type": "stat",
"description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
"gridPos": {
"x": 4,
"y": 0,
"w": 4,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum(llmgw_model_replicas_ready)",
"legendFormat": "ready",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"textMode": "auto",
"colorMode": "value"
}
},
{
"id": 3,
"title": "Models with something serving",
"type": "stat",
"description": "",
"gridPos": {
"x": 8,
"y": 0,
"w": 4,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "count(llmgw_model_replicas_ready > 0)",
"legendFormat": "models",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
]
},
"textMode": "auto",
"colorMode": "value"
}
},
{
"id": 4,
"title": "Requests per second",
"type": "timeseries",
"description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 4
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(llmgw_requests_total[5m]))",
"legendFormat": "all",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
},
{
"refId": "B",
"expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
"legendFormat": "failing",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 5,
"title": "Replicas ready, by model",
"type": "timeseries",
"description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
"gridPos": {
"x": 0,
"y": 4,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_model_replicas_ready",
"legendFormat": "{{model}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
}
},
{
"id": 6,
"title": "Pool saturation",
"type": "timeseries",
"description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
"gridPos": {
"x": 12,
"y": 4,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_pool_saturation",
"legendFormat": "{{model}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"max": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
}
},
{
"id": 7,
"title": "Time to first token",
"type": "timeseries",
"description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
"gridPos": {
"x": 0,
"y": 11,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
"legendFormat": "{{model}} p50",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
},
{
"refId": "B",
"expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
"legendFormat": "{{model}} p99",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 8,
"title": "Tokens per second",
"type": "timeseries",
"description": "",
"gridPos": {
"x": 12,
"y": 11,
"w": 12,
"h": 7
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
"legendFormat": "{{direction}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 9,
"title": "Engine queue depth",
"type": "timeseries",
"description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
"gridPos": {
"x": 0,
"y": 18,
"w": 8,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_endpoint_waiting",
"legendFormat": "{{replica}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
}
},
{
"id": 10,
"title": "KV cache occupancy",
"type": "timeseries",
"description": "",
"gridPos": {
"x": 8,
"y": 18,
"w": 8,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_endpoint_kv_cache_usage",
"legendFormat": "{{replica}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 11,
"title": "Model disk free",
"type": "timeseries",
"description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
"gridPos": {
"x": 16,
"y": 18,
"w": 8,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_node_disk_free_bytes",
"legendFormat": "{{node}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
},
{
"id": 12,
"title": "Seconds since each node last reported",
"type": "timeseries",
"description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "llmgw_node_last_seen_seconds",
"legendFormat": "{{node}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 13,
"title": "Replicas by state",
"type": "timeseries",
"description": "",
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 6
},
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"targets": [
{
"refId": "A",
"expr": "sum by (state) (llmgw_replicas)",
"legendFormat": "{{state}}",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
}
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
}
},
{
"type": "row",
"title": "GPUs",
"id": 14,
"gridPos": {
"x": 0,
"y": 30,
"w": 24,
"h": 1
},
"collapsed": false,
"panels": []
},
{
"type": "timeseries",
"title": "GPU busy, by card",
"description": "Share of each sample period a kernel was running, sampled every heartbeat.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 31,
"w": 12,
"h": 7
},
"id": 15,
"targets": [
{
"expr": "llmgw_gpu_utilization_percent",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU memory in use, by card",
"description": "Framebuffer in use. Engines reserve most of it at start.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 31,
"w": 12,
"h": 7
},
"id": 16,
"targets": [
{
"expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU memory bandwidth busy, by card",
"description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 38,
"w": 12,
"h": 7
},
"id": 17,
"targets": [
{
"expr": "llmgw_gpu_memory_utilization_percent",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU temperature",
"description": "Throttling usually starts in the high 80s.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 38,
"w": 12,
"h": 7
},
"id": 18,
"targets": [
{
"expr": "llmgw_gpu_temperature_celsius",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "celsius"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU power against its limit",
"description": "1 is the enforced limit; a card held there is power-capped.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 45,
"w": 12,
"h": 7
},
"id": 19,
"targets": [
{
"expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"min": 0,
"max": 1.1
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "GPU clock",
"description": "A clock held low under load is the card throttled; see the next panel for why.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 45,
"w": 12,
"h": 7
},
"id": 20,
"targets": [
{
"expr": "llmgw_gpu_sm_clock_mhz",
"legendFormat": "{{node}} gpu {{gpu}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "rotmhz"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "What holds GPUs back",
"description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 52,
"w": 24,
"h": 7
},
"id": 21,
"targets": [
{
"expr": "sum by (reason) (llmgw_gpu_throttled)",
"legendFormat": "throttled: {{reason}}",
"refId": "A"
},
{
"expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
"legendFormat": "ECC errors {{node}} gpu {{gpu}}",
"refId": "B"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "Output tokens/s per GPU, by model",
"description": "What each model gets out of each card it holds.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 0,
"y": 59,
"w": 12,
"h": 7
},
"id": 22,
"targets": [
{
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
},
{
"type": "timeseries",
"title": "Output tokens per watt-hour, by model",
"description": "Tokens for the energy the model's cards draw.",
"datasource": {
"type": "prometheus",
"uid": "llmgw-prom"
},
"gridPos": {
"x": 12,
"y": 59,
"w": 12,
"h": 7
},
"id": 23,
"targets": [
{
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
}
}
]
}