{ "uid": "llmgw-overview", "title": "llmgw \u2014 cluster", "tags": [ "llmgw" ], "timezone": "browser", "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "panels": [ { "id": 1, "title": "Nodes ready", "type": "stat", "description": "Nodes taking work. A node that is known but draining or cordoned is not counted.", "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "count(llmgw_node_up == 1)", "legendFormat": "ready", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 2, "title": "Replicas serving", "type": "stat", "description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.", "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(llmgw_model_replicas_ready)", "legendFormat": "ready", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 3, "title": "Models with something serving", "type": "stat", "description": "", "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "count(llmgw_model_replicas_ready > 0)", "legendFormat": "models", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 4, "title": "Requests per second", "type": "timeseries", "description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.", "gridPos": { "x": 12, "y": 0, "w": 12, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(rate(llmgw_requests_total[5m]))", "legendFormat": "all", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } }, { "refId": "B", "expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))", "legendFormat": "failing", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] } }, { "id": 5, "title": "Replicas ready, by model", "type": "timeseries", "description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.", "gridPos": { "x": 0, "y": 4, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_model_replicas_ready", "legendFormat": "{{model}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] } }, { "id": 6, "title": "Pool saturation", "type": "timeseries", "description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.", "gridPos": { "x": 12, "y": 4, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_pool_saturation", "legendFormat": "{{model}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 7, "title": "Time to first token", "type": "timeseries", "description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.", "gridPos": { "x": 0, "y": 11, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))", "legendFormat": "{{model}} p50", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } }, { "refId": "B", "expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))", "legendFormat": "{{model}} p99", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } }, { "id": 8, "title": "Tokens per second", "type": "timeseries", "description": "", "gridPos": { "x": 12, "y": 11, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))", "legendFormat": "{{direction}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } }, { "id": 9, "title": "Engine queue depth", "type": "timeseries", "description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.", "gridPos": { "x": 0, "y": 18, "w": 8, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_endpoint_waiting", "legendFormat": "{{replica}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] } }, { "id": 10, "title": "KV cache occupancy", "type": "timeseries", "description": "", "gridPos": { "x": 8, "y": 18, "w": 8, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_endpoint_kv_cache_usage", "legendFormat": "{{replica}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] } }, { "id": 11, "title": "Model disk free", "type": "timeseries", "description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.", "gridPos": { "x": 16, "y": 18, "w": 8, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_node_disk_free_bytes", "legendFormat": "{{node}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } }, { "id": 12, "title": "Seconds since each node last reported", "type": "timeseries", "description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.", "gridPos": { "x": 0, "y": 24, "w": 12, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_node_last_seen_seconds", "legendFormat": "{{node}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } }, { "id": 13, "title": "Replicas by state", "type": "timeseries", "description": "", "gridPos": { "x": 12, "y": 24, "w": 12, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (state) (llmgw_replicas)", "legendFormat": "{{state}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] } }, { "type": "row", "title": "GPUs", "id": 14, "gridPos": { "x": 0, "y": 30, "w": 24, "h": 1 }, "collapsed": false, "panels": [] }, { "type": "timeseries", "title": "GPU busy, by card", "description": "Share of each sample period a kernel was running, sampled every heartbeat.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 0, "y": 31, "w": 12, "h": 7 }, "id": 15, "targets": [ { "expr": "llmgw_gpu_utilization_percent", "legendFormat": "{{node}} gpu {{gpu}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "GPU memory in use, by card", "description": "Framebuffer in use. Engines reserve most of it at start.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 12, "y": 31, "w": 12, "h": 7 }, "id": 16, "targets": [ { "expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes", "legendFormat": "{{node}} gpu {{gpu}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "GPU memory bandwidth busy, by card", "description": "Decode is bound by memory: high here with compute low is a card waiting on memory.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 0, "y": 38, "w": 12, "h": 7 }, "id": 17, "targets": [ { "expr": "llmgw_gpu_memory_utilization_percent", "legendFormat": "{{node}} gpu {{gpu}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "GPU temperature", "description": "Throttling usually starts in the high 80s.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 12, "y": 38, "w": 12, "h": 7 }, "id": 18, "targets": [ { "expr": "llmgw_gpu_temperature_celsius", "legendFormat": "{{node}} gpu {{gpu}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "celsius" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "GPU power against its limit", "description": "1 is the enforced limit; a card held there is power-capped.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 0, "y": 45, "w": 12, "h": 7 }, "id": 19, "targets": [ { "expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts", "legendFormat": "{{node}} gpu {{gpu}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1.1 }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "GPU clock", "description": "A clock held low under load is the card throttled; see the next panel for why.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 12, "y": 45, "w": 12, "h": 7 }, "id": 20, "targets": [ { "expr": "llmgw_gpu_sm_clock_mhz", "legendFormat": "{{node}} gpu {{gpu}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "rotmhz" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "What holds GPUs back", "description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 0, "y": 52, "w": 24, "h": 7 }, "id": 21, "targets": [ { "expr": "sum by (reason) (llmgw_gpu_throttled)", "legendFormat": "throttled: {{reason}}", "refId": "A" }, { "expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)", "legendFormat": "ECC errors {{node}} gpu {{gpu}}", "refId": "B" } ], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "Output tokens/s per GPU, by model", "description": "What each model gets out of each card it holds.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 0, "y": 59, "w": 12, "h": 7 }, "id": 22, "targets": [ { "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)", "legendFormat": "{{model}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "Output tokens per watt-hour, by model", "description": "Tokens for the energy the model's cards draw.", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "gridPos": { "x": 12, "y": 59, "w": 12, "h": 7 }, "id": 23, "targets": [ { "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)", "legendFormat": "{{model}}", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } }, { "type": "timeseries", "title": "GPU wear", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "id": 24, "description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.", "gridPos": { "x": 0, "y": 66, "w": 24, "h": 7 }, "targets": [ { "expr": "increase(llmgw_gpu_xid_errors[15m]) > 0", "legendFormat": "XID {{node}} gpu {{gpu}}", "refId": "A" }, { "expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0", "legendFormat": "rows remapped {{node}} gpu {{gpu}}", "refId": "B" }, { "expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0", "legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}", "refId": "C" }, { "expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0", "legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}", "refId": "D" } ], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } } } ] }