{ "uid": "llmgw-overview", "title": "llmgw \u2014 cluster", "tags": [ "llmgw" ], "timezone": "browser", "schemaVersion": 39, "version": 1, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "panels": [ { "id": 1, "title": "Nodes ready", "type": "stat", "description": "Nodes taking work. A node that is known but draining or cordoned is not counted.", "gridPos": { "x": 0, "y": 0, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "count(llmgw_node_up == 1)", "legendFormat": "ready", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 2, "title": "Replicas serving", "type": "stat", "description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.", "gridPos": { "x": 4, "y": 0, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(llmgw_model_replicas_ready)", "legendFormat": "ready", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 3, "title": "Models with something serving", "type": "stat", "description": "", "gridPos": { "x": 8, "y": 0, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "count(llmgw_model_replicas_ready > 0)", "legendFormat": "models", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 4, "title": "Requests per second", "type": "timeseries", "description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.", "gridPos": { "x": 12, "y": 0, "w": 12, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(rate(llmgw_requests_total[5m]))", "legendFormat": "all", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } }, { "refId": "B", "expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))", "legendFormat": "failing", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] } }, { "id": 5, "title": "Replicas ready, by model", "type": "timeseries", "description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.", "gridPos": { "x": 0, "y": 4, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_model_replicas_ready", "legendFormat": "{{model}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] } }, { "id": 6, "title": "Pool saturation", "type": "timeseries", "description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.", "gridPos": { "x": 12, "y": 4, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_pool_saturation", "legendFormat": "{{model}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] } }, { "id": 7, "title": "Time to first token", "type": "timeseries", "description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.", "gridPos": { "x": 0, "y": 11, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))", "legendFormat": "{{model}} p50", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } }, { "refId": "B", "expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))", "legendFormat": "{{model}} p99", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } }, { "id": 8, "title": "Tokens per second", "type": "timeseries", "description": "", "gridPos": { "x": 12, "y": 11, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))", "legendFormat": "{{direction}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "short" }, "overrides": [] } }, { "id": 9, "title": "Engine queue depth", "type": "timeseries", "description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.", "gridPos": { "x": 0, "y": 18, "w": 8, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_endpoint_waiting", "legendFormat": "{{replica}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] } }, { "id": 10, "title": "KV cache occupancy", "type": "timeseries", "description": "", "gridPos": { "x": 8, "y": 18, "w": 8, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_endpoint_kv_cache_usage", "legendFormat": "{{replica}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] } }, { "id": 11, "title": "Model disk free", "type": "timeseries", "description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.", "gridPos": { "x": 16, "y": 18, "w": 8, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_node_disk_free_bytes", "legendFormat": "{{node}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] } }, { "id": 12, "title": "Seconds since each node last reported", "type": "timeseries", "description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.", "gridPos": { "x": 0, "y": 24, "w": 12, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "llmgw_node_last_seen_seconds", "legendFormat": "{{node}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] } }, { "id": 13, "title": "Replicas by state", "type": "timeseries", "description": "", "gridPos": { "x": 12, "y": 24, "w": 12, "h": 6 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (state) (llmgw_replicas)", "legendFormat": "{{state}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": {}, "overrides": [] } } ] }