Ilker ManapandClaude Opus 5
b921fa0d6b
Prometheus, Alertmanager and Grafana, with every rule written against a series this system actually publishes. Nothing here is aimed at a metric that might exist. One scrape target for the whole fleet. Agents publish nothing of their own — the control plane gathers what they report and republishes it — which is worth writing down before somebody goes looking for an exporter to install on each node. It scrapes metrics_listen, not the admin listener. That one is loopback because it also serves pprof, and pprof can dump the heap where API keys sit in plaintext. Metrics are the one thing on it something else genuinely needs, so they get a port of their own rather than becoming a reason to open the other. The README shows how to confirm the new port carries nothing else; it answers 404 to both pprof and the admin API. Nothing is sent anywhere. Alertmanager groups and holds and has no receiver: where an alert should go is a decision with credentials attached, and a monitoring system that quietly starts mailing people is worse than one that is visibly not wired up. critical already routes separately so it can be pointed somewhere else without touching the rest. The rules encode what the metrics mean rather than round numbers. A saturation of 1 is the healthy limit measured by ctl bench, not the point of failure, so the alert waits five minutes above it. A thirty-second p99 first token is the threshold because this is CPU inference and would be absurd on a GPU. A model that is deliberately stopped publishes no series at all, so "nothing serving" cannot fire for one nobody asked for — which is what lets an on-demand catalogue sit quiet. Inhibitions, because a node that is down explains every replica on it and reporting both buries the cause. The dashboard is provisioned from a file rather than clicked together, so it survives a reinstall and can be read in a diff. Checked by firing one: cordoning a node moved llmgw_node_up to 0, the rule went pending and then firing after its two minutes, the alert arrived at Alertmanager with the node in its labels, and uncordoning resolved it. An alerting system nobody has seen fire is an assumption. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
492 lines
11 KiB
JSON
492 lines
11 KiB
JSON
{
|
|
"uid": "llmgw-overview",
|
|
"title": "llmgw \u2014 cluster",
|
|
"tags": [
|
|
"llmgw"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Nodes ready",
|
|
"type": "stat",
|
|
"description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_node_up == 1)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Replicas serving",
|
|
"type": "stat",
|
|
"description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
|
|
"gridPos": {
|
|
"x": 4,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(llmgw_model_replicas_ready)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Models with something serving",
|
|
"type": "stat",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_model_replicas_ready > 0)",
|
|
"legendFormat": "models",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Requests per second",
|
|
"type": "timeseries",
|
|
"description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 0,
|
|
"w": 12,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_requests_total[5m]))",
|
|
"legendFormat": "all",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
|
|
"legendFormat": "failing",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Replicas ready, by model",
|
|
"type": "timeseries",
|
|
"description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_model_replicas_ready",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Pool saturation",
|
|
"type": "timeseries",
|
|
"description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_pool_saturation",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"max": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "Time to first token",
|
|
"type": "timeseries",
|
|
"description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p50",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p99",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "Tokens per second",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
|
|
"legendFormat": "{{direction}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "Engine queue depth",
|
|
"type": "timeseries",
|
|
"description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_waiting",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "KV cache occupancy",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_kv_cache_usage",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Model disk free",
|
|
"type": "timeseries",
|
|
"description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
|
|
"gridPos": {
|
|
"x": 16,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_disk_free_bytes",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"title": "Seconds since each node last reported",
|
|
"type": "timeseries",
|
|
"description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_last_seen_seconds",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"title": "Replicas by state",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (state) (llmgw_replicas)",
|
|
"legendFormat": "{{state}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
}
|
|
]
|
|
} |