diff --git a/CHANGELOG.md b/CHANGELOG.md index 9897e48..5a3a973 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,10 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration `BudgetSpent`, `TenantHittingLimits` (a tenth of requests refused), `TenantErrors`, `TenantTrafficSpike` (three times the weekly average) and `TenantSlowFirstToken`. +- **A Grafana dashboard per tenant**, `deploy/monitoring/grafana/llmgw-tenants.json` + (uid `llmgw-tenants`): requests and tokens by tenant, the share failing and + refused by a limit, first-token p95, every budget's share spent, and who used + most in the last day — with a tenant chooser at the top. ### How each team's requests fared, not only what they cost diff --git a/deploy/monitoring/README.md b/deploy/monitoring/README.md index 2483daa..b81a5f7 100644 --- a/deploy/monitoring/README.md +++ b/deploy/monitoring/README.md @@ -66,6 +66,8 @@ install -m 0644 grafana/datasource.yaml /etc/grafana/provisioning/datasources/ install -m 0644 grafana/dashboards.yaml /etc/grafana/provisioning/dashboards/llmgw.yaml install -m 0644 -o grafana -g grafana grafana/llmgw-overview.json \ /var/lib/grafana/dashboards/llmgw.json +install -m 0644 -o grafana -g grafana grafana/llmgw-tenants.json \ + /var/lib/grafana/dashboards/llmgw-tenants.json systemctl restart grafana-server ``` diff --git a/deploy/monitoring/grafana/llmgw-tenants.json b/deploy/monitoring/grafana/llmgw-tenants.json new file mode 100644 index 0000000..f9b39c5 --- /dev/null +++ b/deploy/monitoring/grafana/llmgw-tenants.json @@ -0,0 +1,650 @@ +{ + "uid": "llmgw-tenants", + "title": "llmgw \u2014 tenants", + "schemaVersion": 39, + "version": 1, + "editable": true, + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "1m", + "tags": [ + "llmgw" + ], + "description": "Requests, tokens, failures, refusals and first-token time per tenant \u2014 the level router.tenant_label names \u2014 and every budget's share spent.", + "templating": { + "list": [ + { + "name": "tenant", + "label": "Tenant", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "query": { + "query": "label_values(llmgw_tenant_requests_total, tenant)", + "refId": "tenant" + }, + "definition": "label_values(llmgw_tenant_requests_total, tenant)", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Requests a second", + "type": "stat", + "description": "Across the tenants chosen above.", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 2, + "title": "Failing", + "type": "stat", + "description": "Requests that failed, not counting refusals by a limit.", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.02 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 3, + "title": "Refused by a limit", + "type": "stat", + "description": "Rate, token and budget limits; 429 to the caller.", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.02 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 4, + "title": "First token, p95", + "type": "stat", + "description": "What people wait for before anything appears.", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[5m])))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 5, + "title": "Requests by tenant", + "type": "timeseries", + "description": "Stacked, so the top edge is the whole of it.", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 30 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "Tokens by tenant", + "type": "timeseries", + "description": "Prompt and completion together: what was consumed.", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 30 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "title": "Refused by a limit, by tenant", + "type": "timeseries", + "description": "The share of each tenant's requests a limit turned away.", + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Failing, by tenant", + "type": "timeseries", + "description": "One tenant alone failing is usually what it sends; all of them together is a model.", + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "title": "First token p95, by tenant", + "type": "timeseries", + "description": "", + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[10m])))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "title": "Budgets spent this period", + "type": "bargauge", + "description": "Every organization, team, project and person with a budget. Requests are refused at 100%.", + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(llmgw_budget_spent_ratio)", + "legendFormat": "{{scope}} {{name}} ({{period}})", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "showUnfilled": true + } + }, + { + "id": 11, + "title": "Who used most, last 24 hours", + "type": "table", + "description": "Spend in money is on the people page's usage tab, which reads it from the store.", + "gridPos": { + "x": 0, + "y": 29, + "w": 24, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[24h])))", + "legendFormat": "requests", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "format": "table", + "instant": true + }, + { + "refId": "B", + "expr": "sum by (tenant) (increase(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[24h]))", + "legendFormat": "tokens", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "format": "table", + "instant": true + }, + { + "refId": "C", + "expr": "sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[24h]))", + "legendFormat": "limited", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "Value #A": "Requests", + "Value #B": "Tokens", + "Value #C": "Refused by a limit" + } + } + } + ] + } + ] +} \ No newline at end of file