{ "uid": "llmgw-tenants", "title": "llmgw \u2014 tenants", "schemaVersion": 39, "version": 1, "editable": true, "time": { "from": "now-24h", "to": "now" }, "refresh": "1m", "tags": [ "llmgw" ], "description": "Requests, tokens, failures, refusals and first-token time per tenant \u2014 the level router.tenant_label names \u2014 and every budget's share spent.", "templating": { "list": [ { "name": "tenant", "label": "Tenant", "type": "query", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "query": { "query": "label_values(llmgw_tenant_requests_total, tenant)", "refId": "tenant" }, "definition": "label_values(llmgw_tenant_requests_total, tenant)", "multi": true, "includeAll": true, "allValue": ".*", "current": { "text": "All", "value": "$__all" }, "refresh": 2, "sort": 1 } ] }, "panels": [ { "id": 1, "title": "Requests a second", "type": "stat", "description": "Across the tenants chosen above.", "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "reqps" }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 2, "title": "Failing", "type": "stat", "description": "Requests that failed, not counting refusals by a limit.", "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.02 }, { "color": "red", "value": 0.05 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 3, "title": "Refused by a limit", "type": "stat", "description": "Rate, token and budget limits; 429 to the caller.", "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.02 }, { "color": "red", "value": 0.1 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 4, "title": "First token, p95", "type": "stat", "description": "What people wait for before anything appears.", "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[5m])))", "legendFormat": "", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 3 }, { "color": "red", "value": 10 } ] } }, "overrides": [] }, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ] }, "textMode": "auto", "colorMode": "value" } }, { "id": 5, "title": "Requests by tenant", "type": "timeseries", "description": "Stacked, so the top edge is the whole of it.", "gridPos": { "x": 0, "y": 4, "w": 12, "h": 9 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "{{tenant}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "stacking": { "mode": "normal" }, "fillOpacity": 30 } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 6, "title": "Tokens by tenant", "type": "timeseries", "description": "Prompt and completion together: what was consumed.", "gridPos": { "x": 12, "y": 4, "w": 12, "h": 9 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (tenant) (rate(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "{{tenant}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "short", "custom": { "stacking": { "mode": "normal" }, "fillOpacity": 30 } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 7, "title": "Refused by a limit, by tenant", "type": "timeseries", "description": "The share of each tenant's requests a limit turned away.", "gridPos": { "x": 0, "y": 13, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "{{tenant}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 8, "title": "Failing, by tenant", "type": "timeseries", "description": "One tenant alone failing is usually what it sends; all of them together is a model.", "gridPos": { "x": 12, "y": 13, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", "legendFormat": "{{tenant}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 9, "title": "First token p95, by tenant", "type": "timeseries", "description": "", "gridPos": { "x": 0, "y": 21, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "histogram_quantile(0.95, sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[10m])))", "legendFormat": "{{tenant}}", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": [ "mean", "max" ] }, "tooltip": { "mode": "multi", "sort": "desc" } } }, { "id": 10, "title": "Budgets spent this period", "type": "bargauge", "description": "Every organization, team, project and person with a budget. Requests are refused at 100%.", "gridPos": { "x": 12, "y": 21, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sort_desc(llmgw_budget_spent_ratio)", "legendFormat": "{{scope}} {{name}} ({{period}})", "datasource": { "type": "prometheus", "uid": "llmgw-prom" } } ], "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ] }, "showUnfilled": true } }, { "id": 11, "title": "Who used most, last 24 hours", "type": "table", "description": "Spend in money is on the people page's usage tab, which reads it from the store.", "gridPos": { "x": 0, "y": 29, "w": 24, "h": 8 }, "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "targets": [ { "refId": "A", "expr": "sort_desc(sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[24h])))", "legendFormat": "requests", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "format": "table", "instant": true }, { "refId": "B", "expr": "sum by (tenant) (increase(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[24h]))", "legendFormat": "tokens", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "format": "table", "instant": true }, { "refId": "C", "expr": "sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[24h]))", "legendFormat": "limited", "datasource": { "type": "prometheus", "uid": "llmgw-prom" }, "format": "table", "instant": true } ], "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "showHeader": true }, "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "Value #A": "Requests", "Value #B": "Tokens", "Value #C": "Refused by a limit" } } } ] } ] }