Ilker ManapandClaude Opus 5.5
17cc2e379b
Requests and tokens by tenant, the shares failing and refused by a limit, first-token p95, budgets spent this period and the day's heaviest users, filtered by a tenant chooser. Installed beside the cluster dashboard. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
650 lines
15 KiB
JSON
650 lines
15 KiB
JSON
{
|
|
"uid": "llmgw-tenants",
|
|
"title": "llmgw \u2014 tenants",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"editable": true,
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"refresh": "1m",
|
|
"tags": [
|
|
"llmgw"
|
|
],
|
|
"description": "Requests, tokens, failures, refusals and first-token time per tenant \u2014 the level router.tenant_label names \u2014 and every budget's share spent.",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "tenant",
|
|
"label": "Tenant",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"query": {
|
|
"query": "label_values(llmgw_tenant_requests_total, tenant)",
|
|
"refId": "tenant"
|
|
},
|
|
"definition": "label_values(llmgw_tenant_requests_total, tenant)",
|
|
"multi": true,
|
|
"includeAll": true,
|
|
"allValue": ".*",
|
|
"current": {
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"refresh": 2,
|
|
"sort": 1
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Requests a second",
|
|
"type": "stat",
|
|
"description": "Across the tenants chosen above.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"w": 6,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Failing",
|
|
"type": "stat",
|
|
"description": "Requests that failed, not counting refusals by a limit.",
|
|
"gridPos": {
|
|
"x": 6,
|
|
"y": 0,
|
|
"w": 6,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.02
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.05
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Refused by a limit",
|
|
"type": "stat",
|
|
"description": "Rate, token and budget limits; 429 to the caller.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 0,
|
|
"w": 6,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.02
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "First token, p95",
|
|
"type": "stat",
|
|
"description": "What people wait for before anything appears.",
|
|
"gridPos": {
|
|
"x": 18,
|
|
"y": 0,
|
|
"w": 6,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
|
|
"legendFormat": "",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 3
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 10
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Requests by tenant",
|
|
"type": "timeseries",
|
|
"description": "Stacked, so the top edge is the whole of it.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 9
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "{{tenant}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {
|
|
"stacking": {
|
|
"mode": "normal"
|
|
},
|
|
"fillOpacity": 30
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Tokens by tenant",
|
|
"type": "timeseries",
|
|
"description": "Prompt and completion together: what was consumed.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 9
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (tenant) (rate(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "{{tenant}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short",
|
|
"custom": {
|
|
"stacking": {
|
|
"mode": "normal"
|
|
},
|
|
"fillOpacity": 30
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "Refused by a limit, by tenant",
|
|
"type": "timeseries",
|
|
"description": "The share of each tenant's requests a limit turned away.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 13,
|
|
"w": 12,
|
|
"h": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "{{tenant}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "Failing, by tenant",
|
|
"type": "timeseries",
|
|
"description": "One tenant alone failing is usually what it sends; all of them together is a model.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 13,
|
|
"w": 12,
|
|
"h": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
|
|
"legendFormat": "{{tenant}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "First token p95, by tenant",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 21,
|
|
"w": 12,
|
|
"h": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.95, sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[10m])))",
|
|
"legendFormat": "{{tenant}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right",
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
]
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "Budgets spent this period",
|
|
"type": "bargauge",
|
|
"description": "Every organization, team, project and person with a budget. Requests are refused at 100%.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 21,
|
|
"w": 12,
|
|
"h": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sort_desc(llmgw_budget_spent_ratio)",
|
|
"legendFormat": "{{scope}} {{name}} ({{period}})",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 0.8
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"showUnfilled": true
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Who used most, last 24 hours",
|
|
"type": "table",
|
|
"description": "Spend in money is on the people page's usage tab, which reads it from the store.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 29,
|
|
"w": 24,
|
|
"h": 8
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sort_desc(sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[24h])))",
|
|
"legendFormat": "requests",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"format": "table",
|
|
"instant": true
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "sum by (tenant) (increase(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[24h]))",
|
|
"legendFormat": "tokens",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"format": "table",
|
|
"instant": true
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"expr": "sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[24h]))",
|
|
"legendFormat": "limited",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"format": "table",
|
|
"instant": true
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true
|
|
},
|
|
"transformations": [
|
|
{
|
|
"id": "merge",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true
|
|
},
|
|
"renameByName": {
|
|
"Value #A": "Requests",
|
|
"Value #B": "Tokens",
|
|
"Value #C": "Refused by a limit"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
} |