Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
860 lines
19 KiB
JSON
860 lines
19 KiB
JSON
{
|
|
"uid": "llmgw-overview",
|
|
"title": "llmgw \u2014 cluster",
|
|
"tags": [
|
|
"llmgw"
|
|
],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Nodes ready",
|
|
"type": "stat",
|
|
"description": "Nodes taking work. A node that is known but draining or cordoned is not counted.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_node_up == 1)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Replicas serving",
|
|
"type": "stat",
|
|
"description": "Across every model. A model deliberately stopped contributes nothing and is not a fault.",
|
|
"gridPos": {
|
|
"x": 4,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(llmgw_model_replicas_ready)",
|
|
"legendFormat": "ready",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Models with something serving",
|
|
"type": "stat",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 0,
|
|
"w": 4,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "count(llmgw_model_replicas_ready > 0)",
|
|
"legendFormat": "models",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto",
|
|
"colorMode": "value"
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Requests per second",
|
|
"type": "timeseries",
|
|
"description": "Failing is counted separately rather than as a percentage, so a quiet cluster reads as quiet instead of as 0% of nothing.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 0,
|
|
"w": 12,
|
|
"h": 4
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum(rate(llmgw_requests_total[5m]))",
|
|
"legendFormat": "all",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "sum(rate(llmgw_requests_total{status=~\"5..\"}[5m]))",
|
|
"legendFormat": "failing",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Replicas ready, by model",
|
|
"type": "timeseries",
|
|
"description": "Only models with a replica placed appear. One that is stopped on purpose publishes no series.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_model_replicas_ready",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Pool saturation",
|
|
"type": "timeseries",
|
|
"description": "The worse of queue depth and KV occupancy. 1 is the limit measured as healthy, not the point of failure.",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 4,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_pool_saturation",
|
|
"legendFormat": "{{model}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"max": 2,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "Time to first token",
|
|
"type": "timeseries",
|
|
"description": "As the router sees it. On CPU these are seconds, not milliseconds; compare against what ctl bench measured.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "histogram_quantile(0.5, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p50",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"expr": "histogram_quantile(0.99, sum by (model, le) (rate(llmgw_ttft_seconds_bucket[5m])))",
|
|
"legendFormat": "{{model}} p99",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "Tokens per second",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 11,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (direction) (rate(llmgw_tokens_total[5m]))",
|
|
"legendFormat": "{{direction}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "Engine queue depth",
|
|
"type": "timeseries",
|
|
"description": "Requests the engine has accepted and not started. Rising here is the first sign of saturation.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_waiting",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "KV cache occupancy",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 8,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_endpoint_kv_cache_usage",
|
|
"legendFormat": "{{replica}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Model disk free",
|
|
"type": "timeseries",
|
|
"description": "The filesystems holding weights. A node that cannot hold the next model is skipped by placement rather than failing.",
|
|
"gridPos": {
|
|
"x": 16,
|
|
"y": 18,
|
|
"w": 8,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_disk_free_bytes",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"title": "Seconds since each node last reported",
|
|
"type": "timeseries",
|
|
"description": "Heartbeats are every 5s. A line that climbs is an agent going quiet; the control plane stops placing there at 15s.",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "llmgw_node_last_seen_seconds",
|
|
"legendFormat": "{{node}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"title": "Replicas by state",
|
|
"type": "timeseries",
|
|
"description": "",
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 24,
|
|
"w": 12,
|
|
"h": 6
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"expr": "sum by (state) (llmgw_replicas)",
|
|
"legendFormat": "{{state}}",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
}
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
}
|
|
},
|
|
{
|
|
"type": "row",
|
|
"title": "GPUs",
|
|
"id": 14,
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 30,
|
|
"w": 24,
|
|
"h": 1
|
|
},
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU busy, by card",
|
|
"description": "Share of each sample period a kernel was running, sampled every heartbeat.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 31,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 15,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_utilization_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU memory in use, by card",
|
|
"description": "Framebuffer in use. Engines reserve most of it at start.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 31,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 16,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU memory bandwidth busy, by card",
|
|
"description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 38,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 17,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_memory_utilization_percent",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU temperature",
|
|
"description": "Throttling usually starts in the high 80s.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 38,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 18,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_temperature_celsius",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "celsius"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU power against its limit",
|
|
"description": "1 is the enforced limit; a card held there is power-capped.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 45,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 19,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1.1
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "GPU clock",
|
|
"description": "A clock held low under load is the card throttled; see the next panel for why.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 45,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 20,
|
|
"targets": [
|
|
{
|
|
"expr": "llmgw_gpu_sm_clock_mhz",
|
|
"legendFormat": "{{node}} gpu {{gpu}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "rotmhz"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "What holds GPUs back",
|
|
"description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 52,
|
|
"w": 24,
|
|
"h": 7
|
|
},
|
|
"id": 21,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (reason) (llmgw_gpu_throttled)",
|
|
"legendFormat": "throttled: {{reason}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
|
|
"legendFormat": "ECC errors {{node}} gpu {{gpu}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Output tokens/s per GPU, by model",
|
|
"description": "What each model gets out of each card it holds.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 59,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 22,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
|
|
"legendFormat": "{{model}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "timeseries",
|
|
"title": "Output tokens per watt-hour, by model",
|
|
"description": "Tokens for the energy the model's cards draw.",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "llmgw-prom"
|
|
},
|
|
"gridPos": {
|
|
"x": 12,
|
|
"y": 59,
|
|
"w": 12,
|
|
"h": 7
|
|
},
|
|
"id": 23,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
|
|
"legendFormat": "{{model}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
} |