Inserted between quota checks and scheduling: admission answers when a request may proceed, the scheduler still answers where it goes. Keeping them apart is the point — an engine queue is FIFO and knows nothing about tenants, so once a batch job is in it, no endpoint selection rescues the request behind it. - saturation detector: one number per pool, the worse of queue depth and KV occupancy, since either alone misleads. - priority bands with per-band saturation ceilings, so the cluster degrades from the bottom up. - tenant round-robin within a band: one caller sending a thousand requests takes one slot in the rotation, not a thousand. - bounded queues with capacity (503) and TTL (504); cancelled requests leave the queue instead of holding capacity. - metrics for queue depth, wait time, rejections and pool saturation. Exit criterion, measured in test/e2e: with 500 concurrent bronze requests running continuously, interactive p99 in the platinum band stays at 1.04-1.24x idle against a 1.5x budget. With admission off the same load pushes it to 55-60x. Both directions are asserted, so the test fails if admission regresses and also if the load generator stops being a real load. Two lags had to be closed, and they needed different answers. Engine metrics are scraped and lag an interval; router inflight does not, but it counts running requests, and treating those as queue makes a healthy engine look overloaded every interval. What is actually queued is inflight minus batch capacity, and capacity is not in the metric contract, so it is learned as the high-water mark of observed running. Separately, between admission granting a request and the router dispatching it the request is invisible everywhere, so a simultaneous burst passed the check en masse; the controller now counts its own outstanding leases against dispatched inflight. One plan correction: a saturation ceiling of 1.0 was documented as "never held back". It is not — it means "flow up to the healthy limit". A band that must never wait needs a ceiling above 1. Plan §7.2.2 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
138 lines
4.4 KiB
Go
138 lines
4.4 KiB
Go
package router
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
)
|
|
|
|
// metrics holds the router's own Prometheus series (plan §8).
|
|
//
|
|
// The scorer series matter more than they look: routing profile weights cannot
|
|
// be tuned without seeing what each scorer actually produced in production.
|
|
type metrics struct {
|
|
requests *prometheus.CounterVec
|
|
duration *prometheus.HistogramVec
|
|
ttft *prometheus.HistogramVec
|
|
tokens *prometheus.CounterVec
|
|
|
|
scorerValue *prometheus.GaugeVec
|
|
filtered *prometheus.CounterVec
|
|
considered prometheus.Histogram
|
|
|
|
admissionWait *prometheus.HistogramVec
|
|
admissionRejected *prometheus.CounterVec
|
|
}
|
|
|
|
func newMetrics() *metrics {
|
|
return &metrics{
|
|
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_requests_total",
|
|
Help: "Requests handled, by model, pool and response status.",
|
|
}, []string{"model", "pool", "status"}),
|
|
|
|
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_request_duration_seconds",
|
|
Help: "End-to-end request duration.",
|
|
Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300},
|
|
}, []string{"model", "pool"}),
|
|
|
|
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_ttft_seconds",
|
|
Help: "Time to first token, as observed by the router.",
|
|
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60},
|
|
}, []string{"model", "pool"}),
|
|
|
|
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tokens_total",
|
|
Help: "Tokens accounted for, by direction.",
|
|
}, []string{"model", "direction"}),
|
|
|
|
scorerValue: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "llmgw_scorer_value",
|
|
Help: "Last score each scorer produced for the endpoint that was chosen.",
|
|
}, []string{"scorer", "replica"}),
|
|
|
|
filtered: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_candidates_filtered_total",
|
|
Help: "Candidate endpoints removed, by filter.",
|
|
}, []string{"filter"}),
|
|
|
|
considered: prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "llmgw_candidates_considered",
|
|
Help: "Candidates remaining after filtering, per scheduling decision.",
|
|
Buckets: []float64{1, 2, 3, 5, 8, 16, 32, 64},
|
|
}),
|
|
|
|
admissionWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_admission_wait_seconds",
|
|
Help: "Time spent waiting in the admission queue.",
|
|
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30},
|
|
}, []string{"band", "model"}),
|
|
|
|
admissionRejected: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_admission_rejected_total",
|
|
Help: "Requests refused by flow control, by reason.",
|
|
}, []string{"band", "reason"}),
|
|
}
|
|
}
|
|
|
|
// collectors returns everything to register, so registration lives in one place.
|
|
func (m *metrics) collectors() []prometheus.Collector {
|
|
return []prometheus.Collector{
|
|
m.requests, m.duration, m.ttft, m.tokens,
|
|
m.scorerValue, m.filtered, m.considered,
|
|
m.admissionWait, m.admissionRejected,
|
|
}
|
|
}
|
|
|
|
func (m *metrics) observeAdmission(band, model string, wait time.Duration) {
|
|
if band == "" {
|
|
band = "none"
|
|
}
|
|
m.admissionWait.WithLabelValues(band, model).Observe(wait.Seconds())
|
|
}
|
|
|
|
func (m *metrics) observeAdmissionRejected(band, reason string) {
|
|
if band == "" {
|
|
band = "unknown"
|
|
}
|
|
m.admissionRejected.WithLabelValues(band, reason).Inc()
|
|
}
|
|
|
|
func (m *metrics) observeRequest(model, pool string, status int, d time.Duration, u *proxy.Usage, ttft time.Duration) {
|
|
if model == "" {
|
|
model = "unknown"
|
|
}
|
|
if pool == "" {
|
|
pool = "none"
|
|
}
|
|
m.requests.WithLabelValues(model, pool, strconv.Itoa(status)).Inc()
|
|
m.duration.WithLabelValues(model, pool).Observe(d.Seconds())
|
|
|
|
if ttft > 0 {
|
|
m.ttft.WithLabelValues(model, pool).Observe(ttft.Seconds())
|
|
}
|
|
if u != nil {
|
|
m.tokens.WithLabelValues(model, "prompt").Add(float64(u.PromptTokens))
|
|
m.tokens.WithLabelValues(model, "completion").Add(float64(u.CompletionTokens))
|
|
if u.CachedTokens > 0 {
|
|
m.tokens.WithLabelValues(model, "cached").Add(float64(u.CachedTokens))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *metrics) observeScores(d scheduler.Decision) {
|
|
for name, v := range d.Scores {
|
|
m.scorerValue.WithLabelValues(name, d.Endpoint.ID).Set(v)
|
|
}
|
|
for filter, n := range d.Dropped {
|
|
m.filtered.WithLabelValues(filter).Add(float64(n))
|
|
}
|
|
m.considered.Observe(float64(d.Considered))
|
|
}
|