CI / test (push) Successful in 1m19s
Tier 1 of plan §7.5: a radix tree of request prefixes that estimates which replica has already prefilled one. Bytes, not tokens — needing the engine's exact tokenizer is what makes tier 2 a separate project. It went in inert, and the decision metric is what caught it: 144 lookups, zero hits. The cause was a rule I had written and tested for. A node's label is the first request that created it, so the second request for the same system prompt diverges in its last few bytes — a new question after the same preamble, the single most common shape there is. Requiring the whole label to match threw away the 700 bytes they shared. What matters is the shared head, because that is what the engine cached, and a partial label match now counts. Two unit tests asserted the wrong rule and have been inverted. With that fixed, measured end to end against engines whose caches are too small for the working set: 22.5% cache hits without, 81.7% with, and the breakdown shows the misses are exactly the first round, when the tree is empty. The scenario matters as much as the number. With a handful of prompts and a large cache, scattering is free — every replica ends up holding everything — and prefix routing can only add constraint. The first version of the test measured that case and showed a loss, which was a true statement about a situation the feature is not for. Two things keep it from being the disaster prefix routing usually is. The imbalance guard, which needs both an absolute and a relative gap before it fires, because either alone misfires — absolute on an idle cluster, relative on a busy one. And it is a scorer rather than a filter: affinity is a vote the queue and inflight scorers can outvote, not a rule. The tree records the replica the picker actually chose, not the one the scorer proposed, and a node prefers the replica it has seen most rather than most recently — a count is stable under a randomizing picker, where recency just chases whatever the picker last did. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
154 lines
5.1 KiB
Go
154 lines
5.1 KiB
Go
package router
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
)
|
|
|
|
// metrics holds the router's own Prometheus series (plan §8).
|
|
//
|
|
// The scorer series matter more than they look: routing profile weights cannot
|
|
// be tuned without seeing what each scorer actually produced in production.
|
|
type metrics struct {
|
|
requests *prometheus.CounterVec
|
|
duration *prometheus.HistogramVec
|
|
ttft *prometheus.HistogramVec
|
|
tokens *prometheus.CounterVec
|
|
|
|
scorerValue *prometheus.GaugeVec
|
|
filtered *prometheus.CounterVec
|
|
considered prometheus.Histogram
|
|
|
|
admissionWait *prometheus.HistogramVec
|
|
admissionRejected *prometheus.CounterVec
|
|
|
|
prefixDecisions *prometheus.CounterVec
|
|
}
|
|
|
|
func newMetrics() *metrics {
|
|
return &metrics{
|
|
prefixDecisions: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_prefix_decisions_total",
|
|
Help: "Prefix routing decisions by branch: cache_hit, cache_miss_min_load, " +
|
|
"load_balance_override, no_tree, no_healthy_worker.",
|
|
}, []string{"decision"}),
|
|
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_requests_total",
|
|
Help: "Requests handled, by model, pool and response status.",
|
|
}, []string{"model", "pool", "status"}),
|
|
|
|
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_request_duration_seconds",
|
|
Help: "End-to-end request duration.",
|
|
Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300},
|
|
}, []string{"model", "pool"}),
|
|
|
|
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_ttft_seconds",
|
|
Help: "Time to first token, as observed by the router.",
|
|
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60},
|
|
}, []string{"model", "pool"}),
|
|
|
|
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tokens_total",
|
|
Help: "Tokens accounted for, by direction.",
|
|
}, []string{"model", "direction"}),
|
|
|
|
scorerValue: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "llmgw_scorer_value",
|
|
Help: "Last score each scorer produced for the endpoint that was chosen.",
|
|
}, []string{"scorer", "replica"}),
|
|
|
|
filtered: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_candidates_filtered_total",
|
|
Help: "Candidate endpoints removed, by filter.",
|
|
}, []string{"filter"}),
|
|
|
|
considered: prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "llmgw_candidates_considered",
|
|
Help: "Candidates remaining after filtering, per scheduling decision.",
|
|
Buckets: []float64{1, 2, 3, 5, 8, 16, 32, 64},
|
|
}),
|
|
|
|
admissionWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_admission_wait_seconds",
|
|
Help: "Time spent waiting in the admission queue.",
|
|
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30},
|
|
}, []string{"band", "model"}),
|
|
|
|
admissionRejected: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_admission_rejected_total",
|
|
Help: "Requests refused by flow control, by reason.",
|
|
}, []string{"band", "reason"}),
|
|
}
|
|
}
|
|
|
|
// collectors returns everything to register, so registration lives in one place.
|
|
func (m *metrics) collectors() []prometheus.Collector {
|
|
return []prometheus.Collector{
|
|
m.requests, m.duration, m.ttft, m.tokens,
|
|
m.scorerValue, m.filtered, m.considered,
|
|
m.admissionWait, m.admissionRejected, m.prefixDecisions,
|
|
}
|
|
}
|
|
|
|
// observePrefix records which branch a prefix decision took.
|
|
//
|
|
// Without it, "affinity is not helping" and "affinity never applied" look
|
|
// identical from the outside, and the thresholds can only be tuned by guessing.
|
|
func (m *metrics) observePrefix(d prefix.Decision) {
|
|
m.prefixDecisions.WithLabelValues(string(d)).Inc()
|
|
}
|
|
|
|
func (m *metrics) observeAdmission(band, model string, wait time.Duration) {
|
|
if band == "" {
|
|
band = "none"
|
|
}
|
|
m.admissionWait.WithLabelValues(band, model).Observe(wait.Seconds())
|
|
}
|
|
|
|
func (m *metrics) observeAdmissionRejected(band, reason string) {
|
|
if band == "" {
|
|
band = "unknown"
|
|
}
|
|
m.admissionRejected.WithLabelValues(band, reason).Inc()
|
|
}
|
|
|
|
func (m *metrics) observeRequest(model, pool string, status int, d time.Duration, u *proxy.Usage, ttft time.Duration) {
|
|
if model == "" {
|
|
model = "unknown"
|
|
}
|
|
if pool == "" {
|
|
pool = "none"
|
|
}
|
|
m.requests.WithLabelValues(model, pool, strconv.Itoa(status)).Inc()
|
|
m.duration.WithLabelValues(model, pool).Observe(d.Seconds())
|
|
|
|
if ttft > 0 {
|
|
m.ttft.WithLabelValues(model, pool).Observe(ttft.Seconds())
|
|
}
|
|
if u != nil {
|
|
m.tokens.WithLabelValues(model, "prompt").Add(float64(u.PromptTokens))
|
|
m.tokens.WithLabelValues(model, "completion").Add(float64(u.CompletionTokens))
|
|
if u.CachedTokens > 0 {
|
|
m.tokens.WithLabelValues(model, "cached").Add(float64(u.CachedTokens))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *metrics) observeScores(d scheduler.Decision) {
|
|
for name, v := range d.Scores {
|
|
m.scorerValue.WithLabelValues(name, d.Endpoint.ID).Set(v)
|
|
}
|
|
for filter, n := range d.Dropped {
|
|
m.filtered.WithLabelValues(filter).Add(float64(n))
|
|
}
|
|
m.considered.Observe(float64(d.Considered))
|
|
}
|