package router import ( "strconv" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler" ) // metrics holds the router's own Prometheus series (plan ยง8). // // The scorer series matter more than they look: routing profile weights cannot // be tuned without seeing what each scorer actually produced in production. type metrics struct { requests *prometheus.CounterVec duration *prometheus.HistogramVec ttft *prometheus.HistogramVec tokens *prometheus.CounterVec scorerValue *prometheus.GaugeVec filtered *prometheus.CounterVec considered prometheus.Histogram admissionWait *prometheus.HistogramVec admissionRejected *prometheus.CounterVec prefixDecisions *prometheus.CounterVec } func newMetrics() *metrics { return &metrics{ prefixDecisions: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_prefix_decisions_total", Help: "Prefix routing decisions by branch: cache_hit, cache_miss_min_load, " + "load_balance_override, no_tree, no_healthy_worker.", }, []string{"decision"}), requests: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_requests_total", Help: "Requests handled, by model, pool and response status.", }, []string{"model", "pool", "status"}), duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "llmgw_request_duration_seconds", Help: "End-to-end request duration.", Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300}, }, []string{"model", "pool"}), ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "llmgw_ttft_seconds", Help: "Time to first token, as observed by the router.", Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60}, }, []string{"model", "pool"}), tokens: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_tokens_total", Help: "Tokens accounted for, by direction.", }, []string{"model", "direction"}), scorerValue: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "llmgw_scorer_value", Help: "Last score each scorer produced for the endpoint that was chosen.", }, []string{"scorer", "replica"}), filtered: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_candidates_filtered_total", Help: "Candidate endpoints removed, by filter.", }, []string{"filter"}), considered: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "llmgw_candidates_considered", Help: "Candidates remaining after filtering, per scheduling decision.", Buckets: []float64{1, 2, 3, 5, 8, 16, 32, 64}, }), admissionWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "llmgw_admission_wait_seconds", Help: "Time spent waiting in the admission queue.", Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30}, }, []string{"band", "model"}), admissionRejected: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_admission_rejected_total", Help: "Requests refused by flow control, by reason.", }, []string{"band", "reason"}), } } // collectors returns everything to register, so registration lives in one place. func (m *metrics) collectors() []prometheus.Collector { return []prometheus.Collector{ m.requests, m.duration, m.ttft, m.tokens, m.scorerValue, m.filtered, m.considered, m.admissionWait, m.admissionRejected, m.prefixDecisions, } } // observePrefix records which branch a prefix decision took. // // Without it, "affinity is not helping" and "affinity never applied" look // identical from the outside, and the thresholds can only be tuned by guessing. func (m *metrics) observePrefix(d prefix.Decision) { m.prefixDecisions.WithLabelValues(string(d)).Inc() } func (m *metrics) observeAdmission(band, model string, wait time.Duration) { if band == "" { band = "none" } m.admissionWait.WithLabelValues(band, model).Observe(wait.Seconds()) } func (m *metrics) observeAdmissionRejected(band, reason string) { if band == "" { band = "unknown" } m.admissionRejected.WithLabelValues(band, reason).Inc() } func (m *metrics) observeRequest(model, pool string, status int, d time.Duration, u *proxy.Usage, ttft time.Duration) { if model == "" { model = "unknown" } if pool == "" { pool = "none" } m.requests.WithLabelValues(model, pool, strconv.Itoa(status)).Inc() m.duration.WithLabelValues(model, pool).Observe(d.Seconds()) if ttft > 0 { m.ttft.WithLabelValues(model, pool).Observe(ttft.Seconds()) } if u != nil { m.tokens.WithLabelValues(model, "prompt").Add(float64(u.PromptTokens)) m.tokens.WithLabelValues(model, "completion").Add(float64(u.CompletionTokens)) if u.CachedTokens > 0 { m.tokens.WithLabelValues(model, "cached").Add(float64(u.CachedTokens)) } } } func (m *metrics) observeScores(d scheduler.Decision) { for name, v := range d.Scores { m.scorerValue.WithLabelValues(name, d.Endpoint.ID).Set(v) } for filter, n := range d.Dropped { m.filtered.WithLabelValues(filter).Add(float64(n)) } m.considered.Observe(float64(d.Considered)) }