A vLLM model can carry PEFT adapters, registered at start, or with max_loaded_adapters held at most N per replica: the router loads an adapter through vLLM's runtime LoRA API when a request first wants it, sends the adapter's requests to a replica that holds it while that one is not saturated, loads each cold adapter once, and unloads the least recently used idle one to make room. llama.cpp keeps loading every adapter at start. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
227 lines
8.1 KiB
Go
227 lines
8.1 KiB
Go
package router
|
|
|
|
import (
|
|
"slices"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
)
|
|
|
|
// metrics holds the router's own Prometheus series (plan §8).
|
|
//
|
|
// The scorer series matter more than they look: routing profile weights cannot
|
|
// be tuned without seeing what each scorer actually produced in production.
|
|
type metrics struct {
|
|
requests *prometheus.CounterVec
|
|
duration *prometheus.HistogramVec
|
|
ttft *prometheus.HistogramVec
|
|
tokens *prometheus.CounterVec
|
|
|
|
scorerValue *prometheus.GaugeVec
|
|
filtered *prometheus.CounterVec
|
|
considered prometheus.Histogram
|
|
|
|
admissionWait *prometheus.HistogramVec
|
|
admissionRejected *prometheus.CounterVec
|
|
|
|
prefixDecisions *prometheus.CounterVec
|
|
|
|
// adapterAffinity is where requests for adapters that load on demand
|
|
// found them; adapterLoads and adapterEvictions what that cost.
|
|
adapterAffinity *prometheus.CounterVec
|
|
adapterLoads *prometheus.CounterVec
|
|
adapterEvictions *prometheus.CounterVec
|
|
|
|
// cacheRequests counts what the answer cache did, cacheBytes and
|
|
// cacheEntries what it holds.
|
|
cacheRequests *prometheus.CounterVec
|
|
cacheBytes prometheus.Gauge
|
|
cacheEntries prometheus.Gauge
|
|
|
|
tenant *tenantMetrics
|
|
guard *guardMetrics
|
|
token *tokenMetrics
|
|
}
|
|
|
|
func newMetrics() *metrics {
|
|
return &metrics{
|
|
prefixDecisions: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_prefix_decisions_total",
|
|
Help: "Prefix routing decisions by branch: cache_hit, cache_miss_min_load, " +
|
|
"load_balance_override, no_tree, no_healthy_worker.",
|
|
}, []string{"decision"}),
|
|
adapterAffinity: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_adapter_affinity_total",
|
|
Help: "Requests for LoRA adapters that load on demand, by where the chosen replica had it: " +
|
|
"hit (loaded), joined (being loaded for another request), loaded (loaded for this one).",
|
|
}, []string{"model", "result"}),
|
|
adapterLoads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_adapter_loads_total",
|
|
Help: "LoRA adapters the router had a replica load, by outcome: ok or failed.",
|
|
}, []string{"model", "outcome"}),
|
|
adapterEvictions: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_adapter_evictions_total",
|
|
Help: "LoRA adapters the router had a replica unload to make room, by outcome: ok or failed.",
|
|
}, []string{"model", "outcome"}),
|
|
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_requests_total",
|
|
Help: "Requests handled, by model, pool and response status.",
|
|
}, []string{"model", "pool", "status"}),
|
|
|
|
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_request_duration_seconds",
|
|
Help: "End-to-end request duration.",
|
|
Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300},
|
|
}, []string{"model", "pool"}),
|
|
|
|
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_ttft_seconds",
|
|
Help: "Time to first token, as observed by the router.",
|
|
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60},
|
|
}, []string{"model", "pool"}),
|
|
|
|
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tokens_total",
|
|
Help: "Tokens accounted for, by direction.",
|
|
}, []string{"model", "direction"}),
|
|
|
|
scorerValue: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "llmgw_scorer_value",
|
|
Help: "Last score each scorer produced for the endpoint that was chosen.",
|
|
}, []string{"scorer", "replica"}),
|
|
|
|
filtered: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_candidates_filtered_total",
|
|
Help: "Candidate endpoints removed, by filter.",
|
|
}, []string{"filter"}),
|
|
|
|
considered: prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "llmgw_candidates_considered",
|
|
Help: "Candidates remaining after filtering, per scheduling decision.",
|
|
Buckets: []float64{1, 2, 3, 5, 8, 16, 32, 64},
|
|
}),
|
|
|
|
admissionWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_admission_wait_seconds",
|
|
Help: "Time spent waiting in the admission queue.",
|
|
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30},
|
|
}, []string{"band", "model"}),
|
|
|
|
admissionRejected: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_admission_rejected_total",
|
|
Help: "Requests refused by flow control, by reason.",
|
|
}, []string{"band", "reason"}),
|
|
cacheRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_cache_requests_total",
|
|
Help: "Requests of projects that reuse answers, by what the answer cache did: " +
|
|
"hit (answered from it), miss (asked the model, and kept the answer if it could), " +
|
|
"bypass (not one whose answer is reused).",
|
|
}, []string{"model", "result"}),
|
|
cacheBytes: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "llmgw_cache_bytes",
|
|
Help: "Memory the answers kept for reuse take in this process, roughly.",
|
|
}),
|
|
cacheEntries: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "llmgw_cache_entries",
|
|
Help: "Answers kept for reuse in this process.",
|
|
}),
|
|
tenant: newTenantMetrics(),
|
|
guard: newGuardMetrics(),
|
|
token: newTokenMetrics(),
|
|
}
|
|
}
|
|
|
|
// collectors returns everything to register, so registration lives in one place.
|
|
func (m *metrics) collectors() []prometheus.Collector {
|
|
return append([]prometheus.Collector{
|
|
m.requests, m.duration, m.ttft, m.tokens,
|
|
m.scorerValue, m.filtered, m.considered,
|
|
m.admissionWait, m.admissionRejected, m.prefixDecisions,
|
|
m.adapterAffinity, m.adapterLoads, m.adapterEvictions,
|
|
m.cacheRequests, m.cacheBytes, m.cacheEntries,
|
|
}, slices.Concat(m.tenant.collectors(), m.guard.collectors(), m.token.collectors())...)
|
|
}
|
|
|
|
// observePrefix records which branch a prefix decision took.
|
|
//
|
|
// Without it, "affinity is not helping" and "affinity never applied" look
|
|
// identical from the outside, and the thresholds can only be tuned by guessing.
|
|
func (m *metrics) observePrefix(d prefix.Decision) {
|
|
m.prefixDecisions.WithLabelValues(string(d)).Inc()
|
|
}
|
|
|
|
// observeAdapter counts a load or an eviction the router made.
|
|
func (m *metrics) observeAdapter(model, what, outcome string) {
|
|
switch what {
|
|
case "load":
|
|
m.adapterLoads.WithLabelValues(model, outcome).Inc()
|
|
case "evict":
|
|
m.adapterEvictions.WithLabelValues(model, outcome).Inc()
|
|
}
|
|
}
|
|
|
|
func (m *metrics) observeAdmission(band, model string, wait time.Duration) {
|
|
if band == "" {
|
|
band = "none"
|
|
}
|
|
m.admissionWait.WithLabelValues(band, model).Observe(wait.Seconds())
|
|
}
|
|
|
|
func (m *metrics) observeAdmissionRejected(band, reason string) {
|
|
if band == "" {
|
|
band = "unknown"
|
|
}
|
|
m.admissionRejected.WithLabelValues(band, reason).Inc()
|
|
}
|
|
|
|
func (m *metrics) observeRequest(model, pool string, status int, d time.Duration, u *proxy.Usage, ttft time.Duration) {
|
|
if model == "" {
|
|
model = "unknown"
|
|
}
|
|
if pool == "" {
|
|
pool = "none"
|
|
}
|
|
m.requests.WithLabelValues(model, pool, strconv.Itoa(status)).Inc()
|
|
m.duration.WithLabelValues(model, pool).Observe(d.Seconds())
|
|
|
|
if ttft > 0 {
|
|
m.ttft.WithLabelValues(model, pool).Observe(ttft.Seconds())
|
|
}
|
|
if u != nil {
|
|
m.tokens.WithLabelValues(model, "prompt").Add(float64(u.PromptTokens))
|
|
m.tokens.WithLabelValues(model, "completion").Add(float64(u.CompletionTokens))
|
|
if u.CachedTokens > 0 {
|
|
m.tokens.WithLabelValues(model, "cached").Add(float64(u.CachedTokens))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (m *metrics) observeScores(d scheduler.Decision) {
|
|
for name, v := range d.Scores {
|
|
m.scorerValue.WithLabelValues(name, d.Endpoint.ID).Set(v)
|
|
}
|
|
for filter, n := range d.Dropped {
|
|
m.filtered.WithLabelValues(filter).Add(float64(n))
|
|
}
|
|
m.considered.Observe(float64(d.Considered))
|
|
}
|
|
|
|
// metricModel is the model label a request is counted under: the catalog's
|
|
// name for it, or "unknown". Taken as sent, any key holder could mint a new
|
|
// series per request — a refused request counted its made-up model name —
|
|
// and grow Prometheus without bound.
|
|
func (r *Router) metricModel(name string) string {
|
|
if name == "" {
|
|
return ""
|
|
}
|
|
if m, ok := r.modelMap()[name]; ok {
|
|
return m.Name
|
|
}
|
|
return "unknown"
|
|
}
|