Files
llmgw/internal/router/metrics.go
T
Ilker ManapandClaude Opus 5.5 3a3db8cd20
CI / test (push) Successful in 8m52s
Answers to similar questions, not only the same one
llmgw keys cache --similar 0.95 also gives a chat question the answer to
one like it: the last user message embedded by router.cache.embedding_model
(asked on the gateway's account, as a judge is, within 2 s), compared by
cosine with the questions answered in the same conversation — system
prompt, earlier turns, tools and sampling the same. The exact cache is
asked first; an exact hit embeds nothing. Said as x-llmgw-cache: similar
with x-llmgw-cache-similarity, charged as a hit.

The vectors live with their answers in the gateway's memory, counted in
max_bytes and evicted with them, not in a pgvector table as BACKLOG I1
planned: the cache keeps answer text out of the store on purpose. A
question over 2,000 characters is reused only when the same. Migration
0051 holds the setting. BACKLOG I1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-25 17:47:11 +01:00

264 lines
10 KiB
Go

package router
import (
"slices"
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/kvindex"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
)
// metrics holds the router's own Prometheus series (plan §8).
//
// The scorer series matter more than they look: routing profile weights cannot
// be tuned without seeing what each scorer actually produced in production.
type metrics struct {
requests *prometheus.CounterVec
duration *prometheus.HistogramVec
ttft *prometheus.HistogramVec
tokens *prometheus.CounterVec
scorerValue *prometheus.GaugeVec
filtered *prometheus.CounterVec
considered prometheus.Histogram
admissionWait *prometheus.HistogramVec
admissionRejected *prometheus.CounterVec
prefixDecisions *prometheus.CounterVec
// kvDecisions is the precise index's counterpart, and kvTokenize how
// long the tokenize call it needs took, by outcome.
kvDecisions *prometheus.CounterVec
kvTokenize *prometheus.HistogramVec
// adapterAffinity is where requests for adapters that load on demand
// found them; adapterLoads and adapterEvictions what that cost.
adapterAffinity *prometheus.CounterVec
adapterLoads *prometheus.CounterVec
adapterEvictions *prometheus.CounterVec
// cacheRequests counts what the answer cache did, cacheBytes and
// cacheEntries what it holds.
cacheRequests *prometheus.CounterVec
// kbRetrievals counts requests answered from knowledge bases by what
// the search found; kbRetrieval is how long finding it took.
kbRetrievals *prometheus.CounterVec
kbRetrieval prometheus.Histogram
cacheBytes prometheus.Gauge
cacheEntries prometheus.Gauge
tenant *tenantMetrics
guard *guardMetrics
token *tokenMetrics
}
func newMetrics() *metrics {
return &metrics{
prefixDecisions: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_prefix_decisions_total",
Help: "Prefix routing decisions by branch: cache_hit, cache_miss_min_load, " +
"load_balance_override, no_tree, no_healthy_worker.",
}, []string{"decision"}),
adapterAffinity: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_adapter_affinity_total",
Help: "Requests for LoRA adapters that load on demand, by where the chosen replica had it: " +
"hit (loaded), joined (being loaded for another request), loaded (loaded for this one).",
}, []string{"model", "result"}),
adapterLoads: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_adapter_loads_total",
Help: "LoRA adapters the router had a replica load, by outcome: ok or failed.",
}, []string{"model", "outcome"}),
adapterEvictions: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_adapter_evictions_total",
Help: "LoRA adapters the router had a replica unload to make room, by outcome: ok or failed.",
}, []string{"model", "outcome"}),
kvDecisions: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_kv_index_decisions_total",
Help: "Precise KV index routing decisions, for models with kv_events, by branch: kv_hit, " +
"kv_miss_min_load, load_balance_override, and fallback_no_index, fallback_unsupported, " +
"fallback_tokenize_failed, where tier 1 decided instead.",
}, []string{"decision"}),
kvTokenize: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_kv_index_tokenize_seconds",
Help: "Time the precise KV index waited for a replica to tokenize a request, by outcome: ok, timeout, error.",
Buckets: []float64{0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5},
}, []string{"outcome"}),
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_requests_total",
Help: "Requests handled, by model, pool and response status.",
}, []string{"model", "pool", "status"}),
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_request_duration_seconds",
Help: "End-to-end request duration.",
Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300},
}, []string{"model", "pool"}),
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_ttft_seconds",
Help: "Time to first token, as observed by the router.",
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60},
}, []string{"model", "pool"}),
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tokens_total",
Help: "Tokens accounted for, by direction.",
}, []string{"model", "direction"}),
scorerValue: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "llmgw_scorer_value",
Help: "Last score each scorer produced for the endpoint that was chosen.",
}, []string{"scorer", "replica"}),
filtered: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_candidates_filtered_total",
Help: "Candidate endpoints removed, by filter.",
}, []string{"filter"}),
considered: prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "llmgw_candidates_considered",
Help: "Candidates remaining after filtering, per scheduling decision.",
Buckets: []float64{1, 2, 3, 5, 8, 16, 32, 64},
}),
admissionWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_admission_wait_seconds",
Help: "Time spent waiting in the admission queue.",
Buckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 10, 30},
}, []string{"band", "model"}),
admissionRejected: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_admission_rejected_total",
Help: "Requests refused by flow control, by reason.",
}, []string{"band", "reason"}),
cacheRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_cache_requests_total",
Help: "Requests of projects that reuse answers, by what the answer cache did: " +
"hit (answered from it), similar (answered from it for a question like this one), miss (asked the model, and kept the answer if it could), " +
"bypass (not one whose answer is reused).",
}, []string{"model", "result"}),
kbRetrievals: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_kb_retrievals_total",
Help: "Chat requests that named knowledge-base collections, by what came of it: " +
"found (passages given to the model), nothing (none matched), refused (not answered).",
}, []string{"result"}),
kbRetrieval: prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "llmgw_kb_retrieval_seconds",
Help: "Time to find a request's passages: the question embedded, and the collections searched.",
Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
}),
cacheBytes: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "llmgw_cache_bytes",
Help: "Memory the answers kept for reuse take in this process, roughly.",
}),
cacheEntries: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "llmgw_cache_entries",
Help: "Answers kept for reuse in this process.",
}),
tenant: newTenantMetrics(),
guard: newGuardMetrics(),
token: newTokenMetrics(),
}
}
// collectors returns everything to register, so registration lives in one place.
func (m *metrics) collectors() []prometheus.Collector {
return append([]prometheus.Collector{
m.requests, m.duration, m.ttft, m.tokens,
m.scorerValue, m.filtered, m.considered,
m.admissionWait, m.admissionRejected, m.prefixDecisions,
m.adapterAffinity, m.adapterLoads, m.adapterEvictions,
m.kvDecisions, m.kvTokenize,
m.cacheRequests, m.cacheBytes, m.cacheEntries,
m.kbRetrievals, m.kbRetrieval,
}, slices.Concat(m.tenant.collectors(), m.guard.collectors(), m.token.collectors())...)
}
// observePrefix records which branch a prefix decision took.
//
// Without it, "affinity is not helping" and "affinity never applied" look
// identical from the outside, and the thresholds can only be tuned by guessing.
func (m *metrics) observePrefix(d prefix.Decision) {
m.prefixDecisions.WithLabelValues(string(d)).Inc()
}
// observeAdapter counts a load or an eviction the router made.
func (m *metrics) observeAdapter(model, what, outcome string) {
switch what {
case "load":
m.adapterLoads.WithLabelValues(model, outcome).Inc()
case "evict":
m.adapterEvictions.WithLabelValues(model, outcome).Inc()
}
}
// observeKV records which branch a precise index decision took.
func (m *metrics) observeKV(d kvindex.Decision) {
m.kvDecisions.WithLabelValues(string(d)).Inc()
}
func (m *metrics) observeAdmission(band, model string, wait time.Duration) {
if band == "" {
band = "none"
}
m.admissionWait.WithLabelValues(band, model).Observe(wait.Seconds())
}
func (m *metrics) observeAdmissionRejected(band, reason string) {
if band == "" {
band = "unknown"
}
m.admissionRejected.WithLabelValues(band, reason).Inc()
}
func (m *metrics) observeRequest(model, pool string, status int, d time.Duration, u *proxy.Usage, ttft time.Duration) {
if model == "" {
model = "unknown"
}
if pool == "" {
pool = "none"
}
m.requests.WithLabelValues(model, pool, strconv.Itoa(status)).Inc()
m.duration.WithLabelValues(model, pool).Observe(d.Seconds())
if ttft > 0 {
m.ttft.WithLabelValues(model, pool).Observe(ttft.Seconds())
}
if u != nil {
m.tokens.WithLabelValues(model, "prompt").Add(float64(u.PromptTokens))
m.tokens.WithLabelValues(model, "completion").Add(float64(u.CompletionTokens))
if u.CachedTokens > 0 {
m.tokens.WithLabelValues(model, "cached").Add(float64(u.CachedTokens))
}
}
}
func (m *metrics) observeScores(d scheduler.Decision) {
for name, v := range d.Scores {
m.scorerValue.WithLabelValues(name, d.Endpoint.ID).Set(v)
}
for filter, n := range d.Dropped {
m.filtered.WithLabelValues(filter).Add(float64(n))
}
m.considered.Observe(float64(d.Considered))
}
// metricModel is the model label a request is counted under: the catalog's
// name for it, or "unknown". Taken as sent, any key holder could mint a new
// series per request — a refused request counted its made-up model name —
// and grow Prometheus without bound.
func (r *Router) metricModel(name string) string {
if name == "" {
return ""
}
if m, ok := r.modelMap()[name]; ok {
return m.Name
}
return "unknown"
}