package scheduler import ( "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" "gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix" ) // PrefixScorer steers a request towards the replica that has probably already // prefilled its prefix (plan §7.5 tier 1). // // A scorer rather than a filter, deliberately. A filter would make affinity a // rule, and a rule that sends every request carrying a popular system prompt to // one replica is how prefix routing produces a beautiful cache hit rate and an // unusable p99. As a scorer it is one voice among several, and the queue and // inflight scorers can outvote it — on top of the imbalance guard inside the // lookup itself, which stops it speaking at all when load has gone lopsided. type PrefixScorer struct { tree *prefix.Tree thresholds prefix.Thresholds // observe reports which branch the decision took, for the metric without // which tuning this is guesswork. observe func(prefix.Decision) } // NewPrefixScorer builds the scorer. A nil tree makes it inert, which is what // a model with prefix routing turned off should get. func NewPrefixScorer(t *prefix.Tree, th prefix.Thresholds, observe func(prefix.Decision)) *PrefixScorer { if observe == nil { observe = func(prefix.Decision) {} } return &PrefixScorer{tree: t, thresholds: th, observe: observe} } // Name identifies the scorer in weights and metrics. func (*PrefixScorer) Name() string { return "prefix" } // Score gives the matched replica 1 and everything else 0. func (p *PrefixScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 { out := make([]float64, len(eps)) idx, decision := prefix.Choose(p.tree, c.Req.PromptPrefix, eps, p.thresholds) p.observe(decision) if idx >= 0 { out[idx] = 1 } return out } // Observe records where a request was actually sent. // // After the decision rather than during it: the scorer proposes, the picker // disposes, and recording the proposal would fill the tree with replicas that // were never sent anything — which is the one thing that would make the // estimate worse than no estimate at all. func (p *PrefixScorer) Observe(promptPrefix, replicaID string) { if p.tree != nil { p.tree.Observe(promptPrefix, replicaID) } } // Forget drops a replica that has gone away. func (p *PrefixScorer) Forget(replicaID string) { if p.tree != nil { p.tree.Forget(replicaID) } }