Files
llmgw/internal/router/prefix/scorer_test.go
T
Ilker ManapandClaude Opus 5 a075ff4120
CI / test (push) Successful in 1m19s
Add approximate prefix routing, and find out it did nothing
Tier 1 of plan §7.5: a radix tree of request prefixes that estimates which
replica has already prefilled one. Bytes, not tokens — needing the engine's
exact tokenizer is what makes tier 2 a separate project.

It went in inert, and the decision metric is what caught it: 144 lookups, zero
hits. The cause was a rule I had written and tested for. A node's label is the
first request that created it, so the second request for the same system
prompt diverges in its last few bytes — a new question after the same
preamble, the single most common shape there is. Requiring the whole label to
match threw away the 700 bytes they shared. What matters is the shared head,
because that is what the engine cached, and a partial label match now counts.
Two unit tests asserted the wrong rule and have been inverted.

With that fixed, measured end to end against engines whose caches are too
small for the working set: 22.5% cache hits without, 81.7% with, and the
breakdown shows the misses are exactly the first round, when the tree is
empty.

The scenario matters as much as the number. With a handful of prompts and a
large cache, scattering is free — every replica ends up holding everything —
and prefix routing can only add constraint. The first version of the test
measured that case and showed a loss, which was a true statement about a
situation the feature is not for.

Two things keep it from being the disaster prefix routing usually is. The
imbalance guard, which needs both an absolute and a relative gap before it
fires, because either alone misfires — absolute on an idle cluster, relative
on a busy one. And it is a scorer rather than a filter: affinity is a vote the
queue and inflight scorers can outvote, not a rule.

The tree records the replica the picker actually chose, not the one the scorer
proposed, and a node prefers the replica it has seen most rather than most
recently — a count is stable under a randomizing picker, where recency just
chases whatever the picker last did.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 12:13:19 +02:00

152 lines
4.8 KiB
Go

package prefix
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func ep(id string, waiting, running, inflight int) endpoint.Snapshot {
return endpoint.Snapshot{
Info: endpoint.Info{ID: id},
Stats: endpoint.Stats{Waiting: waiting, Running: running},
Inflight: inflight,
}
}
const prompt = "You are a helpful assistant answering questions about distributed systems. "
func TestAMatchIsFollowedWhenLoadIsEven(t *testing.T) {
tr := New(Config{MinMatch: 16})
tr.Observe(prompt, "b")
eps := []endpoint.Snapshot{ep("a", 1, 1, 0), ep("b", 1, 1, 0)}
i, d := Choose(tr, prompt+"What is a quorum?", eps, DefaultThresholds())
if d != Hit || i != 1 {
t.Errorf("index %d, decision %s; want a hit on b", i, d)
}
}
func TestAffinityIsAbandonedWhenLoadIsLopsided(t *testing.T) {
// The guard the plan insists on. Without it the cache hit rate rises while
// p99 collapses: affinity keeps feeding a hot prefix to one replica long
// after it stopped being able to serve it quickly.
tr := New(Config{MinMatch: 16})
tr.Observe(prompt, "b")
eps := []endpoint.Snapshot{ep("a", 0, 1, 0), ep("b", 40, 8, 4)}
i, d := Choose(tr, prompt+"What is a quorum?", eps, DefaultThresholds())
if d != LoadOverride {
t.Errorf("decision %s; the guard should have fired", d)
}
if i != -1 {
t.Errorf("index %d; the caller decides on load alone", i)
}
}
func TestBothHalvesOfTheGuardMustTrip(t *testing.T) {
tr := New(Config{MinMatch: 16})
tr.Observe(prompt, "b")
full := prompt + "What is a quorum?"
th := Thresholds{MatchRatio: 0.1, BalanceAbs: 8, BalanceRel: 1.5}
cases := []struct {
name string
eps []endpoint.Snapshot
want Decision
}{
{
// Relative is huge, absolute is tiny: an idle cluster where one
// replica has two requests and another none.
"a big ratio over a small difference",
[]endpoint.Snapshot{ep("a", 0, 0, 0), ep("b", 2, 0, 0)},
Hit,
},
{
// Absolute is large, relative is nothing: a busy cluster where 200
// against 180 is a rounding error.
"a big difference at a small ratio",
[]endpoint.Snapshot{ep("a", 180, 0, 0), ep("b", 200, 0, 0)},
Hit,
},
{
"both, which is what the guard is for",
[]endpoint.Snapshot{ep("a", 2, 0, 0), ep("b", 60, 0, 0)},
LoadOverride,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if _, d := Choose(tr, full, c.eps, th); d != c.want {
t.Errorf("decision %s, want %s", d, c.want)
}
})
}
}
func TestAnIdleReplicaAlwaysWinsOverAffinity(t *testing.T) {
// A ratio against zero is infinite, and an idle replica is exactly where
// the next request should go.
tr := New(Config{MinMatch: 16})
tr.Observe(prompt, "b")
eps := []endpoint.Snapshot{ep("a", 0, 0, 0), ep("b", 20, 0, 0)}
if _, d := Choose(tr, prompt+"x", eps, DefaultThresholds()); d != LoadOverride {
t.Errorf("decision %s; an idle replica should win", d)
}
}
func TestASmallMatchRatioIsAMiss(t *testing.T) {
// A short shared header on a very long prompt saves almost no prefill, and
// pretending otherwise trades load balance for nothing.
tr := New(Config{MinMatch: 8})
tr.Observe("SYSTEM: be brief. ", "b")
long := "SYSTEM: be brief. " + strings.Repeat("unique content here. ", 200)
eps := []endpoint.Snapshot{ep("a", 1, 0, 0), ep("b", 1, 0, 0)}
if _, d := Choose(tr, long, eps, Thresholds{MatchRatio: 0.5}); d != MissMinLoad {
t.Errorf("decision %s, want a miss", d)
}
}
func TestAMatchOnAReplicaThatIsGoneIsAMiss(t *testing.T) {
// The tree outlives a replica leaving the candidate set — drained, breaker
// open, removed. That is a miss, not an error.
tr := New(Config{MinMatch: 16})
tr.Observe(prompt, "vanished")
eps := []endpoint.Snapshot{ep("a", 1, 0, 0), ep("b", 1, 0, 0)}
if i, d := Choose(tr, prompt+"x", eps, DefaultThresholds()); d != MissMinLoad || i != -1 {
t.Errorf("index %d decision %s", i, d)
}
}
func TestTheDegenerateCasesAreNamed(t *testing.T) {
// Every branch is a distinct decision, because "affinity is not helping"
// and "affinity never applied" look identical from the outside otherwise.
tr := New(Config{MinMatch: 4})
if _, d := Choose(tr, "p", nil, DefaultThresholds()); d != NoCandidates {
t.Errorf("no endpoints: %s", d)
}
eps := []endpoint.Snapshot{ep("a", 0, 0, 0)}
if _, d := Choose(nil, "p", eps, DefaultThresholds()); d != NoTree {
t.Errorf("no tree: %s", d)
}
if _, d := Choose(tr, "", eps, DefaultThresholds()); d != NoTree {
t.Errorf("no prompt: %s", d)
}
}
func TestOneCandidateNeverTripsTheGuard(t *testing.T) {
// There is nothing to balance against, so overriding affinity would throw
// away a cache hit for no reason.
tr := New(Config{MinMatch: 16})
tr.Observe(prompt, "only")
eps := []endpoint.Snapshot{ep("only", 500, 500, 500)}
if _, d := Choose(tr, prompt+"x", eps, DefaultThresholds()); d != Hit {
t.Errorf("decision %s, want a hit", d)
}
}