package prefix import ( "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) func ep(id string, waiting, running, inflight int) endpoint.Snapshot { return endpoint.Snapshot{ Info: endpoint.Info{ID: id}, Stats: endpoint.Stats{Waiting: waiting, Running: running}, Inflight: inflight, } } const prompt = "You are a helpful assistant answering questions about distributed systems. " func TestAMatchIsFollowedWhenLoadIsEven(t *testing.T) { tr := New(Config{MinMatch: 16}) tr.Observe(prompt, "b") eps := []endpoint.Snapshot{ep("a", 1, 1, 0), ep("b", 1, 1, 0)} i, d := Choose(tr, prompt+"What is a quorum?", eps, DefaultThresholds()) if d != Hit || i != 1 { t.Errorf("index %d, decision %s; want a hit on b", i, d) } } func TestAffinityIsAbandonedWhenLoadIsLopsided(t *testing.T) { // The guard the plan insists on. Without it the cache hit rate rises while // p99 collapses: affinity keeps feeding a hot prefix to one replica long // after it stopped being able to serve it quickly. tr := New(Config{MinMatch: 16}) tr.Observe(prompt, "b") eps := []endpoint.Snapshot{ep("a", 0, 1, 0), ep("b", 40, 8, 4)} i, d := Choose(tr, prompt+"What is a quorum?", eps, DefaultThresholds()) if d != LoadOverride { t.Errorf("decision %s; the guard should have fired", d) } if i != -1 { t.Errorf("index %d; the caller decides on load alone", i) } } func TestBothHalvesOfTheGuardMustTrip(t *testing.T) { tr := New(Config{MinMatch: 16}) tr.Observe(prompt, "b") full := prompt + "What is a quorum?" th := Thresholds{MatchRatio: 0.1, BalanceAbs: 8, BalanceRel: 1.5} cases := []struct { name string eps []endpoint.Snapshot want Decision }{ { // Relative is huge, absolute is tiny: an idle cluster where one // replica has two requests and another none. "a big ratio over a small difference", []endpoint.Snapshot{ep("a", 0, 0, 0), ep("b", 2, 0, 0)}, Hit, }, { // Absolute is large, relative is nothing: a busy cluster where 200 // against 180 is a rounding error. "a big difference at a small ratio", []endpoint.Snapshot{ep("a", 180, 0, 0), ep("b", 200, 0, 0)}, Hit, }, { "both, which is what the guard is for", []endpoint.Snapshot{ep("a", 2, 0, 0), ep("b", 60, 0, 0)}, LoadOverride, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { if _, d := Choose(tr, full, c.eps, th); d != c.want { t.Errorf("decision %s, want %s", d, c.want) } }) } } func TestAnIdleReplicaAlwaysWinsOverAffinity(t *testing.T) { // A ratio against zero is infinite, and an idle replica is exactly where // the next request should go. tr := New(Config{MinMatch: 16}) tr.Observe(prompt, "b") eps := []endpoint.Snapshot{ep("a", 0, 0, 0), ep("b", 20, 0, 0)} if _, d := Choose(tr, prompt+"x", eps, DefaultThresholds()); d != LoadOverride { t.Errorf("decision %s; an idle replica should win", d) } } func TestASmallMatchRatioIsAMiss(t *testing.T) { // A short shared header on a very long prompt saves almost no prefill, and // pretending otherwise trades load balance for nothing. tr := New(Config{MinMatch: 8}) tr.Observe("SYSTEM: be brief. ", "b") long := "SYSTEM: be brief. " + strings.Repeat("unique content here. ", 200) eps := []endpoint.Snapshot{ep("a", 1, 0, 0), ep("b", 1, 0, 0)} if _, d := Choose(tr, long, eps, Thresholds{MatchRatio: 0.5}); d != MissMinLoad { t.Errorf("decision %s, want a miss", d) } } func TestAMatchOnAReplicaThatIsGoneIsAMiss(t *testing.T) { // The tree outlives a replica leaving the candidate set — drained, breaker // open, removed. That is a miss, not an error. tr := New(Config{MinMatch: 16}) tr.Observe(prompt, "vanished") eps := []endpoint.Snapshot{ep("a", 1, 0, 0), ep("b", 1, 0, 0)} if i, d := Choose(tr, prompt+"x", eps, DefaultThresholds()); d != MissMinLoad || i != -1 { t.Errorf("index %d decision %s", i, d) } } func TestTheDegenerateCasesAreNamed(t *testing.T) { // Every branch is a distinct decision, because "affinity is not helping" // and "affinity never applied" look identical from the outside otherwise. tr := New(Config{MinMatch: 4}) if _, d := Choose(tr, "p", nil, DefaultThresholds()); d != NoCandidates { t.Errorf("no endpoints: %s", d) } eps := []endpoint.Snapshot{ep("a", 0, 0, 0)} if _, d := Choose(nil, "p", eps, DefaultThresholds()); d != NoTree { t.Errorf("no tree: %s", d) } if _, d := Choose(tr, "", eps, DefaultThresholds()); d != NoTree { t.Errorf("no prompt: %s", d) } } func TestOneCandidateNeverTripsTheGuard(t *testing.T) { // There is nothing to balance against, so overriding affinity would throw // away a cache hit for no reason. tr := New(Config{MinMatch: 16}) tr.Observe(prompt, "only") eps := []endpoint.Snapshot{ep("only", 500, 500, 500)} if _, d := Choose(tr, prompt+"x", eps, DefaultThresholds()); d != Hit { t.Errorf("decision %s, want a hit", d) } }