package scheduler import ( "fmt" "math" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) func ep(id string, mut ...func(*endpoint.Snapshot)) endpoint.Snapshot { s := endpoint.Snapshot{ Info: endpoint.Info{ID: id, URL: "http://" + id, Models: []string{"m"}, Weight: 1}, Breaker: breaker.Closed, } for _, f := range mut { f(&s) } return s } func waiting(n int) func(*endpoint.Snapshot) { return func(s *endpoint.Snapshot) { s.Stats.Waiting = n } } func kv(v float64) func(*endpoint.Snapshot) { return func(s *endpoint.Snapshot) { s.Stats.KVCacheUsage = v } } func inflight(n int) func(*endpoint.Snapshot) { return func(s *endpoint.Snapshot) { s.Inflight = n } } func newCtx(req Request) *Ctx { return &Ctx{Req: req, Now: time.Now()} } func TestEmptyCandidateList(t *testing.T) { p := Chat(ProfileConfig{}) if _, err := p.Schedule(newCtx(Request{Model: "m"}), nil); err != ErrNoEndpoints { t.Errorf("expected ErrNoEndpoints, got %v", err) } } func TestPicksShortestQueue(t *testing.T) { p := &Profile{ Name: "t", Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore(), } eps := []endpoint.Snapshot{ ep("a", waiting(3)), ep("b", waiting(0)), ep("c", waiting(2)), } d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err != nil { t.Fatal(err) } if d.Endpoint.ID != "b" { t.Errorf("did not pick the emptiest queue: %s", d.Endpoint.ID) } if d.Scores["queue"] != 1 { t.Errorf("queue score %v, expected 1", d.Scores["queue"]) } } func TestKVCeilingAppliesHardPenalty(t *testing.T) { // A replica just above the ceiling must rank far below one just under it. s := KV(0.9) got := s.Score(newCtx(Request{}), []endpoint.Snapshot{ ep("under", kv(0.89)), ep("over", kv(0.91)), }) if got[0] <= got[1]*2 { t.Errorf("ceiling penalty too weak: under=%v over=%v", got[0], got[1]) } } func TestInflightCompensatesScrapeLag(t *testing.T) { // Both replicas report identical (stale) engine metrics, but the router has // already dispatched to one of them. The inflight scorer must notice. p := &Profile{ Name: "t", Scorers: []WeightedScorer{{Queue(4), 1}, {Inflight(8), 1}}, Picker: MaxScore(), } eps := []endpoint.Snapshot{ ep("busy", waiting(0), inflight(5)), ep("idle", waiting(0), inflight(0)), } d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err != nil { t.Fatal(err) } if d.Endpoint.ID != "idle" { t.Errorf("inflight was ignored, picked %s", d.Endpoint.ID) } } func TestOpenBreakerIsFiltered(t *testing.T) { p := &Profile{ Name: "t", Filters: []Filter{BreakerOpen()}, Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore(), } eps := []endpoint.Snapshot{ ep("open", waiting(0), func(s *endpoint.Snapshot) { s.Breaker = breaker.Open }), ep("closed", waiting(3)), } d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err != nil { t.Fatal(err) } if d.Endpoint.ID != "closed" { t.Errorf("picked an endpoint with an open circuit: %s", d.Endpoint.ID) } if d.Dropped["breaker-open"] != 1 { t.Errorf("the drop was not recorded: %v", d.Dropped) } } func TestHalfOpenBreakerSurvivesFilter(t *testing.T) { // The probe request must get through, or the circuit never closes. p := &Profile{Name: "t", Filters: []Filter{BreakerOpen()}, Picker: MaxScore()} eps := []endpoint.Snapshot{ ep("half", func(s *endpoint.Snapshot) { s.Breaker = breaker.HalfOpen }), } if _, err := p.Schedule(newCtx(Request{Model: "m"}), eps); err != nil { t.Errorf("half-open endpoint was filtered out: %v", err) } } func TestAllCandidatesFiltered(t *testing.T) { p := &Profile{Name: "t", Filters: []Filter{BreakerOpen()}, Picker: MaxScore()} eps := []endpoint.Snapshot{ ep("a", func(s *endpoint.Snapshot) { s.Breaker = breaker.Open }), } _, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err == nil { t.Fatal("expected an error") } if got := fmt.Sprint(err); got == "" { t.Error("error message is empty") } } func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) { p := &Profile{ Name: "t", Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore(), StalePenalty: 0.5, } eps := []endpoint.Snapshot{ ep("stale", waiting(0), func(s *endpoint.Snapshot) { s.Stale = true }), ep("fresh", waiting(1)), } d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err != nil { t.Fatal(err) } // stale: 1.0*0.5 = 0.5 ; fresh: 0.75 if d.Endpoint.ID != "fresh" { t.Errorf("stale endpoint was preferred: %s", d.Endpoint.ID) } // When everything is stale a choice must still be made. allStale := []endpoint.Snapshot{ ep("a", func(s *endpoint.Snapshot) { s.Stale = true }), ep("b", func(s *endpoint.Snapshot) { s.Stale = true }), } if _, err := p.Schedule(newCtx(Request{Model: "m"}), allStale); err != nil { t.Errorf("no candidates left when the whole pool is stale: %v", err) } } func TestPoolPriority(t *testing.T) { p := &Profile{ Name: "t", Filters: []Filter{PoolPriority(4, 0.9)}, Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore(), } eps := []endpoint.Snapshot{ ep("h100", waiting(3), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" }), ep("a100", waiting(0), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" }), } d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err != nil { t.Fatal(err) } // The lower-priority pool must not win for being emptier: the preferred // one is not saturated (3 waiting of a healthy 4). if d.Endpoint.ID != "h100" { t.Errorf("pool priority was ignored: %s", d.Endpoint.ID) } } func TestSessionStickiness(t *testing.T) { p := &Profile{ Name: "t", Scorers: []WeightedScorer{{Session(1.25), 3}, {Queue(4), 1}}, Picker: MaxScore(), } eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")} first, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "session-1"}), eps) if err != nil { t.Fatal(err) } for range 20 { d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "session-1"}), eps) if err != nil { t.Fatal(err) } if d.Endpoint.ID != first.Endpoint.ID { t.Fatalf("same session landed on a different replica: %s != %s", d.Endpoint.ID, first.Endpoint.ID) } } } func TestSessionsSpreadAcrossReplicas(t *testing.T) { p := &Profile{Name: "t", Scorers: []WeightedScorer{{Session(1.25), 1}}, Picker: MaxScore()} eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")} counts := map[string]int{} for i := range 300 { d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: fmt.Sprintf("s-%d", i)}), eps) if err != nil { t.Fatal(err) } counts[d.Endpoint.ID]++ } if len(counts) != 3 { t.Fatalf("sessions did not spread across all replicas: %v", counts) } for id, n := range counts { if n < 50 { t.Errorf("%s received only %d sessions; distribution too skewed: %v", id, n, counts) } } } func TestStickinessYieldsUnderOverload(t *testing.T) { // Bounded load: when the target replica is far above average, affinity must // move on. Otherwise prefix affinity drowns a single replica (plan ยง7.5). eps := []endpoint.Snapshot{ ep("a", inflight(100)), ep("b", inflight(0)), ep("c", inflight(0)), } s := Session(1.25) var chosen string for i := range 50 { got := s.Score(newCtx(Request{SessionID: fmt.Sprintf("k%d", i)}), eps) for j, v := range got { if v == 1 { chosen = eps[j].ID } } if chosen == "a" { t.Fatalf("overloaded replica (inflight=100) was chosen anyway") } } } func TestNoSessionIsNeutral(t *testing.T) { s := Session(1.25) got := s.Score(newCtx(Request{}), []endpoint.Snapshot{ep("a"), ep("b")}) for _, v := range got { if v != 0 { t.Errorf("scored without a session id: %v", got) } } } // --- picker tests --- type fixedRand struct { floats []float64 ints []int fi, ii int } func (r *fixedRand) Float64() float64 { v := r.floats[r.fi%len(r.floats)] r.fi++ return v } func (r *fixedRand) IntN(n int) int { v := r.ints[r.ii%len(r.ints)] r.ii++ return v % n } func TestP2CPicksTheBetterOfTwo(t *testing.T) { p := P2CWith(&fixedRand{ints: []int{0, 0}}) // a ve (0>=0 -> b) eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")} idx := p.Pick(newCtx(Request{}), eps, []float64{0.1, 0.9, 0.5}) if idx != 1 { t.Errorf("P2C did not pick the higher score: idx=%d", idx) } } func TestP2CNeverDrawsTheSameCandidateTwice(t *testing.T) { // When IntN(n-1) lands on a it must be bumped by one. for a := range 5 { p := &p2cPicker{rng: &fixedRand{ints: []int{a, a}}} eps := make([]endpoint.Snapshot, 5) for i := range eps { eps[i] = ep(fmt.Sprintf("e%d", i)) } scores := make([]float64, 5) scores[a] = -1 // make a worst: picking it means the same candidate was drawn twice if idx := p.Pick(newCtx(Request{}), eps, scores); idx == a { t.Errorf("same candidate drawn twice for a=%d", a) } } } func TestSoftmaxAtZeroTemperatureIsArgmax(t *testing.T) { p := SoftmaxWith(0, &fixedRand{floats: []float64{0.99}}) idx := p.Pick(newCtx(Request{}), nil, []float64{0.2, 0.8, 0.5}) if idx != 1 { t.Errorf("temperature=0 did not produce argmax: %d", idx) } } func TestSoftmaxSpreadsAtHighTemperature(t *testing.T) { // At a high temperature a low-scoring candidate must be reachable. p := SoftmaxWith(10, &fixedRand{floats: []float64{0.99}}) idx := p.Pick(newCtx(Request{}), nil, []float64{1.0, 0.0}) if idx != 1 { t.Errorf("no spread at high temperature: %d", idx) } } func TestSoftmaxDoesNotOverflow(t *testing.T) { p := SoftmaxWith(0.001, &fixedRand{floats: []float64{0.5}}) idx := p.Pick(newCtx(Request{}), nil, []float64{1000, 999}) if idx < 0 || math.IsNaN(float64(idx)) { t.Errorf("softmax broke on large scores: %d", idx) } } func TestRoundRobinCycles(t *testing.T) { p := RoundRobin() scores := []float64{0, 0, 0} got := []int{} for range 5 { got = append(got, p.Pick(newCtx(Request{}), nil, scores)) } want := []int{0, 1, 2, 0, 1} for i := range want { if got[i] != want[i] { t.Fatalf("round-robin order is wrong: %v", got) } } } func TestBuiltinProfiles(t *testing.T) { for _, name := range []string{"chat", "batch", "long-context", "debug"} { if p := Builtin(name, ProfileConfig{}); p == nil { t.Errorf("built-in profile not found: %s", name) } } if p := Builtin("no-such-profile", ProfileConfig{}); p != nil { t.Error("expected nil for an unknown profile") } } func TestBuiltinProfilesSchedule(t *testing.T) { eps := []endpoint.Snapshot{ep("a", waiting(1)), ep("b", waiting(0), kv(0.5))} for _, name := range []string{"chat", "batch", "long-context", "debug"} { p := Builtin(name, ProfileConfig{}) d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "s1"}), eps) if err != nil { t.Errorf("%s: %v", name, err) continue } if d.Endpoint.ID == "" { t.Errorf("%s: picked an empty endpoint", name) } } } func TestASaturatedPoolSpillsToTheNext(t *testing.T) { p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)}, Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()} h100 := func(w int) endpoint.Snapshot { return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" }) } a100 := func(w int) endpoint.Snapshot { return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" }) } pick := func(eps ...endpoint.Snapshot) string { d, err := p.Schedule(newCtx(Request{Model: "m"}), eps) if err != nil { t.Fatal(err) } return d.Endpoint.ID } if got := pick(h100(8), a100(0)); got != "a100" { t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got) } if got := pick(h100(8), a100(6)); got != "h100" { t.Errorf("both pools saturated: the preferred one should be kept, got %s", got) } if got := pick(h100(4), a100(0)); got != "a100" { t.Errorf("at exactly its healthy limit a pool is full: %s", got) } // KV pressure saturates a pool as queue depth does. full := h100(0) full.Stats.KVCacheUsage = 0.95 if got := pick(full, a100(0)); got != "a100" { t.Errorf("a pool out of KV cache did not spill: %s", got) } }