Files
llmgw/internal/router/scheduler/scheduler_test.go
T
Ilker ManapandClaude Opus 5.5 d5f02c0a4b
CI / test (push) Successful in 3m40s
Spill a saturated pool's requests to the next pool
The pool-priority filter kept the highest-priority pool however full it
was. It now takes the first pool, by priority, that is not saturated by
admission's own measure, and the preferred one when all are.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 23:23:32 +02:00

420 lines
12 KiB
Go

package scheduler
import (
"fmt"
"math"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func ep(id string, mut ...func(*endpoint.Snapshot)) endpoint.Snapshot {
s := endpoint.Snapshot{
Info: endpoint.Info{ID: id, URL: "http://" + id, Models: []string{"m"}, Weight: 1},
Breaker: breaker.Closed,
}
for _, f := range mut {
f(&s)
}
return s
}
func waiting(n int) func(*endpoint.Snapshot) {
return func(s *endpoint.Snapshot) { s.Stats.Waiting = n }
}
func kv(v float64) func(*endpoint.Snapshot) {
return func(s *endpoint.Snapshot) { s.Stats.KVCacheUsage = v }
}
func inflight(n int) func(*endpoint.Snapshot) {
return func(s *endpoint.Snapshot) { s.Inflight = n }
}
func newCtx(req Request) *Ctx { return &Ctx{Req: req, Now: time.Now()} }
func TestEmptyCandidateList(t *testing.T) {
p := Chat(ProfileConfig{})
if _, err := p.Schedule(newCtx(Request{Model: "m"}), nil); err != ErrNoEndpoints {
t.Errorf("expected ErrNoEndpoints, got %v", err)
}
}
func TestPicksShortestQueue(t *testing.T) {
p := &Profile{
Name: "t",
Scorers: []WeightedScorer{{Queue(4), 1}},
Picker: MaxScore(),
}
eps := []endpoint.Snapshot{
ep("a", waiting(3)),
ep("b", waiting(0)),
ep("c", waiting(2)),
}
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
if d.Endpoint.ID != "b" {
t.Errorf("did not pick the emptiest queue: %s", d.Endpoint.ID)
}
if d.Scores["queue"] != 1 {
t.Errorf("queue score %v, expected 1", d.Scores["queue"])
}
}
func TestKVCeilingAppliesHardPenalty(t *testing.T) {
// A replica just above the ceiling must rank far below one just under it.
s := KV(0.9)
got := s.Score(newCtx(Request{}), []endpoint.Snapshot{
ep("under", kv(0.89)),
ep("over", kv(0.91)),
})
if got[0] <= got[1]*2 {
t.Errorf("ceiling penalty too weak: under=%v over=%v", got[0], got[1])
}
}
func TestInflightCompensatesScrapeLag(t *testing.T) {
// Both replicas report identical (stale) engine metrics, but the router has
// already dispatched to one of them. The inflight scorer must notice.
p := &Profile{
Name: "t",
Scorers: []WeightedScorer{{Queue(4), 1}, {Inflight(8), 1}},
Picker: MaxScore(),
}
eps := []endpoint.Snapshot{
ep("busy", waiting(0), inflight(5)),
ep("idle", waiting(0), inflight(0)),
}
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
if d.Endpoint.ID != "idle" {
t.Errorf("inflight was ignored, picked %s", d.Endpoint.ID)
}
}
func TestOpenBreakerIsFiltered(t *testing.T) {
p := &Profile{
Name: "t",
Filters: []Filter{BreakerOpen()},
Scorers: []WeightedScorer{{Queue(4), 1}},
Picker: MaxScore(),
}
eps := []endpoint.Snapshot{
ep("open", waiting(0), func(s *endpoint.Snapshot) { s.Breaker = breaker.Open }),
ep("closed", waiting(3)),
}
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
if d.Endpoint.ID != "closed" {
t.Errorf("picked an endpoint with an open circuit: %s", d.Endpoint.ID)
}
if d.Dropped["breaker-open"] != 1 {
t.Errorf("the drop was not recorded: %v", d.Dropped)
}
}
func TestHalfOpenBreakerSurvivesFilter(t *testing.T) {
// The probe request must get through, or the circuit never closes.
p := &Profile{Name: "t", Filters: []Filter{BreakerOpen()}, Picker: MaxScore()}
eps := []endpoint.Snapshot{
ep("half", func(s *endpoint.Snapshot) { s.Breaker = breaker.HalfOpen }),
}
if _, err := p.Schedule(newCtx(Request{Model: "m"}), eps); err != nil {
t.Errorf("half-open endpoint was filtered out: %v", err)
}
}
func TestAllCandidatesFiltered(t *testing.T) {
p := &Profile{Name: "t", Filters: []Filter{BreakerOpen()}, Picker: MaxScore()}
eps := []endpoint.Snapshot{
ep("a", func(s *endpoint.Snapshot) { s.Breaker = breaker.Open }),
}
_, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err == nil {
t.Fatal("expected an error")
}
if got := fmt.Sprint(err); got == "" {
t.Error("error message is empty")
}
}
func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) {
p := &Profile{
Name: "t",
Scorers: []WeightedScorer{{Queue(4), 1}},
Picker: MaxScore(),
StalePenalty: 0.5,
}
eps := []endpoint.Snapshot{
ep("stale", waiting(0), func(s *endpoint.Snapshot) { s.Stale = true }),
ep("fresh", waiting(1)),
}
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
// stale: 1.0*0.5 = 0.5 ; fresh: 0.75
if d.Endpoint.ID != "fresh" {
t.Errorf("stale endpoint was preferred: %s", d.Endpoint.ID)
}
// When everything is stale a choice must still be made.
allStale := []endpoint.Snapshot{
ep("a", func(s *endpoint.Snapshot) { s.Stale = true }),
ep("b", func(s *endpoint.Snapshot) { s.Stale = true }),
}
if _, err := p.Schedule(newCtx(Request{Model: "m"}), allStale); err != nil {
t.Errorf("no candidates left when the whole pool is stale: %v", err)
}
}
func TestPoolPriority(t *testing.T) {
p := &Profile{
Name: "t",
Filters: []Filter{PoolPriority(4, 0.9)},
Scorers: []WeightedScorer{{Queue(4), 1}},
Picker: MaxScore(),
}
eps := []endpoint.Snapshot{
ep("h100", waiting(3), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" }),
ep("a100", waiting(0), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" }),
}
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
// The lower-priority pool must not win for being emptier: the preferred
// one is not saturated (3 waiting of a healthy 4).
if d.Endpoint.ID != "h100" {
t.Errorf("pool priority was ignored: %s", d.Endpoint.ID)
}
}
func TestSessionStickiness(t *testing.T) {
p := &Profile{
Name: "t",
Scorers: []WeightedScorer{{Session(1.25), 3}, {Queue(4), 1}},
Picker: MaxScore(),
}
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
first, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "session-1"}), eps)
if err != nil {
t.Fatal(err)
}
for range 20 {
d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "session-1"}), eps)
if err != nil {
t.Fatal(err)
}
if d.Endpoint.ID != first.Endpoint.ID {
t.Fatalf("same session landed on a different replica: %s != %s", d.Endpoint.ID, first.Endpoint.ID)
}
}
}
func TestSessionsSpreadAcrossReplicas(t *testing.T) {
p := &Profile{Name: "t", Scorers: []WeightedScorer{{Session(1.25), 1}}, Picker: MaxScore()}
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
counts := map[string]int{}
for i := range 300 {
d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: fmt.Sprintf("s-%d", i)}), eps)
if err != nil {
t.Fatal(err)
}
counts[d.Endpoint.ID]++
}
if len(counts) != 3 {
t.Fatalf("sessions did not spread across all replicas: %v", counts)
}
for id, n := range counts {
if n < 50 {
t.Errorf("%s received only %d sessions; distribution too skewed: %v", id, n, counts)
}
}
}
func TestStickinessYieldsUnderOverload(t *testing.T) {
// Bounded load: when the target replica is far above average, affinity must
// move on. Otherwise prefix affinity drowns a single replica (plan §7.5).
eps := []endpoint.Snapshot{
ep("a", inflight(100)),
ep("b", inflight(0)),
ep("c", inflight(0)),
}
s := Session(1.25)
var chosen string
for i := range 50 {
got := s.Score(newCtx(Request{SessionID: fmt.Sprintf("k%d", i)}), eps)
for j, v := range got {
if v == 1 {
chosen = eps[j].ID
}
}
if chosen == "a" {
t.Fatalf("overloaded replica (inflight=100) was chosen anyway")
}
}
}
func TestNoSessionIsNeutral(t *testing.T) {
s := Session(1.25)
got := s.Score(newCtx(Request{}), []endpoint.Snapshot{ep("a"), ep("b")})
for _, v := range got {
if v != 0 {
t.Errorf("scored without a session id: %v", got)
}
}
}
// --- picker tests ---
type fixedRand struct {
floats []float64
ints []int
fi, ii int
}
func (r *fixedRand) Float64() float64 {
v := r.floats[r.fi%len(r.floats)]
r.fi++
return v
}
func (r *fixedRand) IntN(n int) int {
v := r.ints[r.ii%len(r.ints)]
r.ii++
return v % n
}
func TestP2CPicksTheBetterOfTwo(t *testing.T) {
p := P2CWith(&fixedRand{ints: []int{0, 0}}) // a ve (0>=0 -> b)
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
idx := p.Pick(newCtx(Request{}), eps, []float64{0.1, 0.9, 0.5})
if idx != 1 {
t.Errorf("P2C did not pick the higher score: idx=%d", idx)
}
}
func TestP2CNeverDrawsTheSameCandidateTwice(t *testing.T) {
// When IntN(n-1) lands on a it must be bumped by one.
for a := range 5 {
p := &p2cPicker{rng: &fixedRand{ints: []int{a, a}}}
eps := make([]endpoint.Snapshot, 5)
for i := range eps {
eps[i] = ep(fmt.Sprintf("e%d", i))
}
scores := make([]float64, 5)
scores[a] = -1 // make a worst: picking it means the same candidate was drawn twice
if idx := p.Pick(newCtx(Request{}), eps, scores); idx == a {
t.Errorf("same candidate drawn twice for a=%d", a)
}
}
}
func TestSoftmaxAtZeroTemperatureIsArgmax(t *testing.T) {
p := SoftmaxWith(0, &fixedRand{floats: []float64{0.99}})
idx := p.Pick(newCtx(Request{}), nil, []float64{0.2, 0.8, 0.5})
if idx != 1 {
t.Errorf("temperature=0 did not produce argmax: %d", idx)
}
}
func TestSoftmaxSpreadsAtHighTemperature(t *testing.T) {
// At a high temperature a low-scoring candidate must be reachable.
p := SoftmaxWith(10, &fixedRand{floats: []float64{0.99}})
idx := p.Pick(newCtx(Request{}), nil, []float64{1.0, 0.0})
if idx != 1 {
t.Errorf("no spread at high temperature: %d", idx)
}
}
func TestSoftmaxDoesNotOverflow(t *testing.T) {
p := SoftmaxWith(0.001, &fixedRand{floats: []float64{0.5}})
idx := p.Pick(newCtx(Request{}), nil, []float64{1000, 999})
if idx < 0 || math.IsNaN(float64(idx)) {
t.Errorf("softmax broke on large scores: %d", idx)
}
}
func TestRoundRobinCycles(t *testing.T) {
p := RoundRobin()
scores := []float64{0, 0, 0}
got := []int{}
for range 5 {
got = append(got, p.Pick(newCtx(Request{}), nil, scores))
}
want := []int{0, 1, 2, 0, 1}
for i := range want {
if got[i] != want[i] {
t.Fatalf("round-robin order is wrong: %v", got)
}
}
}
func TestBuiltinProfiles(t *testing.T) {
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
if p := Builtin(name, ProfileConfig{}); p == nil {
t.Errorf("built-in profile not found: %s", name)
}
}
if p := Builtin("no-such-profile", ProfileConfig{}); p != nil {
t.Error("expected nil for an unknown profile")
}
}
func TestBuiltinProfilesSchedule(t *testing.T) {
eps := []endpoint.Snapshot{ep("a", waiting(1)), ep("b", waiting(0), kv(0.5))}
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
p := Builtin(name, ProfileConfig{})
d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "s1"}), eps)
if err != nil {
t.Errorf("%s: %v", name, err)
continue
}
if d.Endpoint.ID == "" {
t.Errorf("%s: picked an empty endpoint", name)
}
}
}
func TestASaturatedPoolSpillsToTheNext(t *testing.T) {
p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)},
Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()}
h100 := func(w int) endpoint.Snapshot {
return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" })
}
a100 := func(w int) endpoint.Snapshot {
return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" })
}
pick := func(eps ...endpoint.Snapshot) string {
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
if err != nil {
t.Fatal(err)
}
return d.Endpoint.ID
}
if got := pick(h100(8), a100(0)); got != "a100" {
t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got)
}
if got := pick(h100(8), a100(6)); got != "h100" {
t.Errorf("both pools saturated: the preferred one should be kept, got %s", got)
}
if got := pick(h100(4), a100(0)); got != "a100" {
t.Errorf("at exactly its healthy limit a pool is full: %s", got)
}
// KV pressure saturates a pool as queue depth does.
full := h100(0)
full.Stats.KVCacheUsage = 0.95
if got := pick(full, a100(0)); got != "a100" {
t.Errorf("a pool out of KV cache did not spill: %s", got)
}
}