The pool-priority filter kept the highest-priority pool however full it was. It now takes the first pool, by priority, that is not saturated by admission's own measure, and the preferred one when all are. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
420 lines
12 KiB
Go
420 lines
12 KiB
Go
package scheduler
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
func ep(id string, mut ...func(*endpoint.Snapshot)) endpoint.Snapshot {
|
|
s := endpoint.Snapshot{
|
|
Info: endpoint.Info{ID: id, URL: "http://" + id, Models: []string{"m"}, Weight: 1},
|
|
Breaker: breaker.Closed,
|
|
}
|
|
for _, f := range mut {
|
|
f(&s)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func waiting(n int) func(*endpoint.Snapshot) {
|
|
return func(s *endpoint.Snapshot) { s.Stats.Waiting = n }
|
|
}
|
|
func kv(v float64) func(*endpoint.Snapshot) {
|
|
return func(s *endpoint.Snapshot) { s.Stats.KVCacheUsage = v }
|
|
}
|
|
func inflight(n int) func(*endpoint.Snapshot) {
|
|
return func(s *endpoint.Snapshot) { s.Inflight = n }
|
|
}
|
|
|
|
func newCtx(req Request) *Ctx { return &Ctx{Req: req, Now: time.Now()} }
|
|
|
|
func TestEmptyCandidateList(t *testing.T) {
|
|
p := Chat(ProfileConfig{})
|
|
if _, err := p.Schedule(newCtx(Request{Model: "m"}), nil); err != ErrNoEndpoints {
|
|
t.Errorf("expected ErrNoEndpoints, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPicksShortestQueue(t *testing.T) {
|
|
p := &Profile{
|
|
Name: "t",
|
|
Scorers: []WeightedScorer{{Queue(4), 1}},
|
|
Picker: MaxScore(),
|
|
}
|
|
eps := []endpoint.Snapshot{
|
|
ep("a", waiting(3)),
|
|
ep("b", waiting(0)),
|
|
ep("c", waiting(2)),
|
|
}
|
|
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Endpoint.ID != "b" {
|
|
t.Errorf("did not pick the emptiest queue: %s", d.Endpoint.ID)
|
|
}
|
|
if d.Scores["queue"] != 1 {
|
|
t.Errorf("queue score %v, expected 1", d.Scores["queue"])
|
|
}
|
|
}
|
|
|
|
func TestKVCeilingAppliesHardPenalty(t *testing.T) {
|
|
// A replica just above the ceiling must rank far below one just under it.
|
|
s := KV(0.9)
|
|
got := s.Score(newCtx(Request{}), []endpoint.Snapshot{
|
|
ep("under", kv(0.89)),
|
|
ep("over", kv(0.91)),
|
|
})
|
|
if got[0] <= got[1]*2 {
|
|
t.Errorf("ceiling penalty too weak: under=%v over=%v", got[0], got[1])
|
|
}
|
|
}
|
|
|
|
func TestInflightCompensatesScrapeLag(t *testing.T) {
|
|
// Both replicas report identical (stale) engine metrics, but the router has
|
|
// already dispatched to one of them. The inflight scorer must notice.
|
|
p := &Profile{
|
|
Name: "t",
|
|
Scorers: []WeightedScorer{{Queue(4), 1}, {Inflight(8), 1}},
|
|
Picker: MaxScore(),
|
|
}
|
|
eps := []endpoint.Snapshot{
|
|
ep("busy", waiting(0), inflight(5)),
|
|
ep("idle", waiting(0), inflight(0)),
|
|
}
|
|
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Endpoint.ID != "idle" {
|
|
t.Errorf("inflight was ignored, picked %s", d.Endpoint.ID)
|
|
}
|
|
}
|
|
|
|
func TestOpenBreakerIsFiltered(t *testing.T) {
|
|
p := &Profile{
|
|
Name: "t",
|
|
Filters: []Filter{BreakerOpen()},
|
|
Scorers: []WeightedScorer{{Queue(4), 1}},
|
|
Picker: MaxScore(),
|
|
}
|
|
eps := []endpoint.Snapshot{
|
|
ep("open", waiting(0), func(s *endpoint.Snapshot) { s.Breaker = breaker.Open }),
|
|
ep("closed", waiting(3)),
|
|
}
|
|
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Endpoint.ID != "closed" {
|
|
t.Errorf("picked an endpoint with an open circuit: %s", d.Endpoint.ID)
|
|
}
|
|
if d.Dropped["breaker-open"] != 1 {
|
|
t.Errorf("the drop was not recorded: %v", d.Dropped)
|
|
}
|
|
}
|
|
|
|
func TestHalfOpenBreakerSurvivesFilter(t *testing.T) {
|
|
// The probe request must get through, or the circuit never closes.
|
|
p := &Profile{Name: "t", Filters: []Filter{BreakerOpen()}, Picker: MaxScore()}
|
|
eps := []endpoint.Snapshot{
|
|
ep("half", func(s *endpoint.Snapshot) { s.Breaker = breaker.HalfOpen }),
|
|
}
|
|
if _, err := p.Schedule(newCtx(Request{Model: "m"}), eps); err != nil {
|
|
t.Errorf("half-open endpoint was filtered out: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestAllCandidatesFiltered(t *testing.T) {
|
|
p := &Profile{Name: "t", Filters: []Filter{BreakerOpen()}, Picker: MaxScore()}
|
|
eps := []endpoint.Snapshot{
|
|
ep("a", func(s *endpoint.Snapshot) { s.Breaker = breaker.Open }),
|
|
}
|
|
_, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err == nil {
|
|
t.Fatal("expected an error")
|
|
}
|
|
if got := fmt.Sprint(err); got == "" {
|
|
t.Error("error message is empty")
|
|
}
|
|
}
|
|
|
|
func TestStaleEndpointIsPenalizedNotDropped(t *testing.T) {
|
|
p := &Profile{
|
|
Name: "t",
|
|
Scorers: []WeightedScorer{{Queue(4), 1}},
|
|
Picker: MaxScore(),
|
|
StalePenalty: 0.5,
|
|
}
|
|
eps := []endpoint.Snapshot{
|
|
ep("stale", waiting(0), func(s *endpoint.Snapshot) { s.Stale = true }),
|
|
ep("fresh", waiting(1)),
|
|
}
|
|
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// stale: 1.0*0.5 = 0.5 ; fresh: 0.75
|
|
if d.Endpoint.ID != "fresh" {
|
|
t.Errorf("stale endpoint was preferred: %s", d.Endpoint.ID)
|
|
}
|
|
|
|
// When everything is stale a choice must still be made.
|
|
allStale := []endpoint.Snapshot{
|
|
ep("a", func(s *endpoint.Snapshot) { s.Stale = true }),
|
|
ep("b", func(s *endpoint.Snapshot) { s.Stale = true }),
|
|
}
|
|
if _, err := p.Schedule(newCtx(Request{Model: "m"}), allStale); err != nil {
|
|
t.Errorf("no candidates left when the whole pool is stale: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPoolPriority(t *testing.T) {
|
|
p := &Profile{
|
|
Name: "t",
|
|
Filters: []Filter{PoolPriority(4, 0.9)},
|
|
Scorers: []WeightedScorer{{Queue(4), 1}},
|
|
Picker: MaxScore(),
|
|
}
|
|
eps := []endpoint.Snapshot{
|
|
ep("h100", waiting(3), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" }),
|
|
ep("a100", waiting(0), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" }),
|
|
}
|
|
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The lower-priority pool must not win for being emptier: the preferred
|
|
// one is not saturated (3 waiting of a healthy 4).
|
|
if d.Endpoint.ID != "h100" {
|
|
t.Errorf("pool priority was ignored: %s", d.Endpoint.ID)
|
|
}
|
|
}
|
|
|
|
func TestSessionStickiness(t *testing.T) {
|
|
p := &Profile{
|
|
Name: "t",
|
|
Scorers: []WeightedScorer{{Session(1.25), 3}, {Queue(4), 1}},
|
|
Picker: MaxScore(),
|
|
}
|
|
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
|
|
|
|
first, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "session-1"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for range 20 {
|
|
d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "session-1"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if d.Endpoint.ID != first.Endpoint.ID {
|
|
t.Fatalf("same session landed on a different replica: %s != %s", d.Endpoint.ID, first.Endpoint.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSessionsSpreadAcrossReplicas(t *testing.T) {
|
|
p := &Profile{Name: "t", Scorers: []WeightedScorer{{Session(1.25), 1}}, Picker: MaxScore()}
|
|
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
|
|
|
|
counts := map[string]int{}
|
|
for i := range 300 {
|
|
d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: fmt.Sprintf("s-%d", i)}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
counts[d.Endpoint.ID]++
|
|
}
|
|
if len(counts) != 3 {
|
|
t.Fatalf("sessions did not spread across all replicas: %v", counts)
|
|
}
|
|
for id, n := range counts {
|
|
if n < 50 {
|
|
t.Errorf("%s received only %d sessions; distribution too skewed: %v", id, n, counts)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestStickinessYieldsUnderOverload(t *testing.T) {
|
|
// Bounded load: when the target replica is far above average, affinity must
|
|
// move on. Otherwise prefix affinity drowns a single replica (plan §7.5).
|
|
eps := []endpoint.Snapshot{
|
|
ep("a", inflight(100)),
|
|
ep("b", inflight(0)),
|
|
ep("c", inflight(0)),
|
|
}
|
|
s := Session(1.25)
|
|
var chosen string
|
|
for i := range 50 {
|
|
got := s.Score(newCtx(Request{SessionID: fmt.Sprintf("k%d", i)}), eps)
|
|
for j, v := range got {
|
|
if v == 1 {
|
|
chosen = eps[j].ID
|
|
}
|
|
}
|
|
if chosen == "a" {
|
|
t.Fatalf("overloaded replica (inflight=100) was chosen anyway")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNoSessionIsNeutral(t *testing.T) {
|
|
s := Session(1.25)
|
|
got := s.Score(newCtx(Request{}), []endpoint.Snapshot{ep("a"), ep("b")})
|
|
for _, v := range got {
|
|
if v != 0 {
|
|
t.Errorf("scored without a session id: %v", got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- picker tests ---
|
|
|
|
type fixedRand struct {
|
|
floats []float64
|
|
ints []int
|
|
fi, ii int
|
|
}
|
|
|
|
func (r *fixedRand) Float64() float64 {
|
|
v := r.floats[r.fi%len(r.floats)]
|
|
r.fi++
|
|
return v
|
|
}
|
|
|
|
func (r *fixedRand) IntN(n int) int {
|
|
v := r.ints[r.ii%len(r.ints)]
|
|
r.ii++
|
|
return v % n
|
|
}
|
|
|
|
func TestP2CPicksTheBetterOfTwo(t *testing.T) {
|
|
p := P2CWith(&fixedRand{ints: []int{0, 0}}) // a ve (0>=0 -> b)
|
|
eps := []endpoint.Snapshot{ep("a"), ep("b"), ep("c")}
|
|
idx := p.Pick(newCtx(Request{}), eps, []float64{0.1, 0.9, 0.5})
|
|
if idx != 1 {
|
|
t.Errorf("P2C did not pick the higher score: idx=%d", idx)
|
|
}
|
|
}
|
|
|
|
func TestP2CNeverDrawsTheSameCandidateTwice(t *testing.T) {
|
|
// When IntN(n-1) lands on a it must be bumped by one.
|
|
for a := range 5 {
|
|
p := &p2cPicker{rng: &fixedRand{ints: []int{a, a}}}
|
|
eps := make([]endpoint.Snapshot, 5)
|
|
for i := range eps {
|
|
eps[i] = ep(fmt.Sprintf("e%d", i))
|
|
}
|
|
scores := make([]float64, 5)
|
|
scores[a] = -1 // make a worst: picking it means the same candidate was drawn twice
|
|
if idx := p.Pick(newCtx(Request{}), eps, scores); idx == a {
|
|
t.Errorf("same candidate drawn twice for a=%d", a)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSoftmaxAtZeroTemperatureIsArgmax(t *testing.T) {
|
|
p := SoftmaxWith(0, &fixedRand{floats: []float64{0.99}})
|
|
idx := p.Pick(newCtx(Request{}), nil, []float64{0.2, 0.8, 0.5})
|
|
if idx != 1 {
|
|
t.Errorf("temperature=0 did not produce argmax: %d", idx)
|
|
}
|
|
}
|
|
|
|
func TestSoftmaxSpreadsAtHighTemperature(t *testing.T) {
|
|
// At a high temperature a low-scoring candidate must be reachable.
|
|
p := SoftmaxWith(10, &fixedRand{floats: []float64{0.99}})
|
|
idx := p.Pick(newCtx(Request{}), nil, []float64{1.0, 0.0})
|
|
if idx != 1 {
|
|
t.Errorf("no spread at high temperature: %d", idx)
|
|
}
|
|
}
|
|
|
|
func TestSoftmaxDoesNotOverflow(t *testing.T) {
|
|
p := SoftmaxWith(0.001, &fixedRand{floats: []float64{0.5}})
|
|
idx := p.Pick(newCtx(Request{}), nil, []float64{1000, 999})
|
|
if idx < 0 || math.IsNaN(float64(idx)) {
|
|
t.Errorf("softmax broke on large scores: %d", idx)
|
|
}
|
|
}
|
|
|
|
func TestRoundRobinCycles(t *testing.T) {
|
|
p := RoundRobin()
|
|
scores := []float64{0, 0, 0}
|
|
got := []int{}
|
|
for range 5 {
|
|
got = append(got, p.Pick(newCtx(Request{}), nil, scores))
|
|
}
|
|
want := []int{0, 1, 2, 0, 1}
|
|
for i := range want {
|
|
if got[i] != want[i] {
|
|
t.Fatalf("round-robin order is wrong: %v", got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuiltinProfiles(t *testing.T) {
|
|
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
|
|
if p := Builtin(name, ProfileConfig{}); p == nil {
|
|
t.Errorf("built-in profile not found: %s", name)
|
|
}
|
|
}
|
|
if p := Builtin("no-such-profile", ProfileConfig{}); p != nil {
|
|
t.Error("expected nil for an unknown profile")
|
|
}
|
|
}
|
|
|
|
func TestBuiltinProfilesSchedule(t *testing.T) {
|
|
eps := []endpoint.Snapshot{ep("a", waiting(1)), ep("b", waiting(0), kv(0.5))}
|
|
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
|
|
p := Builtin(name, ProfileConfig{})
|
|
d, err := p.Schedule(newCtx(Request{Model: "m", SessionID: "s1"}), eps)
|
|
if err != nil {
|
|
t.Errorf("%s: %v", name, err)
|
|
continue
|
|
}
|
|
if d.Endpoint.ID == "" {
|
|
t.Errorf("%s: picked an empty endpoint", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestASaturatedPoolSpillsToTheNext(t *testing.T) {
|
|
p := &Profile{Name: "t", Filters: []Filter{PoolPriority(4, 0.9)},
|
|
Scorers: []WeightedScorer{{Queue(4), 1}}, Picker: MaxScore()}
|
|
h100 := func(w int) endpoint.Snapshot {
|
|
return ep("h100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 100; s.Pool = "ib-a/h100" })
|
|
}
|
|
a100 := func(w int) endpoint.Snapshot {
|
|
return ep("a100", waiting(w), func(s *endpoint.Snapshot) { s.Priority = 50; s.Pool = "ib-b/a100" })
|
|
}
|
|
pick := func(eps ...endpoint.Snapshot) string {
|
|
d, err := p.Schedule(newCtx(Request{Model: "m"}), eps)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return d.Endpoint.ID
|
|
}
|
|
if got := pick(h100(8), a100(0)); got != "a100" {
|
|
t.Errorf("the preferred pool at twice its healthy queue did not spill: %s", got)
|
|
}
|
|
if got := pick(h100(8), a100(6)); got != "h100" {
|
|
t.Errorf("both pools saturated: the preferred one should be kept, got %s", got)
|
|
}
|
|
if got := pick(h100(4), a100(0)); got != "a100" {
|
|
t.Errorf("at exactly its healthy limit a pool is full: %s", got)
|
|
}
|
|
// KV pressure saturates a pool as queue depth does.
|
|
full := h100(0)
|
|
full.Stats.KVCacheUsage = 0.95
|
|
if got := pick(full, a100(0)); got != "a100" {
|
|
t.Errorf("a pool out of KV cache did not spill: %s", got)
|
|
}
|
|
}
|