package reconciler import ( "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) func measure(h *harness, byKey map[string]Measured) { h.r.SetMeasurements(func() map[string]Measured { return byKey }) } func TestMoreRequestsThanWereMeasuredToMeetTheSLOIsBusy(t *testing.T) { m := testModel(1) m.Scaling.Max = 2 h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8)) h.r.Reconcile(t.Context()) h.markReady() // Six in flight, nothing waiting at the engine: by the queue it is fine. load := map[string]Load{"llama": {Replicas: 1, Running: 6}} hold := func() { h.r.RecordLoad(load) h.advance(61 * time.Second) h.r.RecordLoad(load) h.r.Reconcile(t.Context()) } hold() if n := liveCount(h); n != 1 { t.Fatalf("scaled up with no measurement and an empty queue: %d", n) } // Measured to meet its SLO at four at once: six is past it. measure(h, map[string]Measured{"llama/tp4": {MaxConcurrency: 4, TokPerSec: 200}}) hold() if n := liveCount(h); n != 2 { t.Errorf("%d replicas with 6 in flight against a measured 4", n) } } func TestAReplicaIsTakenBackOnlyIfTheRestWouldHaveRoomToSpare(t *testing.T) { m := testModel(1) m.Scaling.Max = 2 h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8)) measure(h, map[string]Measured{"llama/tp4": {MaxConcurrency: 4, TokPerSec: 200}}) h.r.Reconcile(t.Context()) h.markReady() h.r.mu.Lock() h.r.extra["llama"] = 1 h.r.mu.Unlock() h.r.Reconcile(t.Context()) h.markReady() if liveCount(h) != 2 { t.Fatal("setup: not at two replicas") } quietFor := func(l Load) { for range 3 { h.r.RecordLoad(map[string]Load{"llama": l}) h.advance(DefaultConfig().ScaleDownAfter + time.Second) h.r.RecordLoad(map[string]Load{"llama": l}) h.r.Reconcile(t.Context()) } } // Nothing waiting, but three running: one replica left would be at 3 of // its 4, past 70%. quietFor(Load{Replicas: 2, Running: 3}) if n := liveCount(h); n != 2 { t.Errorf("scaled down to the edge of what one replica holds: %d", n) } quietFor(Load{Replicas: 2, Running: 2}) if n := liveCount(h); n != 1 { t.Errorf("two running fits one replica's 70%% of 4; still %d", n) } } func TestTheProfileMeasuredToServeMostPerGPUIsTriedFirst(t *testing.T) { m := catalog.Model{Name: "llama", Source: "/models/llama", Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{ {Name: "tp4", GPUClass: "h100-80g", TP: 4}, {Name: "tp2", GPUClass: "h100-80g", TP: 2}, {Name: "tp8", GPUClass: "h100-80g", TP: 8, Priority: 10}, }} h := newHarness(t, m, gpuNode("1", 8)) // tp4: 400/4 = 100 per GPU; tp2: 300/2 = 150. tp8 is unmeasured but has // the higher priority, which an operator set and a benchmark does not // overrule. measure(h, map[string]Measured{"llama/tp4": {MaxConcurrency: 8, TokPerSec: 400}, "llama/tp2": {MaxConcurrency: 6, TokPerSec: 300}}) cm, _ := h.r.cat.Load().Resolve("llama") var order []string for _, p := range h.r.orderedProfiles(cm) { order = append(order, p.Name) } if len(order) != 3 || order[0] != "tp8" || order[1] != "tp2" || order[2] != "tp4" { t.Errorf("order %v, expected tp8 (priority), then tp2 (150/GPU), then tp4 (100/GPU)", order) } }