Ilker ManapandClaude Opus 5.5
78e41460f9
Stored ctl bench results reach the reconciler: requests per replica past the measured SLO concurrency scale a model up, a replica is taken back only if the rest stay under 70% of theirs, and among profiles of equal priority the most measured tokens per GPU or core is tried first. ctl model target sizes a model for a throughput against the room there is. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
97 lines
3.1 KiB
Go
97 lines
3.1 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
func measure(h *harness, byKey map[string]Measured) {
|
|
h.r.SetMeasurements(func() map[string]Measured { return byKey })
|
|
}
|
|
|
|
func TestMoreRequestsThanWereMeasuredToMeetTheSLOIsBusy(t *testing.T) {
|
|
m := testModel(1)
|
|
m.Scaling.Max = 2
|
|
h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
// Six in flight, nothing waiting at the engine: by the queue it is fine.
|
|
load := map[string]Load{"llama": {Replicas: 1, Running: 6}}
|
|
hold := func() {
|
|
h.r.RecordLoad(load)
|
|
h.advance(61 * time.Second)
|
|
h.r.RecordLoad(load)
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
hold()
|
|
if n := liveCount(h); n != 1 {
|
|
t.Fatalf("scaled up with no measurement and an empty queue: %d", n)
|
|
}
|
|
// Measured to meet its SLO at four at once: six is past it.
|
|
measure(h, map[string]Measured{"llama/tp4": {MaxConcurrency: 4, TokPerSec: 200}})
|
|
hold()
|
|
if n := liveCount(h); n != 2 {
|
|
t.Errorf("%d replicas with 6 in flight against a measured 4", n)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaIsTakenBackOnlyIfTheRestWouldHaveRoomToSpare(t *testing.T) {
|
|
m := testModel(1)
|
|
m.Scaling.Max = 2
|
|
h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8))
|
|
measure(h, map[string]Measured{"llama/tp4": {MaxConcurrency: 4, TokPerSec: 200}})
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.mu.Lock()
|
|
h.r.extra["llama"] = 1
|
|
h.r.mu.Unlock()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
if liveCount(h) != 2 {
|
|
t.Fatal("setup: not at two replicas")
|
|
}
|
|
quietFor := func(l Load) {
|
|
for range 3 {
|
|
h.r.RecordLoad(map[string]Load{"llama": l})
|
|
h.advance(DefaultConfig().ScaleDownAfter + time.Second)
|
|
h.r.RecordLoad(map[string]Load{"llama": l})
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
}
|
|
// Nothing waiting, but three running: one replica left would be at 3 of
|
|
// its 4, past 70%.
|
|
quietFor(Load{Replicas: 2, Running: 3})
|
|
if n := liveCount(h); n != 2 {
|
|
t.Errorf("scaled down to the edge of what one replica holds: %d", n)
|
|
}
|
|
quietFor(Load{Replicas: 2, Running: 2})
|
|
if n := liveCount(h); n != 1 {
|
|
t.Errorf("two running fits one replica's 70%% of 4; still %d", n)
|
|
}
|
|
}
|
|
|
|
func TestTheProfileMeasuredToServeMostPerGPUIsTriedFirst(t *testing.T) {
|
|
m := catalog.Model{Name: "llama", Source: "/models/llama", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
|
Profiles: []catalog.Profile{
|
|
{Name: "tp4", GPUClass: "h100-80g", TP: 4},
|
|
{Name: "tp2", GPUClass: "h100-80g", TP: 2},
|
|
{Name: "tp8", GPUClass: "h100-80g", TP: 8, Priority: 10},
|
|
}}
|
|
h := newHarness(t, m, gpuNode("1", 8))
|
|
// tp4: 400/4 = 100 per GPU; tp2: 300/2 = 150. tp8 is unmeasured but has
|
|
// the higher priority, which an operator set and a benchmark does not
|
|
// overrule.
|
|
measure(h, map[string]Measured{"llama/tp4": {MaxConcurrency: 8, TokPerSec: 400},
|
|
"llama/tp2": {MaxConcurrency: 6, TokPerSec: 300}})
|
|
cm, _ := h.r.cat.Load().Resolve("llama")
|
|
var order []string
|
|
for _, p := range h.r.orderedProfiles(cm) {
|
|
order = append(order, p.Name)
|
|
}
|
|
if len(order) != 3 || order[0] != "tp8" || order[1] != "tp2" || order[2] != "tp4" {
|
|
t.Errorf("order %v, expected tp8 (priority), then tp2 (150/GPU), then tp4 (100/GPU)", order)
|
|
}
|
|
}
|