Add replicas under load, up to a model's max, and take them back when quiet
A model's max never did anything: desiredFor returned the minimum, or one for an on-demand model someone was asking for, whatever the queue. Six small CPU nodes run out of slots quickly, and a model with room to grow stayed at its minimum while requests waited. The router already scrapes each engine's running and waiting requests. The control plane now reads them per model on every tick, alongside when each replica was last used, and the reconciler keeps how many replicas load has added above a model's base. One or more requests waiting per ready replica, held for a minute, adds one; ten minutes with nothing waiting removes one of those again. It never goes above max or below the base, moves one step at a time, and does not add while the last addition is still placing or starting — on this fleet a replica is minutes of weights and gigabytes of memory, and a spike should not leave a pile of replicas behind it. Decisions are logged with the load behind them. Nothing live changes until a model's max is raised above its min. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bd12b0909a
commit
adb1b3cf2a
5 files changed
+224
No files matched your search
@@ -1932,3 +1932,74 @@ func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) {
|
||||
t.Errorf("health %+v", mh)
|
||||
}
|
||||
}
|
||||
|
||||
func liveCount(h *harness) int {
|
||||
n := 0
|
||||
for _, r := range h.r.Replicas() {
|
||||
if r.State.Counts() {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestLoadAddsReplicasUpToMaxAndQuietTakesThemBack(t *testing.T) {
|
||||
m := testModel(1)
|
||||
m.Scaling.Max = 3
|
||||
h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
busy := map[string]Load{"llama": {Replicas: 1, Running: 4, Waiting: 3}}
|
||||
|
||||
// A spike shorter than ScaleUpAfter adds nothing.
|
||||
h.r.RecordLoad(busy)
|
||||
h.advance(30 * time.Second)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := liveCount(h); n != 1 {
|
||||
t.Fatalf("%d replicas after 30s of load", n)
|
||||
}
|
||||
|
||||
// Held past it, one more.
|
||||
h.advance(31 * time.Second)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := liveCount(h); n != 2 {
|
||||
t.Fatalf("%d replicas after a minute of load; expected 2", n)
|
||||
}
|
||||
|
||||
// Not another while that one is still starting.
|
||||
h.advance(2 * time.Minute)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := liveCount(h); n != 2 {
|
||||
t.Fatalf("stacked a replica on one still starting: %d", n)
|
||||
}
|
||||
|
||||
// Once it is up and load holds, the third — and never a fourth.
|
||||
h.markReady()
|
||||
for i := 0; i < 5; i++ {
|
||||
h.r.RecordLoad(busy)
|
||||
h.advance(61 * time.Second)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
if n := liveCount(h); n != 3 {
|
||||
t.Fatalf("%d replicas under sustained load; max is 3", n)
|
||||
}
|
||||
|
||||
// Quiet: back down one step per ScaleDownAfter, to min and no further.
|
||||
quiet := map[string]Load{"llama": {Replicas: 3}}
|
||||
for i := 0; i < 6; i++ {
|
||||
h.r.RecordLoad(quiet)
|
||||
h.advance(DefaultConfig().ScaleDownAfter + time.Second)
|
||||
h.r.RecordLoad(quiet)
|
||||
for j := 0; j < 3; j++ {
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
}
|
||||
if n := liveCount(h); n != 1 {
|
||||
t.Errorf("%d replicas after a long quiet spell; min is 1", n)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user