An awake on-demand model survives a restart of the control plane

A control plane that has just started or become leader has heard no
requests, and read the on-demand replicas its nodes report as wanted by
nobody: it drained every one of them at every failover. An adopted replica
of an on-demand model now gets a whole idle timeout from its adoption.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 01:10:44 +02:00
1 parent b66dd9e6a7
commit c31bac8d4b
2 files changed
+31 -1

No files matched your search

+8 -1
View File
@@ -111,7 +111,14 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
}
r.mergeAdopted(rep, group)
r.replicas[replicaID] = rep
// An on-demand model that is up was asked for, by somebody this
// control plane has not heard of: it started before a restart or a
// change of leader. Without this it reads as wanted by nobody and is
// drained on the next pass — every awake on-demand model, at every
// failover. It gets a whole idle timeout from now instead.
if _, asked := r.wanted[m.Name]; !asked && m.Scaling.OnDemand() {
r.wanted[m.Name] = r.now()
}
}
return unclaimed
}
@@ -1409,6 +1409,29 @@ func TestUseKeepsAnOnDemandModelAlive(t *testing.T) {
}
}
func TestAnAwakeOnDemandModelSurvivesARestartOfTheControlPlane(t *testing.T) {
// A control plane that has just started, or just become leader, knows
// nobody asked for anything. Taking that to mean nobody wants the
// on-demand models a node reports running drained every one of them at
// every failover, under whoever was using them.
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.r.Reconcile(t.Context())
h.advance(5 * time.Minute)
h.r.Reconcile(t.Context())
if reps := h.r.Replicas(); len(reps) != 1 || !reps[0].State.Counts() {
t.Fatalf("the adopted replica of an on-demand model was taken down: %+v", reps)
}
// Still an on-demand model: unused for a whole idle timeout, it goes.
h.advance(6 * time.Minute)
h.r.Reconcile(t.Context())
for _, rep := range h.r.Replicas() {
if rep.State.Counts() {
t.Errorf("an adopted replica nobody used outlived its idle timeout: %+v", rep)
}
}
}
func TestAModelWithMaxZeroStaysOff(t *testing.T) {
// min 0 max 0 is configured and deliberately not running, which is a
// different and equally useful thing from "load it when wanted".