An awake on-demand model survives a restart of the control plane
A control plane that has just started or become leader has heard no requests, and read the on-demand replicas its nodes report as wanted by nobody: it drained every one of them at every failover. An adopted replica of an on-demand model now gets a whole idle timeout from its adoption. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b66dd9e6a7
commit
c31bac8d4b
2 files changed
+31
-1
No files matched your search
@@ -111,7 +111,14 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
|
||||
}
|
||||
r.mergeAdopted(rep, group)
|
||||
r.replicas[replicaID] = rep
|
||||
|
||||
// An on-demand model that is up was asked for, by somebody this
|
||||
// control plane has not heard of: it started before a restart or a
|
||||
// change of leader. Without this it reads as wanted by nobody and is
|
||||
// drained on the next pass — every awake on-demand model, at every
|
||||
// failover. It gets a whole idle timeout from now instead.
|
||||
if _, asked := r.wanted[m.Name]; !asked && m.Scaling.OnDemand() {
|
||||
r.wanted[m.Name] = r.now()
|
||||
}
|
||||
}
|
||||
return unclaimed
|
||||
}
|
||||
|
||||
@@ -1409,6 +1409,29 @@ func TestUseKeepsAnOnDemandModelAlive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAwakeOnDemandModelSurvivesARestartOfTheControlPlane(t *testing.T) {
|
||||
// A control plane that has just started, or just become leader, knows
|
||||
// nobody asked for anything. Taking that to mean nobody wants the
|
||||
// on-demand models a node reports running drained every one of them at
|
||||
// every failover, under whoever was using them.
|
||||
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
||||
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
||||
h.r.Reconcile(t.Context())
|
||||
h.advance(5 * time.Minute)
|
||||
h.r.Reconcile(t.Context())
|
||||
if reps := h.r.Replicas(); len(reps) != 1 || !reps[0].State.Counts() {
|
||||
t.Fatalf("the adopted replica of an on-demand model was taken down: %+v", reps)
|
||||
}
|
||||
// Still an on-demand model: unused for a whole idle timeout, it goes.
|
||||
h.advance(6 * time.Minute)
|
||||
h.r.Reconcile(t.Context())
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.State.Counts() {
|
||||
t.Errorf("an adopted replica nobody used outlived its idle timeout: %+v", rep)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelWithMaxZeroStaysOff(t *testing.T) {
|
||||
// min 0 max 0 is configured and deliberately not running, which is a
|
||||
// different and equally useful thing from "load it when wanted".
|
||||
|
||||
Reference in new issue
Block a user