From c31bac8d4b602c9e050ca7a27b592e8b834e4a13 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 10 Oct 2026 01:10:44 +0200 Subject: [PATCH] An awake on-demand model survives a restart of the control plane A control plane that has just started or become leader has heard no requests, and read the on-demand replicas its nodes report as wanted by nobody: it drained every one of them at every failover. An adopted replica of an on-demand model now gets a whole idle timeout from its adoption. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/control/reconciler/adopt.go | 9 +++++++- .../control/reconciler/reconciler_test.go | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/control/reconciler/adopt.go b/internal/control/reconciler/adopt.go index e3f4782..0872f8d 100644 --- a/internal/control/reconciler/adopt.go +++ b/internal/control/reconciler/adopt.go @@ -111,7 +111,14 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad } r.mergeAdopted(rep, group) r.replicas[replicaID] = rep - + // An on-demand model that is up was asked for, by somebody this + // control plane has not heard of: it started before a restart or a + // change of leader. Without this it reads as wanted by nobody and is + // drained on the next pass — every awake on-demand model, at every + // failover. It gets a whole idle timeout from now instead. + if _, asked := r.wanted[m.Name]; !asked && m.Scaling.OnDemand() { + r.wanted[m.Name] = r.now() + } } return unclaimed } diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 86f0703..f8a88a2 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1409,6 +1409,29 @@ func TestUseKeepsAnOnDemandModelAlive(t *testing.T) { } } +func TestAnAwakeOnDemandModelSurvivesARestartOfTheControlPlane(t *testing.T) { + // A control plane that has just started, or just become leader, knows + // nobody asked for anything. Taking that to mean nobody wants the + // on-demand models a node reports running drained every one of them at + // every failover, under whoever was using them. + h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8)) + h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) + h.r.Reconcile(t.Context()) + h.advance(5 * time.Minute) + h.r.Reconcile(t.Context()) + if reps := h.r.Replicas(); len(reps) != 1 || !reps[0].State.Counts() { + t.Fatalf("the adopted replica of an on-demand model was taken down: %+v", reps) + } + // Still an on-demand model: unused for a whole idle timeout, it goes. + h.advance(6 * time.Minute) + h.r.Reconcile(t.Context()) + for _, rep := range h.r.Replicas() { + if rep.State.Counts() { + t.Errorf("an adopted replica nobody used outlived its idle timeout: %+v", rep) + } + } +} + func TestAModelWithMaxZeroStaysOff(t *testing.T) { // min 0 max 0 is configured and deliberately not running, which is a // different and equally useful thing from "load it when wanted".