From 0639edfd3bd3ddf30396a57081950c45f4f6001a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 11 Oct 2026 08:55:33 +0200 Subject: [PATCH] An adopted on-demand replica counts as used when adopted After a control plane restart flux, adopted and asked for by nobody, had no last use and so was never idle enough to evict: a 3D job waited out its whole fifteen minutes with no estimate. It counts as used at adoption, as it is already counted as wanted then; a replica in use is reported by the router within seconds. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 3 +++ internal/control/reconciler/adopt.go | 8 ++++++++ .../control/reconciler/reconciler_test.go | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ca1c0..068f83e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,9 @@ APIs change between minors without a deprecation period. endpoints, waited on "loading" for good; they are served only by the leader. A job told its model fits nowhere asks again for five minutes before failing, which an agent re-registering makes true for seconds. +- An on-demand replica adopted after a restart was never idle enough to + evict, so a model waiting for its card waited out its whole idle timeout; + it counts as used at adoption. ## v0.14.3 — 2026-10-10 diff --git a/internal/control/reconciler/adopt.go b/internal/control/reconciler/adopt.go index 0872f8d..4ace166 100644 --- a/internal/control/reconciler/adopt.go +++ b/internal/control/reconciler/adopt.go @@ -119,6 +119,14 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad if _, asked := r.wanted[m.Name]; !asked && m.Scaling.OnDemand() { r.wanted[m.Name] = r.now() } + // And counted as used now, for the same reason. Left unknown it was + // never idle enough to evict: a 3D job waited out the whole fifteen + // minutes of a picture model nobody had asked for since the restart. + // A replica in use is reported by the router within seconds, and + // moves this forward. + if m.Scaling.OnDemand() { + rep.LastUsedAt = r.now() + } } return unclaimed } diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 488fbeb..30898ad 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -680,6 +680,25 @@ func TestAdoptRebuildsReplica(t *testing.T) { } } +func TestAnAdoptedOnDemandReplicaNobodyUsesCanBeEvicted(t *testing.T) { + // After a restart the picture model, adopted and asked for by nobody, + // was never idle enough to evict: a 3D job waited out its whole idle + // timeout, told nothing about how long. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) + + h.r.Want("second") + h.r.Reconcile(t.Context()) + if _, eta, ok := h.r.WaitFor("second"); !ok || eta <= 0 { + t.Errorf("WaitFor: eta %s, ok %v", eta, ok) + } + h.advance(DefaultConfig().MinLifetime + time.Minute) + if got := h.r.evictable("second"); len(got) != 1 || got[0].ID != "llama-99" { + t.Errorf("evictable %+v", got) + } +} + func TestAdoptPreventsDuplicatePlacement(t *testing.T) { // The whole point: after adopting, a reconciliation pass must see the // desired count already met.