After a control plane restart flux, adopted and asked for by nobody, had no last use and so was never idle enough to evict: a 3D job waited out its whole fifteen minutes with no estimate. It counts as used at adoption, as it is already counted as wanted then; a replica in use is reported by the router within seconds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
95c61daf1b
commit
0639edfd3b
3 files changed
+30
No files matched your search
@@ -50,6 +50,9 @@ APIs change between minors without a deprecation period.
|
||||
endpoints, waited on "loading" for good; they are served only by the
|
||||
leader. A job told its model fits nowhere asks again for five minutes
|
||||
before failing, which an agent re-registering makes true for seconds.
|
||||
- An on-demand replica adopted after a restart was never idle enough to
|
||||
evict, so a model waiting for its card waited out its whole idle timeout;
|
||||
it counts as used at adoption.
|
||||
|
||||
## v0.14.3 — 2026-10-10
|
||||
|
||||
|
||||
@@ -119,6 +119,14 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
|
||||
if _, asked := r.wanted[m.Name]; !asked && m.Scaling.OnDemand() {
|
||||
r.wanted[m.Name] = r.now()
|
||||
}
|
||||
// And counted as used now, for the same reason. Left unknown it was
|
||||
// never idle enough to evict: a 3D job waited out the whole fifteen
|
||||
// minutes of a picture model nobody had asked for since the restart.
|
||||
// A replica in use is reported by the router within seconds, and
|
||||
// moves this forward.
|
||||
if m.Scaling.OnDemand() {
|
||||
rep.LastUsedAt = r.now()
|
||||
}
|
||||
}
|
||||
return unclaimed
|
||||
}
|
||||
|
||||
@@ -680,6 +680,25 @@ func TestAdoptRebuildsReplica(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedOnDemandReplicaNobodyUsesCanBeEvicted(t *testing.T) {
|
||||
// After a restart the picture model, adopted and asked for by nobody,
|
||||
// was never idle enough to evict: a 3D job waited out its whole idle
|
||||
// timeout, told nothing about how long.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
||||
|
||||
h.r.Want("second")
|
||||
h.r.Reconcile(t.Context())
|
||||
if _, eta, ok := h.r.WaitFor("second"); !ok || eta <= 0 {
|
||||
t.Errorf("WaitFor: eta %s, ok %v", eta, ok)
|
||||
}
|
||||
h.advance(DefaultConfig().MinLifetime + time.Minute)
|
||||
if got := h.r.evictable("second"); len(got) != 1 || got[0].ID != "llama-99" {
|
||||
t.Errorf("evictable %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
|
||||
// The whole point: after adopting, a reconciliation pass must see the
|
||||
// desired count already met.
|
||||
|
||||
Reference in new issue
Block a user