From 1fabb05fb2f7c340d9640b6e906c45cea5d79928 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 8 Oct 2026 11:42:55 +0200 Subject: [PATCH] A replica restarted in place keeps the room it frees for its replacement The replica was stopped and the pass went on to the other models before placing its replacement, so a model waiting for room took the freed cards. On the PoC a GPU judge waiting for the only card took it from qwen3.8-35b-a3b-distill, restarted there after its definition changed, until the judge was stopped by hand. With the waiting model first in the catalog it happened one pass later. The replacement is now placed in the same step that stops the old replica. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 5 +++ .../control/reconciler/reconciler_test.go | 43 +++++++++++++++++++ internal/control/reconciler/restart.go | 38 +++++++++++++++- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4769e87..d75d526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,11 @@ What was left of the field report from v0.10.6. ### Fixed +- **A replica restarted in place keeps its room**: the replica was stopped + and the pass went on to other models before placing its replacement, so a + model waiting for room took the freed cards. On the reference cluster a + GPU judge waiting for the only card took it from the model being + restarted there. The replacement is now placed in the same step. - **The judge no longer refuses ordinary coding requests**: Llama Guard 3 1B, the judge `server init` installed, could not tell "continue", "devam et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe) diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 2a8b466..86f0703 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -2157,3 +2157,46 @@ func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing t.Errorf("started again, %d ready", len(h.r.ReadyReplicas())) } } + +func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) { + // Seen on the reference cluster: a model restarted in place on the only + // GPU, a judge waiting for that card, and the judge placed on it in the + // same pass, before the replacement — the model was gone until someone + // stopped the judge by hand. + for _, waitingFirst := range []bool{false, true} { + m := testModel(1) + waiting := testModel(1) + waiting.Name, waiting.Source = "judge", "/models/judge" + models := []catalog.Model{m, waiting} + if waitingFirst { + models = []catalog.Model{waiting, m} + } + h := newHarness(t, m, gpuNode("1", 4)) + h.r.Reconcile(t.Context()) + h.markReady() + old := h.r.ReadyReplicas()[0].ID + h.setModels(t, models...) + h.r.Reconcile(t.Context()) // the judge finds no room + + if err := h.r.Restart(old); err != nil { + t.Fatal(err) + } + h.r.Reconcile(t.Context()) + var ofModel, ofWaiting int + for _, rep := range h.r.Replicas() { + if rep.ID == old || !rep.State.Counts() { + continue + } + switch rep.Model { + case m.Name: + ofModel++ + case waiting.Name: + ofWaiting++ + } + } + if ofModel != 1 || ofWaiting != 0 { + t.Errorf("waiting first=%v: the freed GPUs went to %d replacement(s) and %d of the waiting model", + waitingFirst, ofModel, ofWaiting) + } + } +} diff --git a/internal/control/reconciler/restart.go b/internal/control/reconciler/restart.go index 550cbee..8097f12 100644 --- a/internal/control/reconciler/restart.go +++ b/internal/control/reconciler/restart.go @@ -73,11 +73,45 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, } } -// stopForRestart drains a replica and stops it at once, for its replacement -// to take its place. +// stopForRestart drains a replica and stops it at once, and places its +// replacement in what it freed. +// +// At once, not on the next pass: the pass goes on to the other models, and +// one waiting for room took the freed cards first. Seen on the reference +// cluster with a judge waiting for the only GPU: the model restarted there +// lost its card to the judge. func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) { + r.mu.Lock() + rep, ok := r.replicas[id] + model := "" + if ok { + model = rep.Model + } + r.mu.Unlock() r.drainForRestart(ctx, id, why) r.stopReplica(ctx, id, why) + m, found := r.cat.Load().Resolve(model) + if !ok || !found || m.Name != model { + return + } + r.mu.Lock() + var avoid []string + live := 0 + for _, o := range r.replicas { + // As reconcileModel counts: one being retired is not one it has. + if o.Model == model && o.State.Counts() && o.RetiringAt.IsZero() { + live++ + avoid = append(avoid, o.NodeIDs()...) + } + } + r.mu.Unlock() + if live >= r.desiredFor(m) { + return + } + if outcome := r.scaleUp(ctx, m, avoid); outcome != placed { + r.log.Warn("the replacement of a replica restarted in place was not placed in its room", + "replica", id, "model", model, "outcome", outcome.String()) + } } // refuseRestart gives up a replica's restart, leaving it serving, and keeps