A replica restarted in place keeps the room it frees for its replacement

The replica was stopped and the pass went on to the other models before
placing its replacement, so a model waiting for room took the freed cards.
On the PoC a GPU judge waiting for the only card took it from
qwen3.8-35b-a3b-distill, restarted there after its definition changed,
until the judge was stopped by hand. With the waiting model first in the
catalog it happened one pass later. The replacement is now placed in the
same step that stops the old replica.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 11:42:55 +02:00
1 parent 457a1eefac
commit 1fabb05fb2
3 files changed
+84 -2

No files matched your search

+5
View File
@@ -79,6 +79,11 @@ What was left of the field report from v0.10.6.
### Fixed
- **A replica restarted in place keeps its room**: the replica was stopped
and the pass went on to other models before placing its replacement, so a
model waiting for room took the freed cards. On the reference cluster a
GPU judge waiting for the only card took it from the model being
restarted there. The replacement is now placed in the same step.
- **The judge no longer refuses ordinary coding requests**: Llama Guard 3
1B, the judge `server init` installed, could not tell "continue", "devam
et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe)
@@ -2157,3 +2157,46 @@ func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing
t.Errorf("started again, %d ready", len(h.r.ReadyReplicas()))
}
}
func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) {
// Seen on the reference cluster: a model restarted in place on the only
// GPU, a judge waiting for that card, and the judge placed on it in the
// same pass, before the replacement — the model was gone until someone
// stopped the judge by hand.
for _, waitingFirst := range []bool{false, true} {
m := testModel(1)
waiting := testModel(1)
waiting.Name, waiting.Source = "judge", "/models/judge"
models := []catalog.Model{m, waiting}
if waitingFirst {
models = []catalog.Model{waiting, m}
}
h := newHarness(t, m, gpuNode("1", 4))
h.r.Reconcile(t.Context())
h.markReady()
old := h.r.ReadyReplicas()[0].ID
h.setModels(t, models...)
h.r.Reconcile(t.Context()) // the judge finds no room
if err := h.r.Restart(old); err != nil {
t.Fatal(err)
}
h.r.Reconcile(t.Context())
var ofModel, ofWaiting int
for _, rep := range h.r.Replicas() {
if rep.ID == old || !rep.State.Counts() {
continue
}
switch rep.Model {
case m.Name:
ofModel++
case waiting.Name:
ofWaiting++
}
}
if ofModel != 1 || ofWaiting != 0 {
t.Errorf("waiting first=%v: the freed GPUs went to %d replacement(s) and %d of the waiting model",
waitingFirst, ofModel, ofWaiting)
}
}
}
+36 -2
View File
@@ -73,11 +73,45 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string,
}
}
// stopForRestart drains a replica and stops it at once, for its replacement
// to take its place.
// stopForRestart drains a replica and stops it at once, and places its
// replacement in what it freed.
//
// At once, not on the next pass: the pass goes on to the other models, and
// one waiting for room took the freed cards first. Seen on the reference
// cluster with a judge waiting for the only GPU: the model restarted there
// lost its card to the judge.
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
model := ""
if ok {
model = rep.Model
}
r.mu.Unlock()
r.drainForRestart(ctx, id, why)
r.stopReplica(ctx, id, why)
m, found := r.cat.Load().Resolve(model)
if !ok || !found || m.Name != model {
return
}
r.mu.Lock()
var avoid []string
live := 0
for _, o := range r.replicas {
// As reconcileModel counts: one being retired is not one it has.
if o.Model == model && o.State.Counts() && o.RetiringAt.IsZero() {
live++
avoid = append(avoid, o.NodeIDs()...)
}
}
r.mu.Unlock()
if live >= r.desiredFor(m) {
return
}
if outcome := r.scaleUp(ctx, m, avoid); outcome != placed {
r.log.Warn("the replacement of a replica restarted in place was not placed in its room",
"replica", id, "model", model, "outcome", outcome.String())
}
}
// refuseRestart gives up a replica's restart, leaving it serving, and keeps