diff --git a/CHANGELOG.md b/CHANGELOG.md index 4769e87..d75d526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,11 @@ What was left of the field report from v0.10.6. ### Fixed +- **A replica restarted in place keeps its room**: the replica was stopped + and the pass went on to other models before placing its replacement, so a + model waiting for room took the freed cards. On the reference cluster a + GPU judge waiting for the only card took it from the model being + restarted there. The replacement is now placed in the same step. - **The judge no longer refuses ordinary coding requests**: Llama Guard 3 1B, the judge `server init` installed, could not tell "continue", "devam et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe) diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 2a8b466..86f0703 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -2157,3 +2157,46 @@ func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing t.Errorf("started again, %d ready", len(h.r.ReadyReplicas())) } } + +func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) { + // Seen on the reference cluster: a model restarted in place on the only + // GPU, a judge waiting for that card, and the judge placed on it in the + // same pass, before the replacement — the model was gone until someone + // stopped the judge by hand. + for _, waitingFirst := range []bool{false, true} { + m := testModel(1) + waiting := testModel(1) + waiting.Name, waiting.Source = "judge", "/models/judge" + models := []catalog.Model{m, waiting} + if waitingFirst { + models = []catalog.Model{waiting, m} + } + h := newHarness(t, m, gpuNode("1", 4)) + h.r.Reconcile(t.Context()) + h.markReady() + old := h.r.ReadyReplicas()[0].ID + h.setModels(t, models...) + h.r.Reconcile(t.Context()) // the judge finds no room + + if err := h.r.Restart(old); err != nil { + t.Fatal(err) + } + h.r.Reconcile(t.Context()) + var ofModel, ofWaiting int + for _, rep := range h.r.Replicas() { + if rep.ID == old || !rep.State.Counts() { + continue + } + switch rep.Model { + case m.Name: + ofModel++ + case waiting.Name: + ofWaiting++ + } + } + if ofModel != 1 || ofWaiting != 0 { + t.Errorf("waiting first=%v: the freed GPUs went to %d replacement(s) and %d of the waiting model", + waitingFirst, ofModel, ofWaiting) + } + } +} diff --git a/internal/control/reconciler/restart.go b/internal/control/reconciler/restart.go index 550cbee..8097f12 100644 --- a/internal/control/reconciler/restart.go +++ b/internal/control/reconciler/restart.go @@ -73,11 +73,45 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, } } -// stopForRestart drains a replica and stops it at once, for its replacement -// to take its place. +// stopForRestart drains a replica and stops it at once, and places its +// replacement in what it freed. +// +// At once, not on the next pass: the pass goes on to the other models, and +// one waiting for room took the freed cards first. Seen on the reference +// cluster with a judge waiting for the only GPU: the model restarted there +// lost its card to the judge. func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) { + r.mu.Lock() + rep, ok := r.replicas[id] + model := "" + if ok { + model = rep.Model + } + r.mu.Unlock() r.drainForRestart(ctx, id, why) r.stopReplica(ctx, id, why) + m, found := r.cat.Load().Resolve(model) + if !ok || !found || m.Name != model { + return + } + r.mu.Lock() + var avoid []string + live := 0 + for _, o := range r.replicas { + // As reconcileModel counts: one being retired is not one it has. + if o.Model == model && o.State.Counts() && o.RetiringAt.IsZero() { + live++ + avoid = append(avoid, o.NodeIDs()...) + } + } + r.mu.Unlock() + if live >= r.desiredFor(m) { + return + } + if outcome := r.scaleUp(ctx, m, avoid); outcome != placed { + r.log.Warn("the replacement of a replica restarted in place was not placed in its room", + "replica", id, "model", model, "outcome", outcome.String()) + } } // refuseRestart gives up a replica's restart, leaving it serving, and keeps