A replica restarted in place keeps the room it frees for its replacement
The replica was stopped and the pass went on to the other models before placing its replacement, so a model waiting for room took the freed cards. On the PoC a GPU judge waiting for the only card took it from qwen3.8-35b-a3b-distill, restarted there after its definition changed, until the judge was stopped by hand. With the waiting model first in the catalog it happened one pass later. The replacement is now placed in the same step that stops the old replica. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
457a1eefac
commit
1fabb05fb2
3 files changed
+84
-2
No files matched your search
@@ -79,6 +79,11 @@ What was left of the field report from v0.10.6.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A replica restarted in place keeps its room**: the replica was stopped
|
||||
and the pass went on to other models before placing its replacement, so a
|
||||
model waiting for room took the freed cards. On the reference cluster a
|
||||
GPU judge waiting for the only card took it from the model being
|
||||
restarted there. The replacement is now placed in the same step.
|
||||
- **The judge no longer refuses ordinary coding requests**: Llama Guard 3
|
||||
1B, the judge `server init` installed, could not tell "continue", "devam
|
||||
et" or "how do I kill all child processes in bash" (0.51 to 0.92 unsafe)
|
||||
|
||||
@@ -2157,3 +2157,46 @@ func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing
|
||||
t.Errorf("started again, %d ready", len(h.r.ReadyReplicas()))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) {
|
||||
// Seen on the reference cluster: a model restarted in place on the only
|
||||
// GPU, a judge waiting for that card, and the judge placed on it in the
|
||||
// same pass, before the replacement — the model was gone until someone
|
||||
// stopped the judge by hand.
|
||||
for _, waitingFirst := range []bool{false, true} {
|
||||
m := testModel(1)
|
||||
waiting := testModel(1)
|
||||
waiting.Name, waiting.Source = "judge", "/models/judge"
|
||||
models := []catalog.Model{m, waiting}
|
||||
if waitingFirst {
|
||||
models = []catalog.Model{waiting, m}
|
||||
}
|
||||
h := newHarness(t, m, gpuNode("1", 4))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
old := h.r.ReadyReplicas()[0].ID
|
||||
h.setModels(t, models...)
|
||||
h.r.Reconcile(t.Context()) // the judge finds no room
|
||||
|
||||
if err := h.r.Restart(old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.r.Reconcile(t.Context())
|
||||
var ofModel, ofWaiting int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
if rep.ID == old || !rep.State.Counts() {
|
||||
continue
|
||||
}
|
||||
switch rep.Model {
|
||||
case m.Name:
|
||||
ofModel++
|
||||
case waiting.Name:
|
||||
ofWaiting++
|
||||
}
|
||||
}
|
||||
if ofModel != 1 || ofWaiting != 0 {
|
||||
t.Errorf("waiting first=%v: the freed GPUs went to %d replacement(s) and %d of the waiting model",
|
||||
waitingFirst, ofModel, ofWaiting)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -73,11 +73,45 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string,
|
||||
}
|
||||
}
|
||||
|
||||
// stopForRestart drains a replica and stops it at once, for its replacement
|
||||
// to take its place.
|
||||
// stopForRestart drains a replica and stops it at once, and places its
|
||||
// replacement in what it freed.
|
||||
//
|
||||
// At once, not on the next pass: the pass goes on to the other models, and
|
||||
// one waiting for room took the freed cards first. Seen on the reference
|
||||
// cluster with a judge waiting for the only GPU: the model restarted there
|
||||
// lost its card to the judge.
|
||||
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
model := ""
|
||||
if ok {
|
||||
model = rep.Model
|
||||
}
|
||||
r.mu.Unlock()
|
||||
r.drainForRestart(ctx, id, why)
|
||||
r.stopReplica(ctx, id, why)
|
||||
m, found := r.cat.Load().Resolve(model)
|
||||
if !ok || !found || m.Name != model {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
var avoid []string
|
||||
live := 0
|
||||
for _, o := range r.replicas {
|
||||
// As reconcileModel counts: one being retired is not one it has.
|
||||
if o.Model == model && o.State.Counts() && o.RetiringAt.IsZero() {
|
||||
live++
|
||||
avoid = append(avoid, o.NodeIDs()...)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if live >= r.desiredFor(m) {
|
||||
return
|
||||
}
|
||||
if outcome := r.scaleUp(ctx, m, avoid); outcome != placed {
|
||||
r.log.Warn("the replacement of a replica restarted in place was not placed in its room",
|
||||
"replica", id, "model", model, "outcome", outcome.String())
|
||||
}
|
||||
}
|
||||
|
||||
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
|
||||
|
||||
Reference in new issue
Block a user