From b8274d6278f5e24384c6e06151f225a9d17c09c3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 16 Oct 2026 21:33:20 +0200 Subject: [PATCH] A restart is not refused for room another restart is giving back On a 4-core single-machine install, `ctl model resources` was run on the judge (to two cores) and on the starter (to none), then both restarted. The judge's restart was refused: its replacement fit on no node even with its old replica stopped, because the starter still held its two cores. A restart now waits for other restarts under way (queued, retiring or draining) when their going would make room, up to DrainGrace. A drained replica was also kept until its model had enough live replicas, and a replica being restarted that still answers did not count. So with replicas restarted one after another, each drained one waited out the five-minute grace, holding its room. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 32 ++++++------- CHANGELOG.md | 11 +++++ internal/control/reconciler/reconciler.go | 15 +++++- .../control/reconciler/reconciler_test.go | 48 +++++++++++++++++++ internal/control/reconciler/restart.go | 28 +++++++++++ 5 files changed, 116 insertions(+), 18 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 04058fd8..4c5b39b6 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -19,8 +19,8 @@ Conventions this codebase keeps, which every item here must keep too: What is left, most needed first. The detail is in the item each names. -1. **J1** — `ctl restart --model` waits for ever on a node whose cores are - all reserved. A bug, met while testing v0.14.7. (S) +1. ~~**J1**~~ — done 2026-10-16: a restart no longer refused for room + another restart was giving back. 2. **J5** — an upgrade leaves an earlier install's judge as it was. A v0.14.6 install on fewer than eight cores refuses every request until fixed by hand. (S) @@ -1556,21 +1556,19 @@ answering 429 makes the whole fallback answer 429. Three VMs installed as HA from the setup screens, a fourth as a judge node, a fifth as a single 4-core, 8 GB machine; machines powered off under load. -### J1. A restart that has no room waits for ever (S) -`ctl restart --model M` starts each new replica beside the old one before -stopping it. On a node whose cores (or memory) are all reserved, as a -single machine's are once init divides them, the new one fits nowhere and -the restart waits with the old still running. Met on the 4-core VM: the -judge and the starter kept their old figures until their engines were -killed by hand. -- When the new replica fits nowhere beside the old, replace in place: stop - one old replica, then start its successor, saying so (the model has a - replica fewer meanwhile; with one replica it is down for the load time). -- `ctl restart` says which it is doing before it does it. -- Files: the reconciler's rolling restart (`internal/control/reconciler`), - `internal/cli/ctl_restart.go`. -- Done when: on a node with every core reserved, `ctl restart --model` - brings every replica to its new figures without anything killed by hand. +### J1. A restart refused for room another restart was giving back (S) — done 2026-10-16 +Not a wait for ever, as first written: on the 4-core VM the judge's restart +was *refused* ("fits on no node even with this replica stopped") because the +starter, restarted at the same time to reserve fewer cores, still held its +old ones. Once refused, nothing tried again, so the judges kept their old +figures. Two fixes in `internal/control/reconciler`: +- a restart whose replacement fits once other restarts under way have gone + (queued, retiring or draining) waits for them, up to DrainGrace, instead + of being refused (`fitsAfterRestarts`, restart.go); +- a drained replica counts a retiring one that still answers as serving, so + one restart after another no longer leaves the first's drained replica, + and the room it holds, waiting out its grace (reconciler.go). +`TestARestartWaitsForRoomAnotherRestartWillGiveBack` is the 4-core case. ### J2. A judge the leader adds is sized as init's is (S) When a policy names a judge the catalog lacks, the leader serves it itself diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c8f1747..b3702944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,17 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. +## Unreleased + +### Fixed + +- **A restart is no longer refused for room another restart is giving back.** With every core of a node reserved, two + models restarted together, one to need more and one less (as after `ctl model resources` on a judge and a starter), + had the first refused: "fits on no node even with this replica stopped", though the second was about to give back + its cores. A restart now waits for other restarts under way, up to the drain grace, before it is refused. A drained + replica also counts a replica being restarted that still answers as serving, so replicas restarted one after + another no longer wait out the five-minute grace each. + ## v0.14.7 — 2026-10-16 The judge installed by `server init` sized for what it does. On a machine diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 051ad86b..9abdb43c 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -915,7 +915,20 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { // A drained replica is stopped once its replacement is serving, or once the // grace period expires. Stopping it the moment it was drained would cut // in-flight requests, which is the thing draining exists to avoid. - replaced := len(liveIDs) >= desired + // + // Serving counts a replica being restarted that still answers: with + // replicas restarted one after another, the second's retiring kept the + // first's drained one waiting out its grace — and with it the second's + // replacement, on a node with no room until the first had gone. + serving := len(liveIDs) + r.mu.Lock() + for _, id := range retiringIDs { + if rep, ok := r.replicas[id]; ok && rep.State == Ready { + serving++ + } + } + r.mu.Unlock() + replaced := serving >= desired for _, id := range drainingIDs { r.mu.Lock() rep, ok := r.replicas[id] diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index dbfb0741..ce9498a5 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -2436,3 +2436,51 @@ func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) { } } } + +// Two models restarted together on a node whose cores are all reserved, +// one to need more and one less: the judge fits only once the starter +// has given back its cores. The judge's restart waits for that, rather +// than being refused as if nothing would ever make room. Met on a 4-core +// single-machine install after `ctl model resources` on both. +func TestARestartWaitsForRoomAnotherRestartWillGiveBack(t *testing.T) { + cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 16 * 1024, CPUCount: 4} + judge := catalog.Model{Name: "judge", Source: "/m", Scaling: catalog.Scaling{Min: 2, Max: 2}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 2, CPUCores: 1}}} + starter := catalog.Model{Name: "starter", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 2, CPUCores: 2}}} + h := newFleetHarness(t, []catalog.Model{judge, starter}, cpu) + h.settle(t, 3) + var old []string + for _, rep := range h.r.ReadyReplicas() { + old = append(old, rep.ID) + } + if len(old) != 3 { + t.Fatalf("%d replicas ready, want 3", len(old)) + } + + judge.Profiles[0].CPUCores, starter.Profiles[0].CPUCores = 2, 0 + cat, err := catalog.New([]catalog.Model{judge, starter}) + if err != nil { + t.Fatal(err) + } + h.r.cat.Store(cat) + if err := h.r.Restart(old...); err != nil { + t.Fatal(err) + } + for range 12 { + h.r.Reconcile(t.Context()) + h.markReady() + h.advance(10 * time.Second) + } + for _, id := range old { + if st := h.r.RestartState(id); strings.HasPrefix(st, "refused") { + t.Errorf("%s: %s", id, st) + } + if rep, ok := replicaByID(h, id); ok && rep.State == Ready { + t.Errorf("%s still serves on its old figures", id) + } + } + if j, s := replicasOf(h, "judge"), replicasOf(h, "starter"); j != 2 || s != 1 { + t.Errorf("judge %d, starter %d after the restarts", j, s) + } +} diff --git a/internal/control/reconciler/restart.go b/internal/control/reconciler/restart.go index 8097f122..0bcd5146 100644 --- a/internal/control/reconciler/restart.go +++ b/internal/control/reconciler/restart.go @@ -22,6 +22,10 @@ import ( // stopped now, and its replacement takes its place. Waiting would change // nothing; it used to wait DrainGrace anyway, five minutes reading only // "restart replacing", which looked like a restart that never would. +// - other replicas whose own restarts are under way: the replacement fits +// once they have restarted, as a starter edited to reserve fewer cores +// gives back what a judge edited to reserve more needs. Waited for, up +// to DrainGrace, so that two restarts each waiting on the other end. // - more than it: the replacement fits nowhere even with it stopped — its // profile was edited to need more than any node has free. Stopping it // would leave the model with nothing, so the restart is given up, and @@ -62,6 +66,8 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way", "replica", id, "model", m.Name) r.stopForRestart(ctx, id, "restarted in place: it was the only thing in its replacement's way") + case outcome == noRoom && r.now().Sub(since) <= r.cfg.DrainGrace && r.fitsAfterRestarts(m, id): + // Another restart under way gives the room back; waited for. case outcome == noRoom || outcome == overLimit: r.refuseRestart(ctx, id, "its replacement fits on no node even with this replica stopped "+ "(llmgw ctl explain-placement "+m.Name+")") @@ -73,6 +79,28 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, } } +// fitsAfterRestarts is whether a model's replacement for a replica fits with +// that replica gone and every other on its way out: queued to restart, being +// replaced, or draining. Their replacements take room too, so this is +// whether waiting could help, not a promise that it will. +func (r *Reconciler) fitsAfterRestarts(m catalog.Model, id string) bool { + gone := []victim{{ID: id, Model: m.Name}} + r.mu.Lock() + for oid, o := range r.replicas { + if oid == id { + continue + } + if o.State == Draining || !o.RetiringAt.IsZero() || slices.Contains(r.restartQueue[o.Model], oid) { + gone = append(gone, victim{ID: oid, Model: o.Model}) + } + } + r.mu.Unlock() + if len(gone) == 1 { + return false + } + return r.fitsWithout(m, nil, gone) +} + // stopForRestart drains a replica and stops it at once, and places its // replacement in what it freed. //