From a3ee48acedd6951718c198afd288620ce12a304f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 1 Oct 2026 12:11:37 +0200 Subject: [PATCH] Restart in place at once when only the old replica is in the way A restart with no room beside it waited the drain grace period before stopping the old replica, and stopped it even when the replacement would fit nowhere. It now stops at once when the replica alone blocks its replacement, gives the restart up (saying why) when nothing would fit, and ctl restart --in-place stops first on request. Refs #2 (item 3). Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 19 ++++ internal/cli/ctl_restart.go | 20 ++-- internal/control/adminapi/api.go | 9 +- internal/control/reconciler/reconciler.go | 35 +++++-- .../control/reconciler/reconciler_test.go | 74 +++++++++++++-- internal/control/reconciler/restart.go | 93 ++++++++++++++++--- 6 files changed, 212 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd28a976..20a37d46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,25 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### A restart with no room beside it + +`ctl restart` places each replacement beside the replica it replaces. With +no room for it — one GPU, every card that fits held by the replica itself — +the old replica was stopped only after the drain grace period, five minutes +of "restart replacing" with nothing saying why, which read as a restart that +never happened; scaling to zero and back was the way out. And a replacement +that fit nowhere even with the old replica gone was stopped for all the same, +leaving the model with nothing. Now: + +- **The replica is the only thing in the way**, found by placing the + replacement on a copy of the inventory without it: stopped at once, the + replacement taking its place. +- **The replacement fits nowhere even then**: the restart is given up and the + replica keeps serving. `ctl replicas` shows `restart refused:` and why. +- **Room being made** — a lower class preempted, an idle on-demand replica + stopping — is waited for, as before. +- **`ctl restart --in-place`** stops each replica first, whatever room there is. + ### Engines per node, and nodes that do not run them all A cluster with older GPUs could not add them cleanly: `node add` refused a diff --git a/internal/cli/ctl_restart.go b/internal/cli/ctl_restart.go index bf6cb933..59d33847 100644 --- a/internal/cli/ctl_restart.go +++ b/internal/cli/ctl_restart.go @@ -12,8 +12,9 @@ import ( func newRestartCmd(client func() *ctlClient) *cobra.Command { var ( - models []string - all bool + models []string + all bool + inPlace bool ) c := &cobra.Command{ Use: "restart [REPLICA...]", @@ -22,10 +23,14 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { "an edited profile, a new engine version.\n\n" + "Each replacement is placed beside the replica it replaces, and the old one\n" + "keeps serving until the new one is ready. Replicas of one model are\n" + - "replaced one at a time. Where nothing else has room for a second copy —\n" + - "the usual case on a small CPU fleet — the old one is stopped after the\n" + - "drain grace period (5 minutes by default) and the new one starts in its\n" + - "place; a model with only that replica is unserved while it loads.\n\n" + + "replaced one at a time.\n\n" + + "Where there is no room for a second copy and the old replica is all that\n" + + "is in the way — one GPU, every card that fits held — it is stopped at once\n" + + "and the new one starts in its place; a model with only that replica is\n" + + "unserved while it loads. Where the new one would not fit even then — a\n" + + "profile edited to need more than any node has — the restart is given up\n" + + "and the old replica keeps serving: llmgw ctl replicas says so, and why.\n\n" + + "--in-place stops each replica first, whatever room there is.\n\n" + " llmgw ctl restart qwen2.5-7b-3f2a91c0\n" + " llmgw ctl restart --model qwen2.5-7b\n" + " llmgw ctl restart --all", @@ -33,7 +38,7 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { if len(args) == 0 && len(models) == 0 && !all { return errors.New("name replicas, --model, or --all") } - body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all}) + body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace}) var res adminapi.RestartResult if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil { return err @@ -44,6 +49,7 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { } c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable") c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster") + c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before starting its replacement") return c } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 76d548e0..cfc3ebd5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -1242,6 +1242,9 @@ type RestartRequest struct { Replicas []string `json:"replicas,omitempty"` Models []string `json:"models,omitempty"` All bool `json:"all,omitempty"` + // InPlace stops each replica before starting its replacement, rather + // than starting the replacement beside it first. + InPlace bool `json:"in_place,omitempty"` } // RestartResult lists the replicas queued. @@ -1265,7 +1268,11 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "nothing ready matches; name replicas, models, or all") return } - if err := a.ctl.Reconciler().Restart(ids...); err != nil { + restart := a.ctl.Reconciler().Restart + if req.InPlace { + restart = a.ctl.Reconciler().RestartInPlace + } + if err := restart(ids...); err != nil { writeError(w, http.StatusNotFound, err.Error()) return } diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index ed10e68f..d392d70c 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -255,6 +255,11 @@ type Reconciler struct { // restarted and not yet begun. One is replaced at a time: replacing them // all at once would need the memory of every replica twice over. restartQueue map[string][]string + // restartInPlace are the replicas asked to restart by stopping first. + restartInPlace map[string]bool + // restartRefused says, by replica, why a restart was given up rather + // than leave the model without it. + restartRefused map[string]string // parked holds models that failed ParkAfter times in a row. parked map[string]parkRecord // extra is how many replicas load has added above a model's base, and @@ -304,6 +309,8 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l reservedPorts: map[int]struct{}{}, wanted: map[string]time.Time{}, restartQueue: map[string][]string{}, + restartInPlace: map[string]bool{}, + restartRefused: map[string]string{}, parked: map[string]parkRecord{}, extra: map[string]int{}, busySince: map[string]time.Time{}, @@ -729,6 +736,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { r.mu.Unlock() desired := r.desiredFor(m) + outcome := notTried switch { case len(liveIDs) < desired && isParked: // Not placed. Its failure is on every status the operator reads. @@ -742,7 +750,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { // the real error in noise. return } - r.scaleUp(ctx, m, avoid) + outcome = r.scaleUp(ctx, m, avoid) case len(liveIDs) > desired: r.scaleDown(ctx, liveIDs[desired:]) } @@ -781,7 +789,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { } } - r.retire(ctx, retiringIDs, readyLive, len(liveIDs), desired) + r.retire(ctx, m, retiringIDs, readyLive, len(liveIDs), desired, outcome, isParked) } // scaleUp places and starts one replica per pass. @@ -789,7 +797,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { // One at a time on purpose: placement decisions depend on what is already // allocated, and starting several at once from the same inventory snapshot // would hand them the same GPUs. -func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) { +func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome { profiles := r.orderedProfiles(m) res := r.sched.Place(profiles, avoid) if res.Placement == nil && len(avoid) > 0 { @@ -813,13 +821,13 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin // the pass that reaps it, and the next pass places this model — which // keeps the stop-then-start ordering in one place rather than two. if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) { - return + return makingRoom } if r.awaitingPreemption(m.Name) || r.preemptFor(ctx, m, avoid) { - return + return makingRoom } r.notePlacementFailure(m.Name, res.Explain()) - return + return noRoom } if why := r.overLimit(m, res.Placement); why != "" { // Recorded as the placement's answer, so the model reads as not @@ -829,13 +837,26 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin {Profile: res.Placement.Profile.Name, Reason: why}}} r.mu.Unlock() r.notePlacementFailure(m.Name, why) - return + return overLimit } if err := r.startReplica(ctx, m, res.Placement); err != nil { r.log.Error("starting replica failed", "model", m.Name, "err", err) + return noRoom } + return placed } +// scaleOutcome is what one scaleUp did. +type scaleOutcome int + +const ( + notTried scaleOutcome = iota // nothing was asked of placement this pass + placed // a replica was started + makingRoom // room is being made: a preemption, an idle replica stopping + noRoom // placement found nowhere, and nothing is making room + overLimit // there is room, but the owner's quota does not allow it +) + func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error { prof, ok := m.ProfileByName(pl.Profile.Name) if !ok { diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 872c4e0d..0c693049 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -114,6 +114,17 @@ type harness struct { inv *inventory.Store disp *fakeDispatcher clock *clock + live *catalog.Live +} + +// setModels replaces the catalog the reconciler works from, as an edit does. +func (h *harness) setModels(t *testing.T, models ...catalog.Model) { + t.Helper() + cat, err := catalog.New(models) + if err != nil { + t.Fatal(err) + } + h.live.Store(cat) } func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness { @@ -135,7 +146,8 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness cfg.UnreachableAfter = 15 * time.Second cfg.LostAfter = 60 * time.Second - r := New(cfg, catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler)) + live := catalog.NewLive(cat) + r := New(cfg, live, inv, disp, slog.New(slog.DiscardHandler)) r.SetClock(c.now) // Nodes were just registered, so their LastSeen is wall-clock now, which is // far from the fake clock's epoch. Re-stamp them onto the fake timeline. @@ -150,7 +162,7 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness for _, n := range nodes { touchAt(inv, n.ID, c.now()) } - return &harness{r: r, inv: inv, disp: disp, clock: c} + return &harness{r: r, inv: inv, disp: disp, clock: c, live: live} } // touchAt rewrites a node's LastSeen onto the fake clock's timeline. @@ -1787,7 +1799,9 @@ func TestARestartedReplicaServesUntilItsReplacementIsReady(t *testing.T) { func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { // One node, room for one replica: the replacement can only start once the - // old one has gone. Waiting forever would mean never restarting at all. + // old one has gone. It is the only thing in the way, so it goes now — + // waiting out a grace period first changed nothing but the wait, which + // in the field read as a restart that never happened. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.Reconcile(t.Context()) h.markReady() @@ -1796,14 +1810,8 @@ func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { t.Fatal(err) } h.r.Reconcile(t.Context()) - if rep, _ := replicaByID(h, old); rep.State != Ready { - t.Fatalf("stopped before the grace period: %s", rep.State) - } - - h.advance(h.r.cfg.DrainGrace + time.Second) - h.r.Reconcile(t.Context()) if rep, ok := replicaByID(h, old); ok && rep.State == Ready { - t.Fatal("still serving after the grace period with no replacement possible") + t.Fatal("still serving with no replacement possible beside it") } for i := 0; i < 3; i++ { h.r.Reconcile(t.Context()) @@ -1815,6 +1823,52 @@ func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { } } +func TestARestartWhoseReplacementFitsNowhereIsGivenUp(t *testing.T) { + // The profile was edited to need more cards than the node has: stopping + // the old replica would leave the model with nothing at all. + m := testModel(1) + h := newHarness(t, m, gpuNode("1", 4)) + h.r.Reconcile(t.Context()) + h.markReady() + old := h.r.ReadyReplicas()[0].ID + + m.Profiles[0].TP, m.Profiles[0].GPUsPerNode = 8, 8 + h.setModels(t, m) + if err := h.r.Restart(old); err != nil { + t.Fatal(err) + } + for range 3 { + h.r.Reconcile(t.Context()) + h.advance(h.r.cfg.DrainGrace + time.Second) + } + if rep, ok := replicaByID(h, old); !ok || rep.State != Ready { + t.Fatalf("the only replica was stopped for a replacement that fits nowhere: %+v", rep) + } + if st := h.r.RestartState(old); !strings.HasPrefix(st, "refused: ") || !strings.Contains(st, "fits on no node") { + t.Errorf("restart state %q should say it was given up, and why", st) + } + // Asked again, it is tried again. + if err := h.r.Restart(old); err != nil || h.r.RestartState(old) != "queued" { + t.Errorf("a new restart did not clear the refusal: %q, %v", h.r.RestartState(old), err) + } +} + +func TestARestartAskedInPlaceStopsFirst(t *testing.T) { + // Room beside it, but in place was asked for: an engine that must not + // run twice, or memory somebody knows is short. + h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4)) + h.r.Reconcile(t.Context()) + h.markReady() + old := h.r.ReadyReplicas()[0].ID + if err := h.r.RestartInPlace(old); err != nil { + t.Fatal(err) + } + h.r.Reconcile(t.Context()) + if rep, ok := replicaByID(h, old); ok && rep.State == Ready { + t.Fatalf("asked in place, it kept serving beside its replacement: %+v", rep) + } +} + func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) { h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) for i := 0; i < 3; i++ { diff --git a/internal/control/reconciler/restart.go b/internal/control/reconciler/restart.go index 6ff7e91e..970d7315 100644 --- a/internal/control/reconciler/restart.go +++ b/internal/control/reconciler/restart.go @@ -6,17 +6,34 @@ import ( "fmt" "slices" "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // retire finishes restarting replicas. // // A retiring replica keeps serving until its replacement is ready, and is then -// drained like any other. When no replacement could be placed at all — -// nowhere else has the memory, which on a small CPU fleet is the usual case — -// it is stopped after DrainGrace so its replacement can take its place: a -// restart in place, with the model unserved while it loads if it had no other -// replica. Said in the log, since that is the one case with an outage. -func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, desired int) { +// drained like any other. When no replacement could be placed beside it, what +// happens depends on what is in the way: +// +// - the replica itself — one node, or every card that fits held: it is +// stopped now, and its replacement takes its place. Waiting would change +// nothing; it used to wait DrainGrace anyway, five minutes reading only +// "restart replacing", which looked like a restart that never would. +// - more than it: the replacement fits nowhere even with it stopped — its +// profile was edited to need more than any node has free. Stopping it +// would leave the model with nothing, so the restart is given up, and +// says why where the replica is shown. +// - something being moved out of the way (a preemption, an idle model +// stopping): waited for. +// - nothing tried this pass (a backoff): the old rule, stopped after +// DrainGrace with no replacement in sight. +// +// A restart asked for in place (ctl restart --in-place) stops first, always. +// Each stop is said in the log, since it is the one case with an outage. +func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, readyLive, live, desired int, + outcome scaleOutcome, parked bool, +) { for _, id := range ids { r.mu.Lock() rep, ok := r.replicas[id] @@ -24,6 +41,7 @@ func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, if ok { since = rep.RetiringAt } + inPlace := r.restartInPlace[id] r.mu.Unlock() if !ok { continue @@ -31,15 +49,48 @@ func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, switch { case readyLive >= desired: r.drainForRestart(ctx, id) - case live < desired && r.now().Sub(since) > r.cfg.DrainGrace: - r.log.Warn("restarting a replica in place: no room for its replacement beside it", + case inPlace: + r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id) + r.stopForRestart(ctx, id) + case live >= desired: + // A replacement is placed and loading; it is waited for. + case parked: + r.refuseRestart(id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")") + case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}): + r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way", + "replica", id, "model", m.Name) + r.stopForRestart(ctx, id) + case outcome == noRoom || outcome == overLimit: + r.refuseRestart(id, "its replacement fits on no node even with this replica stopped "+ + "(llmgw ctl explain-placement "+m.Name+")") + case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace: + r.log.Warn("restarting a replica in place: no replacement beside it", "replica", id, "after", r.cfg.DrainGrace) - r.drainForRestart(ctx, id) - r.stopReplica(ctx, id) + r.stopForRestart(ctx, id) } } } +// stopForRestart drains a replica and stops it at once, for its replacement +// to take its place. +func (r *Reconciler) stopForRestart(ctx context.Context, id string) { + r.drainForRestart(ctx, id) + r.stopReplica(ctx, id) +} + +// refuseRestart gives up a replica's restart, leaving it serving, and keeps +// why for whoever looks at the replica. +func (r *Reconciler) refuseRestart(id, why string) { + r.mu.Lock() + if rep, ok := r.replicas[id]; ok { + rep.RetiringAt = time.Time{} + } + delete(r.restartInPlace, id) + r.restartRefused[id] = why + r.mu.Unlock() + r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why) +} + func (r *Reconciler) drainForRestart(ctx context.Context, id string) { r.mu.Lock() rep, ok := r.replicas[id] @@ -48,6 +99,7 @@ func (r *Reconciler) drainForRestart(ctx context.Context, id string) { rep.State, rep.DrainingAt, rep.RetiringAt = Draining, r.now(), time.Time{} shards = slices.Clone(rep.Shards) } + delete(r.restartInPlace, id) r.mu.Unlock() for _, s := range shards { if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil { @@ -78,7 +130,14 @@ var ErrNoSuchReplica = errors.New("no ready replica with that id") // Restart queues replicas to be replaced, each by a new one started beside it // before it is stopped — to pick up an edited profile, an engine key, a new // engine version. Replicas of one model are replaced one at a time. -func (r *Reconciler) Restart(ids ...string) error { +func (r *Reconciler) Restart(ids ...string) error { return r.restart(false, ids) } + +// RestartInPlace queues replicas to be replaced by stopping each first and +// then starting its replacement: for when there is knowingly no room beside +// it, and the outage while it loads is accepted. +func (r *Reconciler) RestartInPlace(ids ...string) error { return r.restart(true, ids) } + +func (r *Reconciler) restart(inPlace bool, ids []string) error { r.mu.Lock() for _, id := range ids { rep, ok := r.replicas[id] @@ -86,6 +145,10 @@ func (r *Reconciler) Restart(ids ...string) error { r.mu.Unlock() return fmt.Errorf("%w: %s", ErrNoSuchReplica, id) } + delete(r.restartRefused, id) + if inPlace { + r.restartInPlace[id] = true + } if !slices.Contains(r.restartQueue[rep.Model], id) && rep.RetiringAt.IsZero() { r.restartQueue[rep.Model] = append(r.restartQueue[rep.Model], id) } @@ -95,8 +158,8 @@ func (r *Reconciler) Restart(ids ...string) error { return nil } -// RestartState says where a replica's restart stands: "queued", "replacing", -// or "" when none was asked for. +// RestartState says where a replica's restart stands: "queued", +// "replacing", "in place", "refused: why", or "" when none was asked for. func (r *Reconciler) RestartState(id string) string { r.mu.Lock() defer r.mu.Unlock() @@ -104,10 +167,14 @@ func (r *Reconciler) RestartState(id string) string { switch { case !ok: return "" + case !rep.RetiringAt.IsZero() && r.restartInPlace[id]: + return "in place" case !rep.RetiringAt.IsZero(): return "replacing" case slices.Contains(r.restartQueue[rep.Model], id): return "queued" + case r.restartRefused[id] != "": + return "refused: " + r.restartRefused[id] } return "" }