diff --git a/CHANGELOG.md b/CHANGELOG.md index dd28a97..20a37d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,25 @@ as nvidia-smi does, by PCI bus, and CUDA's own order is fastest first — on a node with two kinds of GPU, a shard could get a different card from the one it was given. +### A restart with no room beside it + +`ctl restart` places each replacement beside the replica it replaces. With +no room for it — one GPU, every card that fits held by the replica itself — +the old replica was stopped only after the drain grace period, five minutes +of "restart replacing" with nothing saying why, which read as a restart that +never happened; scaling to zero and back was the way out. And a replacement +that fit nowhere even with the old replica gone was stopped for all the same, +leaving the model with nothing. Now: + +- **The replica is the only thing in the way**, found by placing the + replacement on a copy of the inventory without it: stopped at once, the + replacement taking its place. +- **The replacement fits nowhere even then**: the restart is given up and the + replica keeps serving. `ctl replicas` shows `restart refused:` and why. +- **Room being made** — a lower class preempted, an idle on-demand replica + stopping — is waited for, as before. +- **`ctl restart --in-place`** stops each replica first, whatever room there is. + ### Engines per node, and nodes that do not run them all A cluster with older GPUs could not add them cleanly: `node add` refused a diff --git a/internal/cli/ctl_restart.go b/internal/cli/ctl_restart.go index bf6cb93..59d3384 100644 --- a/internal/cli/ctl_restart.go +++ b/internal/cli/ctl_restart.go @@ -12,8 +12,9 @@ import ( func newRestartCmd(client func() *ctlClient) *cobra.Command { var ( - models []string - all bool + models []string + all bool + inPlace bool ) c := &cobra.Command{ Use: "restart [REPLICA...]", @@ -22,10 +23,14 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { "an edited profile, a new engine version.\n\n" + "Each replacement is placed beside the replica it replaces, and the old one\n" + "keeps serving until the new one is ready. Replicas of one model are\n" + - "replaced one at a time. Where nothing else has room for a second copy —\n" + - "the usual case on a small CPU fleet — the old one is stopped after the\n" + - "drain grace period (5 minutes by default) and the new one starts in its\n" + - "place; a model with only that replica is unserved while it loads.\n\n" + + "replaced one at a time.\n\n" + + "Where there is no room for a second copy and the old replica is all that\n" + + "is in the way — one GPU, every card that fits held — it is stopped at once\n" + + "and the new one starts in its place; a model with only that replica is\n" + + "unserved while it loads. Where the new one would not fit even then — a\n" + + "profile edited to need more than any node has — the restart is given up\n" + + "and the old replica keeps serving: llmgw ctl replicas says so, and why.\n\n" + + "--in-place stops each replica first, whatever room there is.\n\n" + " llmgw ctl restart qwen2.5-7b-3f2a91c0\n" + " llmgw ctl restart --model qwen2.5-7b\n" + " llmgw ctl restart --all", @@ -33,7 +38,7 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { if len(args) == 0 && len(models) == 0 && !all { return errors.New("name replicas, --model, or --all") } - body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all}) + body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace}) var res adminapi.RestartResult if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil { return err @@ -44,6 +49,7 @@ func newRestartCmd(client func() *ctlClient) *cobra.Command { } c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable") c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster") + c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before starting its replacement") return c } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 76d548e..cfc3ebd 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -1242,6 +1242,9 @@ type RestartRequest struct { Replicas []string `json:"replicas,omitempty"` Models []string `json:"models,omitempty"` All bool `json:"all,omitempty"` + // InPlace stops each replica before starting its replacement, rather + // than starting the replacement beside it first. + InPlace bool `json:"in_place,omitempty"` } // RestartResult lists the replicas queued. @@ -1265,7 +1268,11 @@ func (a *API) restart(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "nothing ready matches; name replicas, models, or all") return } - if err := a.ctl.Reconciler().Restart(ids...); err != nil { + restart := a.ctl.Reconciler().Restart + if req.InPlace { + restart = a.ctl.Reconciler().RestartInPlace + } + if err := restart(ids...); err != nil { writeError(w, http.StatusNotFound, err.Error()) return } diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index ed10e68..d392d70 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -255,6 +255,11 @@ type Reconciler struct { // restarted and not yet begun. One is replaced at a time: replacing them // all at once would need the memory of every replica twice over. restartQueue map[string][]string + // restartInPlace are the replicas asked to restart by stopping first. + restartInPlace map[string]bool + // restartRefused says, by replica, why a restart was given up rather + // than leave the model without it. + restartRefused map[string]string // parked holds models that failed ParkAfter times in a row. parked map[string]parkRecord // extra is how many replicas load has added above a model's base, and @@ -304,6 +309,8 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l reservedPorts: map[int]struct{}{}, wanted: map[string]time.Time{}, restartQueue: map[string][]string{}, + restartInPlace: map[string]bool{}, + restartRefused: map[string]string{}, parked: map[string]parkRecord{}, extra: map[string]int{}, busySince: map[string]time.Time{}, @@ -729,6 +736,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { r.mu.Unlock() desired := r.desiredFor(m) + outcome := notTried switch { case len(liveIDs) < desired && isParked: // Not placed. Its failure is on every status the operator reads. @@ -742,7 +750,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { // the real error in noise. return } - r.scaleUp(ctx, m, avoid) + outcome = r.scaleUp(ctx, m, avoid) case len(liveIDs) > desired: r.scaleDown(ctx, liveIDs[desired:]) } @@ -781,7 +789,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { } } - r.retire(ctx, retiringIDs, readyLive, len(liveIDs), desired) + r.retire(ctx, m, retiringIDs, readyLive, len(liveIDs), desired, outcome, isParked) } // scaleUp places and starts one replica per pass. @@ -789,7 +797,7 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { // One at a time on purpose: placement decisions depend on what is already // allocated, and starting several at once from the same inventory snapshot // would hand them the same GPUs. -func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) { +func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome { profiles := r.orderedProfiles(m) res := r.sched.Place(profiles, avoid) if res.Placement == nil && len(avoid) > 0 { @@ -813,13 +821,13 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin // the pass that reaps it, and the next pass places this model — which // keeps the stop-then-start ordering in one place rather than two. if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) { - return + return makingRoom } if r.awaitingPreemption(m.Name) || r.preemptFor(ctx, m, avoid) { - return + return makingRoom } r.notePlacementFailure(m.Name, res.Explain()) - return + return noRoom } if why := r.overLimit(m, res.Placement); why != "" { // Recorded as the placement's answer, so the model reads as not @@ -829,13 +837,26 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin {Profile: res.Placement.Profile.Name, Reason: why}}} r.mu.Unlock() r.notePlacementFailure(m.Name, why) - return + return overLimit } if err := r.startReplica(ctx, m, res.Placement); err != nil { r.log.Error("starting replica failed", "model", m.Name, "err", err) + return noRoom } + return placed } +// scaleOutcome is what one scaleUp did. +type scaleOutcome int + +const ( + notTried scaleOutcome = iota // nothing was asked of placement this pass + placed // a replica was started + makingRoom // room is being made: a preemption, an idle replica stopping + noRoom // placement found nowhere, and nothing is making room + overLimit // there is room, but the owner's quota does not allow it +) + func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error { prof, ok := m.ProfileByName(pl.Profile.Name) if !ok { diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 872c4e0..0c69304 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -114,6 +114,17 @@ type harness struct { inv *inventory.Store disp *fakeDispatcher clock *clock + live *catalog.Live +} + +// setModels replaces the catalog the reconciler works from, as an edit does. +func (h *harness) setModels(t *testing.T, models ...catalog.Model) { + t.Helper() + cat, err := catalog.New(models) + if err != nil { + t.Fatal(err) + } + h.live.Store(cat) } func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness { @@ -135,7 +146,8 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness cfg.UnreachableAfter = 15 * time.Second cfg.LostAfter = 60 * time.Second - r := New(cfg, catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler)) + live := catalog.NewLive(cat) + r := New(cfg, live, inv, disp, slog.New(slog.DiscardHandler)) r.SetClock(c.now) // Nodes were just registered, so their LastSeen is wall-clock now, which is // far from the fake clock's epoch. Re-stamp them onto the fake timeline. @@ -150,7 +162,7 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness for _, n := range nodes { touchAt(inv, n.ID, c.now()) } - return &harness{r: r, inv: inv, disp: disp, clock: c} + return &harness{r: r, inv: inv, disp: disp, clock: c, live: live} } // touchAt rewrites a node's LastSeen onto the fake clock's timeline. @@ -1787,7 +1799,9 @@ func TestARestartedReplicaServesUntilItsReplacementIsReady(t *testing.T) { func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { // One node, room for one replica: the replacement can only start once the - // old one has gone. Waiting forever would mean never restarting at all. + // old one has gone. It is the only thing in the way, so it goes now — + // waiting out a grace period first changed nothing but the wait, which + // in the field read as a restart that never happened. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.Reconcile(t.Context()) h.markReady() @@ -1796,14 +1810,8 @@ func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { t.Fatal(err) } h.r.Reconcile(t.Context()) - if rep, _ := replicaByID(h, old); rep.State != Ready { - t.Fatalf("stopped before the grace period: %s", rep.State) - } - - h.advance(h.r.cfg.DrainGrace + time.Second) - h.r.Reconcile(t.Context()) if rep, ok := replicaByID(h, old); ok && rep.State == Ready { - t.Fatal("still serving after the grace period with no replacement possible") + t.Fatal("still serving with no replacement possible beside it") } for i := 0; i < 3; i++ { h.r.Reconcile(t.Context()) @@ -1815,6 +1823,52 @@ func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { } } +func TestARestartWhoseReplacementFitsNowhereIsGivenUp(t *testing.T) { + // The profile was edited to need more cards than the node has: stopping + // the old replica would leave the model with nothing at all. + m := testModel(1) + h := newHarness(t, m, gpuNode("1", 4)) + h.r.Reconcile(t.Context()) + h.markReady() + old := h.r.ReadyReplicas()[0].ID + + m.Profiles[0].TP, m.Profiles[0].GPUsPerNode = 8, 8 + h.setModels(t, m) + if err := h.r.Restart(old); err != nil { + t.Fatal(err) + } + for range 3 { + h.r.Reconcile(t.Context()) + h.advance(h.r.cfg.DrainGrace + time.Second) + } + if rep, ok := replicaByID(h, old); !ok || rep.State != Ready { + t.Fatalf("the only replica was stopped for a replacement that fits nowhere: %+v", rep) + } + if st := h.r.RestartState(old); !strings.HasPrefix(st, "refused: ") || !strings.Contains(st, "fits on no node") { + t.Errorf("restart state %q should say it was given up, and why", st) + } + // Asked again, it is tried again. + if err := h.r.Restart(old); err != nil || h.r.RestartState(old) != "queued" { + t.Errorf("a new restart did not clear the refusal: %q, %v", h.r.RestartState(old), err) + } +} + +func TestARestartAskedInPlaceStopsFirst(t *testing.T) { + // Room beside it, but in place was asked for: an engine that must not + // run twice, or memory somebody knows is short. + h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4)) + h.r.Reconcile(t.Context()) + h.markReady() + old := h.r.ReadyReplicas()[0].ID + if err := h.r.RestartInPlace(old); err != nil { + t.Fatal(err) + } + h.r.Reconcile(t.Context()) + if rep, ok := replicaByID(h, old); ok && rep.State == Ready { + t.Fatalf("asked in place, it kept serving beside its replacement: %+v", rep) + } +} + func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) { h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) for i := 0; i < 3; i++ { diff --git a/internal/control/reconciler/restart.go b/internal/control/reconciler/restart.go index 6ff7e91..970d731 100644 --- a/internal/control/reconciler/restart.go +++ b/internal/control/reconciler/restart.go @@ -6,17 +6,34 @@ import ( "fmt" "slices" "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // retire finishes restarting replicas. // // A retiring replica keeps serving until its replacement is ready, and is then -// drained like any other. When no replacement could be placed at all — -// nowhere else has the memory, which on a small CPU fleet is the usual case — -// it is stopped after DrainGrace so its replacement can take its place: a -// restart in place, with the model unserved while it loads if it had no other -// replica. Said in the log, since that is the one case with an outage. -func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, desired int) { +// drained like any other. When no replacement could be placed beside it, what +// happens depends on what is in the way: +// +// - the replica itself — one node, or every card that fits held: it is +// stopped now, and its replacement takes its place. Waiting would change +// nothing; it used to wait DrainGrace anyway, five minutes reading only +// "restart replacing", which looked like a restart that never would. +// - more than it: the replacement fits nowhere even with it stopped — its +// profile was edited to need more than any node has free. Stopping it +// would leave the model with nothing, so the restart is given up, and +// says why where the replica is shown. +// - something being moved out of the way (a preemption, an idle model +// stopping): waited for. +// - nothing tried this pass (a backoff): the old rule, stopped after +// DrainGrace with no replacement in sight. +// +// A restart asked for in place (ctl restart --in-place) stops first, always. +// Each stop is said in the log, since it is the one case with an outage. +func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, readyLive, live, desired int, + outcome scaleOutcome, parked bool, +) { for _, id := range ids { r.mu.Lock() rep, ok := r.replicas[id] @@ -24,6 +41,7 @@ func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, if ok { since = rep.RetiringAt } + inPlace := r.restartInPlace[id] r.mu.Unlock() if !ok { continue @@ -31,15 +49,48 @@ func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, switch { case readyLive >= desired: r.drainForRestart(ctx, id) - case live < desired && r.now().Sub(since) > r.cfg.DrainGrace: - r.log.Warn("restarting a replica in place: no room for its replacement beside it", + case inPlace: + r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id) + r.stopForRestart(ctx, id) + case live >= desired: + // A replacement is placed and loading; it is waited for. + case parked: + r.refuseRestart(id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")") + case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}): + r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way", + "replica", id, "model", m.Name) + r.stopForRestart(ctx, id) + case outcome == noRoom || outcome == overLimit: + r.refuseRestart(id, "its replacement fits on no node even with this replica stopped "+ + "(llmgw ctl explain-placement "+m.Name+")") + case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace: + r.log.Warn("restarting a replica in place: no replacement beside it", "replica", id, "after", r.cfg.DrainGrace) - r.drainForRestart(ctx, id) - r.stopReplica(ctx, id) + r.stopForRestart(ctx, id) } } } +// stopForRestart drains a replica and stops it at once, for its replacement +// to take its place. +func (r *Reconciler) stopForRestart(ctx context.Context, id string) { + r.drainForRestart(ctx, id) + r.stopReplica(ctx, id) +} + +// refuseRestart gives up a replica's restart, leaving it serving, and keeps +// why for whoever looks at the replica. +func (r *Reconciler) refuseRestart(id, why string) { + r.mu.Lock() + if rep, ok := r.replicas[id]; ok { + rep.RetiringAt = time.Time{} + } + delete(r.restartInPlace, id) + r.restartRefused[id] = why + r.mu.Unlock() + r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why) +} + func (r *Reconciler) drainForRestart(ctx context.Context, id string) { r.mu.Lock() rep, ok := r.replicas[id] @@ -48,6 +99,7 @@ func (r *Reconciler) drainForRestart(ctx context.Context, id string) { rep.State, rep.DrainingAt, rep.RetiringAt = Draining, r.now(), time.Time{} shards = slices.Clone(rep.Shards) } + delete(r.restartInPlace, id) r.mu.Unlock() for _, s := range shards { if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil { @@ -78,7 +130,14 @@ var ErrNoSuchReplica = errors.New("no ready replica with that id") // Restart queues replicas to be replaced, each by a new one started beside it // before it is stopped — to pick up an edited profile, an engine key, a new // engine version. Replicas of one model are replaced one at a time. -func (r *Reconciler) Restart(ids ...string) error { +func (r *Reconciler) Restart(ids ...string) error { return r.restart(false, ids) } + +// RestartInPlace queues replicas to be replaced by stopping each first and +// then starting its replacement: for when there is knowingly no room beside +// it, and the outage while it loads is accepted. +func (r *Reconciler) RestartInPlace(ids ...string) error { return r.restart(true, ids) } + +func (r *Reconciler) restart(inPlace bool, ids []string) error { r.mu.Lock() for _, id := range ids { rep, ok := r.replicas[id] @@ -86,6 +145,10 @@ func (r *Reconciler) Restart(ids ...string) error { r.mu.Unlock() return fmt.Errorf("%w: %s", ErrNoSuchReplica, id) } + delete(r.restartRefused, id) + if inPlace { + r.restartInPlace[id] = true + } if !slices.Contains(r.restartQueue[rep.Model], id) && rep.RetiringAt.IsZero() { r.restartQueue[rep.Model] = append(r.restartQueue[rep.Model], id) } @@ -95,8 +158,8 @@ func (r *Reconciler) Restart(ids ...string) error { return nil } -// RestartState says where a replica's restart stands: "queued", "replacing", -// or "" when none was asked for. +// RestartState says where a replica's restart stands: "queued", +// "replacing", "in place", "refused: why", or "" when none was asked for. func (r *Reconciler) RestartState(id string) string { r.mu.Lock() defer r.mu.Unlock() @@ -104,10 +167,14 @@ func (r *Reconciler) RestartState(id string) string { switch { case !ok: return "" + case !rep.RetiringAt.IsZero() && r.restartInPlace[id]: + return "in place" case !rep.RetiringAt.IsZero(): return "replacing" case slices.Contains(r.restartQueue[rep.Model], id): return "queued" + case r.restartRefused[id] != "": + return "refused: " + r.restartRefused[id] } return "" }