Files
llmgw/internal/control/reconciler/restart.go
T
Ilker ManapandClaude Opus 5.5 b8274d6278
CI / test (push) Successful in 8m51s
A restart is not refused for room another restart is giving back
On a 4-core single-machine install, `ctl model resources` was run on the
judge (to two cores) and on the starter (to none), then both restarted.
The judge's restart was refused: its replacement fit on no node even with
its old replica stopped, because the starter still held its two cores. A
restart now waits for other restarts under way (queued, retiring or
draining) when their going would make room, up to DrainGrace.

A drained replica was also kept until its model had enough live replicas,
and a replica being restarted that still answers did not count. So with
replicas restarted one after another, each drained one waited out the
five-minute grace, holding its room.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-16 21:33:20 +02:00

282 lines
9.5 KiB
Go

package reconciler
import (
"context"
"errors"
"fmt"
"slices"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// retire finishes restarting replicas.
//
// A retiring replica keeps serving until its replacement is ready, and is then
// drained like any other. When no replacement could be placed beside it, what
// happens depends on what is in the way:
//
// - the replica itself — one node, or every card that fits held: it is
// stopped now, and its replacement takes its place. Waiting would change
// nothing; it used to wait DrainGrace anyway, five minutes reading only
// "restart replacing", which looked like a restart that never would.
// - other replicas whose own restarts are under way: the replacement fits
// once they have restarted, as a starter edited to reserve fewer cores
// gives back what a judge edited to reserve more needs. Waited for, up
// to DrainGrace, so that two restarts each waiting on the other end.
// - more than it: the replacement fits nowhere even with it stopped — its
// profile was edited to need more than any node has free. Stopping it
// would leave the model with nothing, so the restart is given up, and
// says why where the replica is shown.
// - something being moved out of the way (a preemption, an idle model
// stopping): waited for.
// - nothing tried this pass (a backoff): the old rule, stopped after
// DrainGrace with no replacement in sight.
//
// A restart asked for in place (ctl restart --in-place) stops first, always.
// Each stop is said in the log, since it is the one case with an outage.
func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, readyLive, live, desired int,
outcome scaleOutcome, parked bool,
) {
for _, id := range ids {
r.mu.Lock()
rep, ok := r.replicas[id]
var since time.Time
if ok {
since = rep.RetiringAt
}
inPlace := r.restartInPlace[id]
r.mu.Unlock()
if !ok {
continue
}
switch {
case readyLive >= desired:
r.drainForRestart(ctx, id, "restarted: its replacement serves")
case inPlace:
r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id)
r.stopForRestart(ctx, id, "restarted in place, as asked")
case live >= desired:
// A replacement is placed and loading; it is waited for.
case parked:
r.refuseRestart(ctx, id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}):
r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way",
"replica", id, "model", m.Name)
r.stopForRestart(ctx, id, "restarted in place: it was the only thing in its replacement's way")
case outcome == noRoom && r.now().Sub(since) <= r.cfg.DrainGrace && r.fitsAfterRestarts(m, id):
// Another restart under way gives the room back; waited for.
case outcome == noRoom || outcome == overLimit:
r.refuseRestart(ctx, id, "its replacement fits on no node even with this replica stopped "+
"(llmgw ctl explain-placement "+m.Name+")")
case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace:
r.log.Warn("restarting a replica in place: no replacement beside it",
"replica", id, "after", r.cfg.DrainGrace)
r.stopForRestart(ctx, id, "restarted in place: no replacement beside it within "+r.cfg.DrainGrace.String())
}
}
}
// fitsAfterRestarts is whether a model's replacement for a replica fits with
// that replica gone and every other on its way out: queued to restart, being
// replaced, or draining. Their replacements take room too, so this is
// whether waiting could help, not a promise that it will.
func (r *Reconciler) fitsAfterRestarts(m catalog.Model, id string) bool {
gone := []victim{{ID: id, Model: m.Name}}
r.mu.Lock()
for oid, o := range r.replicas {
if oid == id {
continue
}
if o.State == Draining || !o.RetiringAt.IsZero() || slices.Contains(r.restartQueue[o.Model], oid) {
gone = append(gone, victim{ID: oid, Model: o.Model})
}
}
r.mu.Unlock()
if len(gone) == 1 {
return false
}
return r.fitsWithout(m, nil, gone)
}
// stopForRestart drains a replica and stops it at once, and places its
// replacement in what it freed.
//
// At once, not on the next pass: the pass goes on to the other models, and
// one waiting for room took the freed cards first. Seen on the reference
// cluster with a judge waiting for the only GPU: the model restarted there
// lost its card to the judge.
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
model := ""
if ok {
model = rep.Model
}
r.mu.Unlock()
r.drainForRestart(ctx, id, why)
r.stopReplica(ctx, id, why)
m, found := r.cat.Load().Resolve(model)
if !ok || !found || m.Name != model {
return
}
r.mu.Lock()
var avoid []string
live := 0
for _, o := range r.replicas {
// As reconcileModel counts: one being retired is not one it has.
if o.Model == model && o.State.Counts() && o.RetiringAt.IsZero() {
live++
avoid = append(avoid, o.NodeIDs()...)
}
}
r.mu.Unlock()
if live >= r.desiredFor(m) {
return
}
if outcome := r.scaleUp(ctx, m, avoid); outcome != placed {
r.log.Warn("the replacement of a replica restarted in place was not placed in its room",
"replica", id, "model", model, "outcome", outcome.String())
}
}
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
// why for whoever looks at the replica.
func (r *Reconciler) refuseRestart(ctx context.Context, id, why string) {
r.mu.Lock()
if rep, ok := r.replicas[id]; ok {
rep.RetiringAt = time.Time{}
}
delete(r.restartInPlace, id)
r.restartRefused[id] = why
r.mu.Unlock()
r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why)
_, span := startSpan(ctx, "restart refused",
attribute.String("llmgw.replica", id), attribute.String("llmgw.reason", why))
span.End()
}
func (r *Reconciler) drainForRestart(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
var shards []Shard
if ok {
rep.State, rep.DrainingAt, rep.RetiringAt = Draining, r.now(), time.Time{}
shards = slices.Clone(rep.Shards)
}
delete(r.restartInPlace, id)
r.mu.Unlock()
if ok {
r.drainShards(ctx, id, shards, why)
}
}
// nextRestartLocked pops the next queued replica of a model that is still
// ready to be restarted. Called with mu held.
func (r *Reconciler) nextRestartLocked(model string) (string, bool) {
q := r.restartQueue[model]
for len(q) > 0 {
id := q[0]
q = q[1:]
if rep, ok := r.replicas[id]; ok && rep.State == Ready && rep.RetiringAt.IsZero() {
r.restartQueue[model] = q
return id, true
}
}
delete(r.restartQueue, model)
return "", false
}
// RestartPace is how many of a model's replicas restart at once.
type RestartPace struct {
// MaxSurge are replaced beside themselves at once: their replacements
// placed first, then they stop. Zero is one.
MaxSurge int
// MaxUnavailable are stopped first at once, their replacements starting
// in their place: an outage for each while it loads, needing no room.
MaxUnavailable int
}
// width is how many may be restarting at once.
func (p RestartPace) width() int {
surge := p.MaxSurge
if surge <= 0 && p.MaxUnavailable <= 0 {
surge = 1
}
return surge + max(p.MaxUnavailable, 0)
}
// RestartPaced queues replicas like Restart (or RestartInPlace), as many of
// each model's at once as pace says.
func (r *Reconciler) RestartPaced(pace RestartPace, inPlace bool, ids ...string) error {
if err := r.restart(inPlace, ids); err != nil {
return err
}
r.mu.Lock()
for _, id := range ids {
if rep, ok := r.replicas[id]; ok {
r.restartPace[rep.Model] = pace
}
}
r.mu.Unlock()
r.Notify()
return nil
}
// ErrNoSuchReplica means no ready replica has that id.
var ErrNoSuchReplica = errors.New("no ready replica with that id")
// Restart queues replicas to be replaced, each by a new one started beside it
// before it is stopped — to pick up an edited profile, an engine key, a new
// engine version. Replicas of one model are replaced one at a time.
func (r *Reconciler) Restart(ids ...string) error { return r.restart(false, ids) }
// RestartInPlace queues replicas to be replaced by stopping each first and
// then starting its replacement: for when there is knowingly no room beside
// it, and the outage while it loads is accepted.
func (r *Reconciler) RestartInPlace(ids ...string) error { return r.restart(true, ids) }
func (r *Reconciler) restart(inPlace bool, ids []string) error {
r.mu.Lock()
for _, id := range ids {
rep, ok := r.replicas[id]
if !ok || rep.State != Ready {
r.mu.Unlock()
return fmt.Errorf("%w: %s", ErrNoSuchReplica, id)
}
delete(r.restartRefused, id)
if inPlace {
r.restartInPlace[id] = true
}
if !slices.Contains(r.restartQueue[rep.Model], id) && rep.RetiringAt.IsZero() {
r.restartQueue[rep.Model] = append(r.restartQueue[rep.Model], id)
}
}
r.mu.Unlock()
r.Notify()
return nil
}
// RestartState says where a replica's restart stands: "queued",
// "replacing", "in place", "refused: why", or "" when none was asked for.
func (r *Reconciler) RestartState(id string) string {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[id]
switch {
case !ok:
return ""
case !rep.RetiringAt.IsZero() && r.restartInPlace[id]:
return "in place"
case !rep.RetiringAt.IsZero():
return "replacing"
case slices.Contains(r.restartQueue[rep.Model], id):
return "queued"
case r.restartRefused[id] != "":
return "refused: " + r.restartRefused[id]
}
return ""
}