Files
llmgw/internal/control/reconciler/restart.go
T
Ilker ManapandClaude Opus 5.5 a3ee48aced Restart in place at once when only the old replica is in the way
A restart with no room beside it waited the drain grace period before
stopping the old replica, and stopped it even when the replacement
would fit nowhere. It now stops at once when the replica alone blocks
its replacement, gives the restart up (saying why) when nothing would
fit, and ctl restart --in-place stops first on request.

Refs #2 (item 3).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 12:11:37 +02:00

181 lines
6.0 KiB
Go

package reconciler
import (
"context"
"errors"
"fmt"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// retire finishes restarting replicas.
//
// A retiring replica keeps serving until its replacement is ready, and is then
// drained like any other. When no replacement could be placed beside it, what
// happens depends on what is in the way:
//
// - the replica itself — one node, or every card that fits held: it is
// stopped now, and its replacement takes its place. Waiting would change
// nothing; it used to wait DrainGrace anyway, five minutes reading only
// "restart replacing", which looked like a restart that never would.
// - more than it: the replacement fits nowhere even with it stopped — its
// profile was edited to need more than any node has free. Stopping it
// would leave the model with nothing, so the restart is given up, and
// says why where the replica is shown.
// - something being moved out of the way (a preemption, an idle model
// stopping): waited for.
// - nothing tried this pass (a backoff): the old rule, stopped after
// DrainGrace with no replacement in sight.
//
// A restart asked for in place (ctl restart --in-place) stops first, always.
// Each stop is said in the log, since it is the one case with an outage.
func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, readyLive, live, desired int,
outcome scaleOutcome, parked bool,
) {
for _, id := range ids {
r.mu.Lock()
rep, ok := r.replicas[id]
var since time.Time
if ok {
since = rep.RetiringAt
}
inPlace := r.restartInPlace[id]
r.mu.Unlock()
if !ok {
continue
}
switch {
case readyLive >= desired:
r.drainForRestart(ctx, id)
case inPlace:
r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id)
r.stopForRestart(ctx, id)
case live >= desired:
// A replacement is placed and loading; it is waited for.
case parked:
r.refuseRestart(id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}):
r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way",
"replica", id, "model", m.Name)
r.stopForRestart(ctx, id)
case outcome == noRoom || outcome == overLimit:
r.refuseRestart(id, "its replacement fits on no node even with this replica stopped "+
"(llmgw ctl explain-placement "+m.Name+")")
case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace:
r.log.Warn("restarting a replica in place: no replacement beside it",
"replica", id, "after", r.cfg.DrainGrace)
r.stopForRestart(ctx, id)
}
}
}
// stopForRestart drains a replica and stops it at once, for its replacement
// to take its place.
func (r *Reconciler) stopForRestart(ctx context.Context, id string) {
r.drainForRestart(ctx, id)
r.stopReplica(ctx, id)
}
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
// why for whoever looks at the replica.
func (r *Reconciler) refuseRestart(id, why string) {
r.mu.Lock()
if rep, ok := r.replicas[id]; ok {
rep.RetiringAt = time.Time{}
}
delete(r.restartInPlace, id)
r.restartRefused[id] = why
r.mu.Unlock()
r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why)
}
func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
r.mu.Lock()
rep, ok := r.replicas[id]
var shards []Shard
if ok {
rep.State, rep.DrainingAt, rep.RetiringAt = Draining, r.now(), time.Time{}
shards = slices.Clone(rep.Shards)
}
delete(r.restartInPlace, id)
r.mu.Unlock()
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
// nextRestartLocked pops the next queued replica of a model that is still
// ready to be restarted. Called with mu held.
func (r *Reconciler) nextRestartLocked(model string) (string, bool) {
q := r.restartQueue[model]
for len(q) > 0 {
id := q[0]
q = q[1:]
if rep, ok := r.replicas[id]; ok && rep.State == Ready && rep.RetiringAt.IsZero() {
r.restartQueue[model] = q
return id, true
}
}
delete(r.restartQueue, model)
return "", false
}
// ErrNoSuchReplica means no ready replica has that id.
var ErrNoSuchReplica = errors.New("no ready replica with that id")
// Restart queues replicas to be replaced, each by a new one started beside it
// before it is stopped — to pick up an edited profile, an engine key, a new
// engine version. Replicas of one model are replaced one at a time.
func (r *Reconciler) Restart(ids ...string) error { return r.restart(false, ids) }
// RestartInPlace queues replicas to be replaced by stopping each first and
// then starting its replacement: for when there is knowingly no room beside
// it, and the outage while it loads is accepted.
func (r *Reconciler) RestartInPlace(ids ...string) error { return r.restart(true, ids) }
func (r *Reconciler) restart(inPlace bool, ids []string) error {
r.mu.Lock()
for _, id := range ids {
rep, ok := r.replicas[id]
if !ok || rep.State != Ready {
r.mu.Unlock()
return fmt.Errorf("%w: %s", ErrNoSuchReplica, id)
}
delete(r.restartRefused, id)
if inPlace {
r.restartInPlace[id] = true
}
if !slices.Contains(r.restartQueue[rep.Model], id) && rep.RetiringAt.IsZero() {
r.restartQueue[rep.Model] = append(r.restartQueue[rep.Model], id)
}
}
r.mu.Unlock()
r.Notify()
return nil
}
// RestartState says where a replica's restart stands: "queued",
// "replacing", "in place", "refused: why", or "" when none was asked for.
func (r *Reconciler) RestartState(id string) string {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[id]
switch {
case !ok:
return ""
case !rep.RetiringAt.IsZero() && r.restartInPlace[id]:
return "in place"
case !rep.RetiringAt.IsZero():
return "replacing"
case slices.Contains(r.restartQueue[rep.Model], id):
return "queued"
case r.restartRefused[id] != "":
return "refused: " + r.restartRefused[id]
}
return ""
}