Files
llmgw/internal/control/reconciler/restart.go
T
Ilker ManapandClaude Opus 5.5 5ae870e1b2 ctl restart --max-surge and --max-unavailable
A model's replicas restarted one at a time, always. RestartPace says how
many restart at once: MaxSurge beside themselves (replacement first),
MaxUnavailable stopped first (no room needed, out while loading); the
reconciler begins that many per model per pass. The admin API takes
max_surge and max_unavailable on POST /api/v1/restart.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-07 13:09:10 +02:00

220 lines
7.4 KiB
Go

package reconciler
import (
"context"
"errors"
"fmt"
"slices"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// retire finishes restarting replicas.
//
// A retiring replica keeps serving until its replacement is ready, and is then
// drained like any other. When no replacement could be placed beside it, what
// happens depends on what is in the way:
//
// - the replica itself — one node, or every card that fits held: it is
// stopped now, and its replacement takes its place. Waiting would change
// nothing; it used to wait DrainGrace anyway, five minutes reading only
// "restart replacing", which looked like a restart that never would.
// - more than it: the replacement fits nowhere even with it stopped — its
// profile was edited to need more than any node has free. Stopping it
// would leave the model with nothing, so the restart is given up, and
// says why where the replica is shown.
// - something being moved out of the way (a preemption, an idle model
// stopping): waited for.
// - nothing tried this pass (a backoff): the old rule, stopped after
// DrainGrace with no replacement in sight.
//
// A restart asked for in place (ctl restart --in-place) stops first, always.
// Each stop is said in the log, since it is the one case with an outage.
func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string, readyLive, live, desired int,
outcome scaleOutcome, parked bool,
) {
for _, id := range ids {
r.mu.Lock()
rep, ok := r.replicas[id]
var since time.Time
if ok {
since = rep.RetiringAt
}
inPlace := r.restartInPlace[id]
r.mu.Unlock()
if !ok {
continue
}
switch {
case readyLive >= desired:
r.drainForRestart(ctx, id, "restarted: its replacement serves")
case inPlace:
r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id)
r.stopForRestart(ctx, id, "restarted in place, as asked")
case live >= desired:
// A replacement is placed and loading; it is waited for.
case parked:
r.refuseRestart(ctx, id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}):
r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way",
"replica", id, "model", m.Name)
r.stopForRestart(ctx, id, "restarted in place: it was the only thing in its replacement's way")
case outcome == noRoom || outcome == overLimit:
r.refuseRestart(ctx, id, "its replacement fits on no node even with this replica stopped "+
"(llmgw ctl explain-placement "+m.Name+")")
case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace:
r.log.Warn("restarting a replica in place: no replacement beside it",
"replica", id, "after", r.cfg.DrainGrace)
r.stopForRestart(ctx, id, "restarted in place: no replacement beside it within "+r.cfg.DrainGrace.String())
}
}
}
// stopForRestart drains a replica and stops it at once, for its replacement
// to take its place.
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
r.drainForRestart(ctx, id, why)
r.stopReplica(ctx, id, why)
}
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
// why for whoever looks at the replica.
func (r *Reconciler) refuseRestart(ctx context.Context, id, why string) {
r.mu.Lock()
if rep, ok := r.replicas[id]; ok {
rep.RetiringAt = time.Time{}
}
delete(r.restartInPlace, id)
r.restartRefused[id] = why
r.mu.Unlock()
r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why)
_, span := startSpan(ctx, "restart refused",
attribute.String("llmgw.replica", id), attribute.String("llmgw.reason", why))
span.End()
}
func (r *Reconciler) drainForRestart(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
var shards []Shard
if ok {
rep.State, rep.DrainingAt, rep.RetiringAt = Draining, r.now(), time.Time{}
shards = slices.Clone(rep.Shards)
}
delete(r.restartInPlace, id)
r.mu.Unlock()
if ok {
r.drainShards(ctx, id, shards, why)
}
}
// nextRestartLocked pops the next queued replica of a model that is still
// ready to be restarted. Called with mu held.
func (r *Reconciler) nextRestartLocked(model string) (string, bool) {
q := r.restartQueue[model]
for len(q) > 0 {
id := q[0]
q = q[1:]
if rep, ok := r.replicas[id]; ok && rep.State == Ready && rep.RetiringAt.IsZero() {
r.restartQueue[model] = q
return id, true
}
}
delete(r.restartQueue, model)
return "", false
}
// RestartPace is how many of a model's replicas restart at once.
type RestartPace struct {
// MaxSurge are replaced beside themselves at once: their replacements
// placed first, then they stop. Zero is one.
MaxSurge int
// MaxUnavailable are stopped first at once, their replacements starting
// in their place: an outage for each while it loads, needing no room.
MaxUnavailable int
}
// width is how many may be restarting at once.
func (p RestartPace) width() int {
surge := p.MaxSurge
if surge <= 0 && p.MaxUnavailable <= 0 {
surge = 1
}
return surge + max(p.MaxUnavailable, 0)
}
// RestartPaced queues replicas like Restart (or RestartInPlace), as many of
// each model's at once as pace says.
func (r *Reconciler) RestartPaced(pace RestartPace, inPlace bool, ids ...string) error {
if err := r.restart(inPlace, ids); err != nil {
return err
}
r.mu.Lock()
for _, id := range ids {
if rep, ok := r.replicas[id]; ok {
r.restartPace[rep.Model] = pace
}
}
r.mu.Unlock()
r.Notify()
return nil
}
// ErrNoSuchReplica means no ready replica has that id.
var ErrNoSuchReplica = errors.New("no ready replica with that id")
// Restart queues replicas to be replaced, each by a new one started beside it
// before it is stopped — to pick up an edited profile, an engine key, a new
// engine version. Replicas of one model are replaced one at a time.
func (r *Reconciler) Restart(ids ...string) error { return r.restart(false, ids) }
// RestartInPlace queues replicas to be replaced by stopping each first and
// then starting its replacement: for when there is knowingly no room beside
// it, and the outage while it loads is accepted.
func (r *Reconciler) RestartInPlace(ids ...string) error { return r.restart(true, ids) }
func (r *Reconciler) restart(inPlace bool, ids []string) error {
r.mu.Lock()
for _, id := range ids {
rep, ok := r.replicas[id]
if !ok || rep.State != Ready {
r.mu.Unlock()
return fmt.Errorf("%w: %s", ErrNoSuchReplica, id)
}
delete(r.restartRefused, id)
if inPlace {
r.restartInPlace[id] = true
}
if !slices.Contains(r.restartQueue[rep.Model], id) && rep.RetiringAt.IsZero() {
r.restartQueue[rep.Model] = append(r.restartQueue[rep.Model], id)
}
}
r.mu.Unlock()
r.Notify()
return nil
}
// RestartState says where a replica's restart stands: "queued",
// "replacing", "in place", "refused: why", or "" when none was asked for.
func (r *Reconciler) RestartState(id string) string {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[id]
switch {
case !ok:
return ""
case !rep.RetiringAt.IsZero() && r.restartInPlace[id]:
return "in place"
case !rep.RetiringAt.IsZero():
return "replacing"
case slices.Contains(r.restartQueue[rep.Model], id):
return "queued"
case r.restartRefused[id] != "":
return "refused: " + r.restartRefused[id]
}
return ""
}