Files
llmgw/internal/control/reconciler/reconciler.go
T
Ilker ManapandClaude Opus 5.5 a3ee48aced Restart in place at once when only the old replica is in the way
A restart with no room beside it waited the drain grace period before
stopping the old replica, and stopped it even when the replacement
would fit nowhere. It now stops at once when the replica alone blocks
its replacement, gives the restart up (saying why) when nothing would
fit, and ctl restart --in-place stops first on request.

Refs #2 (item 3).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 12:11:37 +02:00

1219 lines
41 KiB
Go

package reconciler
import (
"context"
crypterand "crypto/rand"
"fmt"
"log/slog"
"math/rand/v2"
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
)
// ShardCommand is everything an agent needs to launch one shard.
type ShardCommand struct {
ReplicaID string
ShardID string
NodeID string
Spec engines.ShardSpec
// StartupTimeout comes from the profile; model loading dominates it.
StartupTimeout time.Duration
// Pool and Fabric are control plane concepts the agent cannot derive. They
// travel with the command so the agent can report them back after a
// control plane restart (plan §6.5).
Pool string
Fabric string
// ProfileFP travels for the same reason, and answers a different question:
// whether the profile has been edited since this shard was started.
ProfileFP string
}
// Dispatcher sends commands to agents.
//
// It is an interface so the reconciliation loop can be tested without a network:
// the loop's logic is where the interesting failures are, not the transport.
type Dispatcher interface {
StartShard(ctx context.Context, cmd ShardCommand) error
StopShard(ctx context.Context, nodeID, shardID string) error
DrainShard(ctx context.Context, nodeID, shardID string) error
}
// Config tunes the reconciliation loop.
type Config struct {
// Interval is how often desired and actual are compared when nothing else
// happens. Events wake the loop sooner.
Interval time.Duration
// BaseBackoff is how long to wait before rebuilding a failed replica.
BaseBackoff time.Duration
// MaxBackoff caps the exponential growth.
MaxBackoff time.Duration
// UnreachableAfter is how long without a heartbeat before a node is
// unreachable.
UnreachableAfter time.Duration
// LostAfter is how long unreachable before its replicas are declared dead.
// Longer than UnreachableAfter on purpose: a brief network blip must not
// tear down a replica that is still serving perfectly well.
LostAfter time.Duration
// DrainGrace bounds how long a draining replica waits for its replacement.
// Without a bound, draining a node whose model cannot be placed anywhere
// else would never finish.
DrainGrace time.Duration
// PreemptGrace is how long a replica stopped to make room for another
// is given to finish what it is serving: a request, not a replacement.
PreemptGrace time.Duration
// PreemptHold is how long a model that gave up room may not take it back,
// so the replica it was given up for is placed first.
PreemptHold time.Duration
// ScaleUpWaiting is the average number of requests waiting per ready
// replica, held for ScaleUpAfter, that adds a replica — up to the model's
// max. ScaleDownAfter is how long with nothing waiting before one of the
// replicas added that way is taken away again, down to its min.
ScaleUpWaiting float64
ScaleUpAfter time.Duration
ScaleDownAfter time.Duration
// ParkAfter is how many failures in a row stop a model being retried.
//
// Backoff alone never gives up: it grows to MaxBackoff and keeps trying,
// and a model whose weights its engine cannot load — an architecture the
// installed llama.cpp does not know — was started 155 times over eight
// hours on the live cluster, each time failing the same way, with nothing
// to show for it but the log. A parked model is not placed again until its
// definition changes or an operator runs `llmgw ctl retry`.
ParkAfter int
// PortRange is the per-node port range for engine HTTP and dist-init.
PortRange [2]int
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
MgmtInterface string
// EngineKey gives each replica the key its engine demands. Derived from
// the replica id rather than stored, so a control plane that restarts —
// or a second one — computes the same key for a replica it adopts. Nil
// starts engines without one.
EngineKey func(replicaID string) string
// EvictionGrace is how long a replica must have gone unasked-for before it
// may be stopped to make room for another model.
//
// Shorter than IdleTimeout on purpose: a replica idle for five minutes is
// not worth reclaiming on its own, and is worth reclaiming when something
// else needs the cards. Still minutes rather than seconds, because stopping
// one throws away the KV cache prefix routing spent filling.
EvictionGrace time.Duration
// MinLifetime is how long a replica is safe from eviction after it starts
// serving, whatever else wants its cards.
//
// The thing that stops two models on a cluster with room for one from
// taking turns evicting each other. A load costs minutes; oscillating at a
// load per cycle is worse for both models than one of them waiting.
MinLifetime time.Duration
// IdleTimeout is how long an on-demand model stays loaded with nobody
// asking for it. A model may override it.
IdleTimeout time.Duration
// SettleFor is how long after start to leave placement alone while agents
// reconnect.
//
// A control plane that has just started knows nothing, and the first
// reconcile tick runs before any agent has registered. It sees an empty
// fleet and acts on it: "needs 2 nodes on one fabric, but no node reports a
// fabric" is that, and it is harmless only because adoption catches up.
// What is not harmless is placing on the fraction of the fleet that
// happens to have reported first — after one restart both replicas of a
// model landed on the single node that had registered, when four were
// coming.
//
// Adoption still runs throughout; only placement waits. The window ends
// early once the fleet stops growing, so a small cluster does not sit idle
// for the whole duration — but not too early: an agent that lost its
// connection reconnects on a backoff reaching thirty seconds, so "no new
// node for a few seconds" means nothing.
SettleFor time.Duration
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Interval: 5 * time.Second,
BaseBackoff: 10 * time.Second,
MaxBackoff: 10 * time.Minute,
UnreachableAfter: 15 * time.Second,
LostAfter: 60 * time.Second,
SettleFor: 60 * time.Second,
IdleTimeout: 15 * time.Minute,
EvictionGrace: 2 * time.Minute,
MinLifetime: 5 * time.Minute,
DrainGrace: 5 * time.Minute,
PreemptGrace: 30 * time.Second,
PreemptHold: 5 * time.Minute,
ParkAfter: 5,
ScaleUpWaiting: 1,
ScaleUpAfter: time.Minute,
ScaleDownAfter: 10 * time.Minute,
PortRange: [2]int{20000, 20999},
MgmtInterface: "eth0",
}
}
func (c *Config) normalize() {
d := DefaultConfig()
if c.Interval <= 0 {
c.Interval = d.Interval
}
if c.BaseBackoff <= 0 {
c.BaseBackoff = d.BaseBackoff
}
if c.PreemptGrace <= 0 {
c.PreemptGrace = d.PreemptGrace
}
if c.PreemptHold <= 0 {
c.PreemptHold = d.PreemptHold
}
if c.MaxBackoff < c.BaseBackoff {
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
}
if c.IdleTimeout <= 0 {
c.IdleTimeout = d.IdleTimeout
}
if c.EvictionGrace <= 0 {
c.EvictionGrace = d.EvictionGrace
}
if c.MinLifetime <= 0 {
c.MinLifetime = d.MinLifetime
}
if c.SettleFor < 0 {
c.SettleFor = 0
} else if c.SettleFor == 0 {
c.SettleFor = d.SettleFor
}
if c.UnreachableAfter <= 0 {
c.UnreachableAfter = d.UnreachableAfter
}
if c.LostAfter < c.UnreachableAfter {
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
}
if c.DrainGrace <= 0 {
c.DrainGrace = d.DrainGrace
}
if c.ParkAfter <= 0 {
c.ParkAfter = d.ParkAfter
}
if c.ScaleUpWaiting <= 0 {
c.ScaleUpWaiting = d.ScaleUpWaiting
}
if c.ScaleUpAfter <= 0 {
c.ScaleUpAfter = d.ScaleUpAfter
}
if c.ScaleDownAfter <= 0 {
c.ScaleDownAfter = d.ScaleDownAfter
}
if c.PortRange[1] <= c.PortRange[0] {
c.PortRange = d.PortRange
}
if c.MgmtInterface == "" {
c.MgmtInterface = d.MgmtInterface
}
}
// Reconciler compares the catalog against what is running and closes the gap.
type Reconciler struct {
// leader gates every action; nil means act unconditionally.
leader Leader
cfg Config
cat *catalog.Live
inv *inventory.Store
sched *placement.Scheduler
disp Dispatcher
log *slog.Logger
now func() time.Time
mu sync.Mutex
replicas map[string]*Replica
// nextAttempt is the earliest a model may be rebuilt, per model.
nextAttempt map[string]time.Time
// attempts counts consecutive failures per model, for the backoff.
attempts map[string]int
// lastExplain keeps the most recent placement failure per model, so
// `ctl explain-placement` can answer without re-running placement.
lastExplain map[string]placement.Result
seq int
// reservedPorts holds ports the fleet reports as in use but no replica
// claims, so the allocator cannot hand out what something is still bound to.
reservedPorts map[int]struct{}
// wanted records when an on-demand model was last asked for.
//
// The whole of on-demand loading, kept here rather than in the catalog
// because it is state about this moment and not about what the model is.
// A model nobody has wanted has no entry and is simply not running.
wanted map[string]time.Time
// restartQueue holds, per model, the replicas an operator asked to be
// restarted and not yet begun. One is replaced at a time: replacing them
// all at once would need the memory of every replica twice over.
restartQueue map[string][]string
// restartInPlace are the replicas asked to restart by stopping first.
restartInPlace map[string]bool
// restartRefused says, by replica, why a restart was given up rather
// than leave the model without it.
restartRefused map[string]string
// parked holds models that failed ParkAfter times in a row.
parked map[string]parkRecord
// extra is how many replicas load has added above a model's base, and
// busySince/quietSince when its load last crossed the thresholds.
extra map[string]int
busySince map[string]time.Time
quietSince map[string]time.Time
// placementLogged remembers the last "no placement" said per model, so it
// is said when it changes rather than on every five-second pass.
placementLogged map[string]placementNote
// preempting is, per model, the replicas stopped to make room for it and
// not gone yet; held is, per model that gave them up, until when it may
// not take the room back; preempted counts, per model, replicas it lost.
preempting map[string][]string
held map[string]hold
preempted map[string]int
// quotas is the tenancy tree's capacity quotas as last read; nil is none.
quotas func() *quota.Policy
// measurements are benchmark results by model/profile; nil is none.
measurements func() map[string]Measured
wake chan struct{}
stop chan struct{}
done chan struct{}
once sync.Once
// startedAt and lastNodeCount track the settling window: placement waits
// until the fleet has stopped growing, or the window expires.
startedAt time.Time
lastNodeCount int
steadySince time.Time
}
// New builds a reconciler.
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
cfg.normalize()
if log == nil {
log = slog.Default()
}
return &Reconciler{
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
sched: placement.New(inv),
now: time.Now,
replicas: map[string]*Replica{},
nextAttempt: map[string]time.Time{},
attempts: map[string]int{},
reservedPorts: map[int]struct{}{},
wanted: map[string]time.Time{},
restartQueue: map[string][]string{},
restartInPlace: map[string]bool{},
restartRefused: map[string]string{},
parked: map[string]parkRecord{},
extra: map[string]int{},
busySince: map[string]time.Time{},
quietSince: map[string]time.Time{},
placementLogged: map[string]placementNote{},
lastExplain: map[string]placement.Result{},
preempting: map[string][]string{},
held: map[string]hold{},
preempted: map[string]int{},
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
startedAt: time.Now(),
}
}
// SetClock replaces the time source, for tests.
func (r *Reconciler) SetClock(now func() time.Time) {
r.now = now
r.startedAt = now()
}
// Leader reports whether this process should be acting on the fleet.
type Leader interface{ IsLeader() bool }
// SetLeadership gates every action on holding leadership.
//
// A setter rather than a constructor argument because the thing that decides
// leadership needs a store, and the store is wired after the controller. Nil
// leaves the reconciler acting unconditionally, which is what a single control
// plane wants and what every test expects.
func (r *Reconciler) SetLeadership(l Leader) { r.leader = l }
// Start runs the loop until Close.
func (r *Reconciler) Start(ctx context.Context) {
go func() {
defer close(r.done)
t := time.NewTicker(r.cfg.Interval)
defer t.Stop()
r.Reconcile(ctx)
for {
select {
case <-r.stop:
return
case <-ctx.Done():
return
case <-r.wake:
r.Reconcile(ctx)
case <-t.C:
r.Reconcile(ctx)
}
}
}()
}
// Close stops the loop.
func (r *Reconciler) Close() {
r.once.Do(func() {
close(r.stop)
<-r.done
})
}
// Notify wakes the loop, for when something changed and waiting for the next
// tick would add pointless latency.
func (r *Reconciler) Notify() {
select {
case r.wake <- struct{}{}:
default:
}
}
// Reconcile performs one pass. It is exported so tests can drive it directly
// instead of racing a ticker.
func (r *Reconciler) Reconcile(ctx context.Context) {
// A standby does nothing at all here. Everything below issues commands to
// agents — placing, evicting, draining, reaping — and two control planes
// doing that against one fleet is how a model ends up with twice the
// replicas it asked for, each holding GPUs the other believes are free.
//
// Unset means leader, so a single control plane and every test behave
// exactly as they did before leadership existed.
if r.leader != nil && !r.leader.IsLeader() {
return
}
r.expireNodes()
r.expireStuckReplicas()
r.drainCordonedNodes(ctx)
r.reapFailed(ctx)
// Everything above is about what is already running and stays correct on a
// partial fleet. Placement is the part that must not act on one.
if !r.fleetHasSettled() {
return
}
for _, m := range r.cat.Load().Models() {
r.reconcileModel(ctx, m)
}
r.stopReplicasOfRemovedModels(ctx)
}
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
//
// reconcileModel only ever visits models that are in the catalog, so a replica
// whose model has been removed was visited by nothing: it kept its GPUs, kept
// counting towards the inventory, and kept running an engine for a name no
// client could ask for. The endpoint publisher already skipped it with the
// comment "the reconciler will tear it down", which was simply not true.
//
// It mattered little while a catalog could only change by restarting the
// process. It matters now that models come and go while the cluster runs, which
// is the whole point of making the catalog live.
//
// Ordered after the model loop so a rebuild elsewhere has already claimed what
// it needs: freeing these GPUs first would let a replacement land on hardware
// that is about to be freed anyway, and churn two nodes instead of one.
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
cat := r.cat.Load()
r.mu.Lock()
var gone []*Replica
for _, rep := range r.replicas {
if _, ok := cat.Resolve(rep.Model); !ok {
gone = append(gone, rep)
}
}
r.mu.Unlock()
for _, rep := range gone {
r.log.Info("stopping a replica whose model left the catalog",
"replica", rep.ID, "model", rep.Model)
r.stopReplica(ctx, rep.ID)
}
}
// fleetHasSettled reports whether placement may run yet.
//
// Two ways out, because either alone is wrong. Waiting for the window to expire
// would make every restart cost that much downtime even when the whole fleet
// reconnected in a second. Waiting only for the count to stop changing would
// hang forever on a cluster whose nodes never all come back.
func (r *Reconciler) fleetHasSettled() bool {
if r.cfg.SettleFor <= 0 {
return true
}
now := r.now()
if now.Sub(r.startedAt) >= r.cfg.SettleFor {
return true
}
n := len(r.inv.Nodes())
if n != r.lastNodeCount {
r.lastNodeCount, r.steadySince = n, now
return false
}
// Steady for a moment, and at least one node present. A count that has been
// zero since start is not a settled fleet, it is one that has not arrived.
if n == 0 {
return false
}
// Half the window, not a quarter of it, and the window is a minute.
//
// Measured on the cluster: after a restart node05 registered in one second
// and the other four arrived seventeen seconds later, because a
// disconnected agent reconnects on a backoff that reaches thirty seconds.
// A five-second steady period declared a fleet of one settled and resumed
// placement while four nodes were still backing off — the exact situation
// this exists to avoid, reached by being impatient about it.
settled := now.Sub(r.steadySince) >= r.cfg.SettleFor/2
if settled {
r.log.Info("fleet settled; placement resuming",
"nodes", n, "waited", now.Sub(r.startedAt).Round(time.Millisecond))
r.startedAt = time.Time{} // never wait again in this process
}
return settled
}
// expireStuckReplicas fails replicas that never finished coming up.
//
// Every other path out of Starting depends on the agent reporting something. A
// replica whose agent never reports — the start was a duplicate shard id and
// became a no-op, the node went quiet in a way expiry did not catch, the engine
// wedged before its first health check — sits in Starting for as long as the
// process lives, holding its GPUs and counting towards the model's replica
// count while serving nothing.
//
// Found on a real cluster: two replicas sat in Starting for eight minutes past
// a two-minute startup timeout, and nothing would ever have moved them.
func (r *Reconciler) expireStuckReplicas() {
now := r.now()
var stuck []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Starting && rep.State != Placing {
continue
}
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
continue
}
rep.State = Failed
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
stuck = append(stuck, rep)
}
r.mu.Unlock()
for _, rep := range stuck {
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
"replica", rep.ID, "model", rep.Model, "state", "starting",
"timeout", r.startupTimeout(rep))
}
}
// startupTimeout is the profile's, or the configured default when the profile
// does not say — a replica with no deadline at all is the thing this exists to
// prevent.
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
return p.StartupTimeout
}
}
// No profile deadline: a generous default, because the thing being prevented
// is a replica with no deadline at all rather than one with a loose one.
// Loading a large model across nodes genuinely takes tens of minutes.
return 30 * time.Minute
}
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
// replicas of nodes that are gone.
//
// The two-stage delay is deliberate: a brief network blip must not tear down
// replicas that are still serving traffic perfectly well, and rebuilding a
// multi-node replica costs tens of minutes of model loading.
func (r *Reconciler) expireNodes() {
now := r.now()
for _, n := range r.inv.Nodes() {
silent := now.Sub(n.LastSeen)
switch {
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
r.inv.SetState(n.ID, inventory.Unreachable)
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
}
}
}
// drainCordonedNodes moves work off nodes an operator is taking out of service.
//
// Draining a node is not the same as losing one: the engines are healthy and
// still serving, so their replicas are marked Draining rather than Failed. That
// makes the model look short of replicas, a replacement is placed elsewhere,
// and the original is stopped only once the replacement is serving — which is
// the difference between a rolling move and an outage.
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
var toDrain []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
continue
}
for _, s := range rep.Shards {
n, ok := r.inv.Get(s.NodeID)
if ok && n.State == inventory.Draining {
rep.State = Draining
rep.DrainingAt = r.now()
toDrain = append(toDrain, rep)
break
}
}
}
shards := map[string][]Shard{}
for _, rep := range toDrain {
shards[rep.ID] = slices.Clone(rep.Shards)
}
r.mu.Unlock()
for _, rep := range toDrain {
r.log.Info("draining replica off a node being taken out of service",
"replica", rep.ID, "nodes", rep.NodeIDs())
for _, s := range shards[rep.ID] {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
r.mu.Lock()
defer r.mu.Unlock()
for _, rep := range r.replicas {
if rep.State.Terminal() || !rep.OnNode(nodeID) {
continue
}
rep.State = Failed
rep.Err = reason
}
}
// reapFailed releases the resources of terminal replicas and tells the agents
// to clean up whatever might still be running.
func (r *Reconciler) reapFailed(ctx context.Context) {
r.mu.Lock()
var doomed []*Replica
for _, rep := range r.replicas {
if rep.State.Terminal() {
doomed = append(doomed, rep)
}
}
for _, rep := range doomed {
delete(r.replicas, rep.ID)
if rep.State == Failed {
// Carry the failure count on the model, not the replica: the
// replica is about to disappear, and the backoff has to survive it.
r.attempts[rep.Model]++
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
if n := r.attempts[rep.Model]; n >= r.cfg.ParkAfter {
if _, already := r.parked[rep.Model]; !already {
m, _ := r.cat.Load().Resolve(rep.Model)
r.parked[rep.Model] = parkRecord{
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
}
r.log.Error("model parked: it failed every time it started, and is not retried "+
"until its definition changes or `llmgw ctl retry` is run",
"model", rep.Model, "failures", n, "last_error", rep.Err)
}
}
}
}
r.mu.Unlock()
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", retry.Round(time.Second))
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
for _, s := range rep.Shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard of a dead replica failed",
"replica", rep.ID, "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(rep.ID)
}
}
// backoff grows with consecutive failures and is jittered, so a cluster-wide
// problem does not produce synchronized retry storms across every model.
func (r *Reconciler) backoff(attempt int) time.Duration {
if attempt <= 0 {
return 0
}
d := r.cfg.BaseBackoff << min(attempt-1, 16)
if d > r.cfg.MaxBackoff || d <= 0 {
d = r.cfg.MaxBackoff
}
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
return d - d/8 + jitter
}
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// Snapshot under the lock and work from values afterwards. A replica
// pointer stops being safe the moment it is in the map: an agent's status
// report can mutate it from another goroutine at any time.
r.mu.Lock()
var liveIDs, drainingIDs, retiringIDs, avoid []string
readyLive := 0
for _, rep := range r.replicas {
if rep.Model != m.Name {
continue
}
switch {
case rep.State.Counts() && !rep.RetiringAt.IsZero():
retiringIDs = append(retiringIDs, rep.ID)
avoid = append(avoid, rep.NodeIDs()...)
case rep.State.Counts():
liveIDs = append(liveIDs, rep.ID)
avoid = append(avoid, rep.NodeIDs()...)
if rep.State == Ready {
readyLive++
}
case rep.State == Draining:
drainingIDs = append(drainingIDs, rep.ID)
}
}
// The next restart an operator queued, once the last one is done.
if len(retiringIDs) == 0 {
if id, ok := r.nextRestartLocked(m.Name); ok {
r.replicas[id].RetiringAt = r.now()
retiringIDs = append(retiringIDs, id)
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
readyLive--
r.log.Info("restarting a replica: placing its replacement first",
"replica", id, "model", m.Name)
}
}
blockedUntil := r.nextAttempt[m.Name]
park, isParked := r.parked[m.Name]
if isParked && park.fp != modelFingerprint(m) {
// Somebody changed the model — a new source, an edited profile — so
// the reason it kept failing may be gone. Worth one more streak.
delete(r.parked, m.Name)
r.attempts[m.Name] = 0
delete(r.nextAttempt, m.Name)
isParked, blockedUntil = false, time.Time{}
r.log.Info("model unparked: its definition changed", "model", m.Name)
}
r.mu.Unlock()
desired := r.desiredFor(m)
outcome := notTried
switch {
case len(liveIDs) < desired && isParked:
// Not placed. Its failure is on every status the operator reads.
case len(liveIDs) < desired && r.isHeld(m.Name):
// It gave up room to a model of a higher class, which is placed
// first; taking it back now would undo the preemption.
case len(liveIDs) < desired:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
// model that cannot start — a missing weights path, say — and bury
// the real error in noise.
return
}
outcome = r.scaleUp(ctx, m, avoid)
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
}
// A drained replica is stopped once its replacement is serving, or once the
// grace period expires. Stopping it the moment it was drained would cut
// in-flight requests, which is the thing draining exists to avoid.
replaced := len(liveIDs) >= desired
for _, id := range drainingIDs {
r.mu.Lock()
rep, ok := r.replicas[id]
var expired bool
grace, preemptedFor := r.cfg.DrainGrace, ""
if ok {
if rep.PreemptedFor != "" {
grace, preemptedFor = r.cfg.PreemptGrace, rep.PreemptedFor
}
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace
}
r.mu.Unlock()
switch {
case !ok:
case replaced:
r.stopReplica(ctx, id)
case expired && preemptedFor != "":
r.log.Info("stopping a preempted replica", "replica", id,
"for", preemptedFor, "after", grace)
r.stopReplica(ctx, id)
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", grace)
r.stopReplica(ctx, id)
}
}
r.retire(ctx, m, retiringIDs, readyLive, len(liveIDs), desired, outcome, isParked)
}
// scaleUp places and starts one replica per pass.
//
// One at a time on purpose: placement decisions depend on what is already
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome {
profiles := r.orderedProfiles(m)
res := r.sched.Place(profiles, avoid)
if res.Placement == nil && len(avoid) > 0 {
// Spreading across failure domains is a preference, not a rule. If it
// is the only thing standing in the way, take the placement anyway:
// a second replica on one node still survives an engine crash.
res = r.sched.Place(profiles, nil)
}
r.mu.Lock()
r.lastExplain[m.Name] = res
r.mu.Unlock()
if res.Placement == nil {
// Full, with a model somebody has just asked for and cards held by one
// nobody has asked for in a while. Stopping one is the whole point of
// on-demand loading; without it the feature works until the cluster is
// full, which is the only time it matters.
//
// Not retried here. The replica is stopped, its GPUs are released on
// the pass that reaps it, and the next pass places this model — which
// keeps the stop-then-start ordering in one place rather than two.
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
return makingRoom
}
if r.awaitingPreemption(m.Name) || r.preemptFor(ctx, m, avoid) {
return makingRoom
}
r.notePlacementFailure(m.Name, res.Explain())
return noRoom
}
if why := r.overLimit(m, res.Placement); why != "" {
// Recorded as the placement's answer, so the model reads as not
// placeable, and says why, wherever that is shown.
r.mu.Lock()
r.lastExplain[m.Name] = placement.Result{Rejections: []placement.Rejection{
{Profile: res.Placement.Profile.Name, Reason: why}}}
r.mu.Unlock()
r.notePlacementFailure(m.Name, why)
return overLimit
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
r.log.Error("starting replica failed", "model", m.Name, "err", err)
return noRoom
}
return placed
}
// scaleOutcome is what one scaleUp did.
type scaleOutcome int
const (
notTried scaleOutcome = iota // nothing was asked of placement this pass
placed // a replica was started
makingRoom // room is being made: a preemption, an idle replica stopping
noRoom // placement found nowhere, and nothing is making room
overLimit // there is room, but the owner's quota does not allow it
)
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
prof, ok := m.ProfileByName(pl.Profile.Name)
if !ok {
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
}
espec, ok := m.EngineProfile(prof.Name)
if !ok {
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
}
r.mu.Lock()
r.seq++
id := replicaID(m.Name, r.seq)
attempt := r.attempts[m.Name]
r.mu.Unlock()
rep := &Replica{
ID: id, Model: m.Name, Profile: prof.Name, ProfileFP: prof.Fingerprint(),
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
WantShards: len(pl.Nodes),
}
// The leader's address and port are fixed before any shard starts: every
// rank needs the same rendezvous point, and a worker that starts first must
// already know where to dial.
leader := pl.Nodes[0]
leaderNode, ok := r.inv.Get(leader.NodeID)
if !ok {
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
}
basePort := r.allocatePorts(pl)
for _, a := range pl.Nodes {
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
rep.Shards = append(rep.Shards, Shard{
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
HTTPPort: basePort, State: supervisor.Pending,
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof),
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
// Everything read from rep has to be read now: once it is in the map an
// agent status report can mutate it concurrently, and the race detector is
// right to object.
//
// The leader first, then workers ascending. Both engines put the rendezvous
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
// worker started first has nothing to connect to and spends its startup
// budget retrying.
//
// An earlier version started workers first, on the theory that the head
// should wait for them. That is backwards: vLLM's server blocks until the
// Ray cluster has the GPUs it needs, so the leader can be started
// immediately and will wait on its own.
order := slices.Clone(rep.Shards)
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
nodeIDs := rep.NodeIDs()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
for _, s := range order {
cmd := ShardCommand{
ReplicaID: id,
ShardID: s.ID,
NodeID: s.NodeID,
Spec: engines.ShardSpec{
ReplicaID: id,
ShardID: s.ID,
Rank: s.Rank,
NNodes: len(pl.Nodes),
LeaderAddr: addrOf(leaderNode),
LeaderPort: basePort + 1,
GPUIndices: s.GPUIndices,
HCAs: s.HCAs,
MgmtInterface: r.cfg.MgmtInterface,
HTTPPort: s.HTTPPort,
Profile: espec,
APIKey: r.engineKey(id),
Threads: cpuCores(prof),
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
Fabric: pl.Fabric,
ProfileFP: prof.Fingerprint(),
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
return fmt.Errorf("starting shard %s: %w", s.ID, err)
}
r.markShard(id, s.ID, supervisor.Starting, "")
}
r.log.Info("replica placed",
"replica", id, "model", m.Name, "profile", prof.Name,
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
return nil
}
// replicaID names a replica uniquely for the life of the cluster, not the life
// of this process.
//
// The counter alone is not enough, and the way it fails is ugly. A control
// plane that restarts begins counting again, so the replica it places next is
// called llama-1 — which is also what an engine started before the restart and
// still running on some other node calls itself. Two nodes then claim the same
// shard id, adoption hands the id back and forth between them, and every
// placement lands on a port an orphan is already holding. Observed on a real
// cluster as a bind-conflict loop that never settled.
//
// The random suffix costs nothing and makes the collision impossible.
func replicaID(model string, seq int) string {
var buf [4]byte
if _, err := crypterand.Read(buf[:]); err != nil {
// Randomness failing is not a reason to stop placing replicas; the
// counter alone is still unique within this process.
return fmt.Sprintf("%s-%d", model, seq)
}
return fmt.Sprintf("%s-%x", model, buf)
}
// allocatePorts picks a port shared by every shard of a replica.
//
// The same port on every node keeps the leader address predictable and the
// port bookkeeping trivial; nodes do not share a network namespace, so there is
// no conflict in reusing it.
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
used := map[int]struct{}{}
r.mu.Lock()
for _, rep := range r.replicas {
for _, s := range rep.Shards {
used[s.HTTPPort] = struct{}{}
}
}
// Ports an agent has reported but nothing owns — a stray mid-shutdown, an
// engine whose replica was just reaped. The process is still listening
// whatever the bookkeeping says, and handing its port to a new shard means
// that shard fails to bind and takes the whole replica with it.
for p := range r.reservedPorts {
used[p] = struct{}{}
}
r.mu.Unlock()
// Two ports per replica: HTTP and the distributed init rendezvous.
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
if _, taken := used[p]; !taken {
return p
}
}
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
continue
}
rep.State = Draining
rep.DrainingAt = r.now()
shards := slices.Clone(rep.Shards)
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
return
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
r.mu.Unlock()
for _, s := range shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(id)
r.mu.Lock()
delete(r.replicas, id)
r.mu.Unlock()
}
// OnFetchProgress records how far a shard's download has got.
//
// Separate from OnShardStatus because it is not a transition: the shard is
// still starting, and treating each progress report as a state change would
// reset the timers that decide when a slow start becomes a failed one.
func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID != shardID {
continue
}
rep.Shards[i].Fetch = &FetchProgress{
Source: source, BytesDone: done, BytesTotal: total,
}
}
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
r.Notify()
}
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID != shardID {
continue
}
rep.Shards[i].State = st
if errMsg != "" {
rep.Shards[i].Err = errMsg
}
// A real transition means the download is over, one way or the other.
// Left in place it would sit in the listing at ninety-nine percent
// under a shard that has been serving for an hour.
rep.Shards[i].Fetch = nil
break
}
prev := rep.State
rep.deriveState(r.now())
if rep.State == Ready && prev != Ready {
// A model that comes up clears its backoff; the next failure starts
// counting from scratch rather than from an old streak.
r.attempts[rep.Model] = 0
delete(r.nextAttempt, rep.Model)
}
}
// Replicas returns a snapshot of every replica, sorted by id.
func (r *Reconciler) Replicas() []Replica {
r.mu.Lock()
defer r.mu.Unlock()
out := make([]Replica, 0, len(r.replicas))
for _, rep := range r.replicas {
clone := *rep
clone.Shards = slices.Clone(rep.Shards)
out = append(out, clone)
}
slices.SortFunc(out, func(a, b Replica) int {
if a.ID < b.ID {
return -1
}
if a.ID > b.ID {
return 1
}
return 0
})
return out
}
// ReadyReplicas returns the replicas the router may send traffic to.
func (r *Reconciler) ReadyReplicas() []Replica {
var out []Replica
for _, rep := range r.Replicas() {
if rep.State == Ready {
out = append(out, rep)
}
}
return out
}
// Explain returns the most recent placement result for a model, which is what
// `ctl explain-placement` prints.
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
r.mu.Lock()
defer r.mu.Unlock()
res, ok := r.lastExplain[model]
return res, ok
}
func addrOf(n inventory.Node) string {
if n.MgmtAddr != "" {
return n.MgmtAddr
}
if n.Hostname != "" {
return n.Hostname
}
return n.ID
}
// profileMemory is what a model's profile declared it needs, or zero.
//
// Looked up when a replica is adopted after a restart, because the allocation
// has to carry the same claim it did before — a control plane that forgot every
// claim on restart would place a second large model onto a node already full of
// the first.
func (r *Reconciler) profileMemory(model, profile string) int {
p, _ := r.profileOf(model, profile)
return p.MinMemoryGB
}
// profileOf is a replica's profile as the catalog holds it now.
func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) {
m, ok := r.cat.Load().Resolve(model)
if !ok {
return catalog.Profile{}, false
}
for _, p := range m.Profiles {
if p.Name == profile {
return p, true
}
}
return catalog.Profile{}, false
}
// cpuCores is what a profile reserves of a node's CPUs: its cores when it
// computes on them, nothing when a GPU does the work.
func cpuCores(p catalog.Profile) int {
if !p.CPUOnly() {
return 0
}
return p.CPUCores
}
func (r *Reconciler) engineKey(replicaID string) string {
if r.cfg.EngineKey == nil {
return ""
}
return r.cfg.EngineKey(replicaID)
}