Files
llmgw/internal/control/reconciler/reconciler.go
T
Ilker ManapandClaude Opus 5 9b32531ce3
CI / test (push) Successful in 1m29s
Stop an idle model to make room for one somebody asked for
On-demand loading worked right up to the moment the cluster was full and
then stopped meaning anything: the model somebody had just asked for could
not be placed, and the cards were held by a model nobody had asked for in an
hour. That is the only situation in which any of it matters.

Deliberately narrow. Only idle replicas of other on-demand models — a model
at min 1 is a promise that it stays up, and breaking it to serve a different
request would make every guarantee in the catalog conditional on load. Only
single-node replicas, since evicting a multi-node one frees several machines
and costs tens of minutes to rebuild. One at a time, because freeing
everything that qualifies would empty a cluster for a request that needed
one node. And never a replica with no reported usage: after a router restart
that is all of them, and unknown is not idle.

Writing the test found a gap the feature had rather than a bug in the test.
Two on-demand models on a cluster with room for one would take turns
evicting each other, at a model load per cycle — worse for both than one of
them waiting. A replica is now safe from eviction for a few minutes after it
starts serving, which bounds that.

Which in turn found a latent bug: deriveState stamped ReadyAt with
time.Now() inside a package that has an injectable clock, so the new
lifetime rule could not be tested at all — the fake clock said three minutes
and the wall clock said eight months. Invisible while nothing compared
ReadyAt to anything.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 17:51:07 +02:00

1301 lines
42 KiB
Go

package reconciler
import (
"context"
crypterand "crypto/rand"
"fmt"
"log/slog"
"math/rand/v2"
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// ShardCommand is everything an agent needs to launch one shard.
type ShardCommand struct {
ReplicaID string
ShardID string
NodeID string
Spec engines.ShardSpec
// StartupTimeout comes from the profile; model loading dominates it.
StartupTimeout time.Duration
// Pool and Fabric are control plane concepts the agent cannot derive. They
// travel with the command so the agent can report them back after a
// control plane restart (plan §6.5).
Pool string
Fabric string
}
// Dispatcher sends commands to agents.
//
// It is an interface so the reconciliation loop can be tested without a network:
// the loop's logic is where the interesting failures are, not the transport.
type Dispatcher interface {
StartShard(ctx context.Context, cmd ShardCommand) error
StopShard(ctx context.Context, nodeID, shardID string) error
DrainShard(ctx context.Context, nodeID, shardID string) error
}
// Config tunes the reconciliation loop.
type Config struct {
// Interval is how often desired and actual are compared when nothing else
// happens. Events wake the loop sooner.
Interval time.Duration
// BaseBackoff is how long to wait before rebuilding a failed replica.
BaseBackoff time.Duration
// MaxBackoff caps the exponential growth.
MaxBackoff time.Duration
// UnreachableAfter is how long without a heartbeat before a node is
// unreachable.
UnreachableAfter time.Duration
// LostAfter is how long unreachable before its replicas are declared dead.
// Longer than UnreachableAfter on purpose: a brief network blip must not
// tear down a replica that is still serving perfectly well.
LostAfter time.Duration
// DrainGrace bounds how long a draining replica waits for its replacement.
// Without a bound, draining a node whose model cannot be placed anywhere
// else would never finish.
DrainGrace time.Duration
// PortRange is the per-node port range for engine HTTP and dist-init.
PortRange [2]int
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
MgmtInterface string
// EvictionGrace is how long a replica must have gone unasked-for before it
// may be stopped to make room for another model.
//
// Shorter than IdleTimeout on purpose: a replica idle for five minutes is
// not worth reclaiming on its own, and is worth reclaiming when something
// else needs the cards. Still minutes rather than seconds, because stopping
// one throws away the KV cache prefix routing spent filling.
EvictionGrace time.Duration
// MinLifetime is how long a replica is safe from eviction after it starts
// serving, whatever else wants its cards.
//
// The thing that stops two models on a cluster with room for one from
// taking turns evicting each other. A load costs minutes; oscillating at a
// load per cycle is worse for both models than one of them waiting.
MinLifetime time.Duration
// IdleTimeout is how long an on-demand model stays loaded with nobody
// asking for it. A model may override it.
IdleTimeout time.Duration
// SettleFor is how long after start to leave placement alone while agents
// reconnect.
//
// A control plane that has just started knows nothing, and the first
// reconcile tick runs before any agent has registered. It sees an empty
// fleet and acts on it: "needs 2 nodes on one fabric, but no node reports a
// fabric" is that, and it is harmless only because adoption catches up.
// What is not harmless is placing on the fraction of the fleet that
// happens to have reported first — after one restart both replicas of a
// model landed on the single node that had registered, when four were
// coming.
//
// Adoption still runs throughout; only placement waits. The window ends
// early once the fleet stops growing, so a small cluster does not sit idle
// for the whole duration — but not too early: an agent that lost its
// connection reconnects on a backoff reaching thirty seconds, so "no new
// node for a few seconds" means nothing.
SettleFor time.Duration
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Interval: 5 * time.Second,
BaseBackoff: 10 * time.Second,
MaxBackoff: 10 * time.Minute,
UnreachableAfter: 15 * time.Second,
LostAfter: 60 * time.Second,
SettleFor: 60 * time.Second,
IdleTimeout: 15 * time.Minute,
EvictionGrace: 2 * time.Minute,
MinLifetime: 5 * time.Minute,
DrainGrace: 5 * time.Minute,
PortRange: [2]int{20000, 20999},
MgmtInterface: "eth0",
}
}
func (c *Config) normalize() {
d := DefaultConfig()
if c.Interval <= 0 {
c.Interval = d.Interval
}
if c.BaseBackoff <= 0 {
c.BaseBackoff = d.BaseBackoff
}
if c.MaxBackoff < c.BaseBackoff {
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
}
if c.IdleTimeout <= 0 {
c.IdleTimeout = d.IdleTimeout
}
if c.EvictionGrace <= 0 {
c.EvictionGrace = d.EvictionGrace
}
if c.MinLifetime <= 0 {
c.MinLifetime = d.MinLifetime
}
if c.SettleFor < 0 {
c.SettleFor = 0
} else if c.SettleFor == 0 {
c.SettleFor = d.SettleFor
}
if c.UnreachableAfter <= 0 {
c.UnreachableAfter = d.UnreachableAfter
}
if c.LostAfter < c.UnreachableAfter {
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
}
if c.DrainGrace <= 0 {
c.DrainGrace = d.DrainGrace
}
if c.PortRange[1] <= c.PortRange[0] {
c.PortRange = d.PortRange
}
if c.MgmtInterface == "" {
c.MgmtInterface = d.MgmtInterface
}
}
// Reconciler compares the catalog against what is running and closes the gap.
type Reconciler struct {
cfg Config
cat *catalog.Live
inv *inventory.Store
sched *placement.Scheduler
disp Dispatcher
log *slog.Logger
now func() time.Time
mu sync.Mutex
replicas map[string]*Replica
// nextAttempt is the earliest a model may be rebuilt, per model.
nextAttempt map[string]time.Time
// attempts counts consecutive failures per model, for the backoff.
attempts map[string]int
// lastExplain keeps the most recent placement failure per model, so
// `ctl explain-placement` can answer without re-running placement.
lastExplain map[string]placement.Result
seq int
// reservedPorts holds ports the fleet reports as in use but no replica
// claims, so the allocator cannot hand out what something is still bound to.
reservedPorts map[int]struct{}
// wanted records when an on-demand model was last asked for.
//
// The whole of on-demand loading, kept here rather than in the catalog
// because it is state about this moment and not about what the model is.
// A model nobody has wanted has no entry and is simply not running.
wanted map[string]time.Time
wake chan struct{}
stop chan struct{}
done chan struct{}
once sync.Once
// startedAt and lastNodeCount track the settling window: placement waits
// until the fleet has stopped growing, or the window expires.
startedAt time.Time
lastNodeCount int
steadySince time.Time
}
// New builds a reconciler.
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
cfg.normalize()
if log == nil {
log = slog.Default()
}
return &Reconciler{
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
sched: placement.New(inv),
now: time.Now,
replicas: map[string]*Replica{},
nextAttempt: map[string]time.Time{},
attempts: map[string]int{},
reservedPorts: map[int]struct{}{},
wanted: map[string]time.Time{},
lastExplain: map[string]placement.Result{},
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
startedAt: time.Now(),
}
}
// SetClock replaces the time source, for tests.
func (r *Reconciler) SetClock(now func() time.Time) {
r.now = now
r.startedAt = now()
}
// Start runs the loop until Close.
func (r *Reconciler) Start(ctx context.Context) {
go func() {
defer close(r.done)
t := time.NewTicker(r.cfg.Interval)
defer t.Stop()
r.Reconcile(ctx)
for {
select {
case <-r.stop:
return
case <-ctx.Done():
return
case <-r.wake:
r.Reconcile(ctx)
case <-t.C:
r.Reconcile(ctx)
}
}
}()
}
// Close stops the loop.
func (r *Reconciler) Close() {
r.once.Do(func() {
close(r.stop)
<-r.done
})
}
// Notify wakes the loop, for when something changed and waiting for the next
// tick would add pointless latency.
func (r *Reconciler) Notify() {
select {
case r.wake <- struct{}{}:
default:
}
}
// Reconcile performs one pass. It is exported so tests can drive it directly
// instead of racing a ticker.
func (r *Reconciler) Reconcile(ctx context.Context) {
r.expireNodes()
r.expireStuckReplicas()
r.drainCordonedNodes(ctx)
r.reapFailed(ctx)
// Everything above is about what is already running and stays correct on a
// partial fleet. Placement is the part that must not act on one.
if !r.fleetHasSettled() {
return
}
for _, m := range r.cat.Load().Models() {
r.reconcileModel(ctx, m)
}
r.stopReplicasOfRemovedModels(ctx)
}
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
//
// reconcileModel only ever visits models that are in the catalog, so a replica
// whose model has been removed was visited by nothing: it kept its GPUs, kept
// counting towards the inventory, and kept running an engine for a name no
// client could ask for. The endpoint publisher already skipped it with the
// comment "the reconciler will tear it down", which was simply not true.
//
// It mattered little while a catalog could only change by restarting the
// process. It matters now that models come and go while the cluster runs, which
// is the whole point of making the catalog live.
//
// Ordered after the model loop so a rebuild elsewhere has already claimed what
// it needs: freeing these GPUs first would let a replacement land on hardware
// that is about to be freed anyway, and churn two nodes instead of one.
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
cat := r.cat.Load()
r.mu.Lock()
var gone []*Replica
for _, rep := range r.replicas {
if _, ok := cat.Resolve(rep.Model); !ok {
gone = append(gone, rep)
}
}
r.mu.Unlock()
for _, rep := range gone {
r.log.Info("stopping a replica whose model left the catalog",
"replica", rep.ID, "model", rep.Model)
r.stopReplica(ctx, rep.ID)
}
}
// fleetHasSettled reports whether placement may run yet.
//
// Two ways out, because either alone is wrong. Waiting for the window to expire
// would make every restart cost that much downtime even when the whole fleet
// reconnected in a second. Waiting only for the count to stop changing would
// hang forever on a cluster whose nodes never all come back.
func (r *Reconciler) fleetHasSettled() bool {
if r.cfg.SettleFor <= 0 {
return true
}
now := r.now()
if now.Sub(r.startedAt) >= r.cfg.SettleFor {
return true
}
n := len(r.inv.Nodes())
if n != r.lastNodeCount {
r.lastNodeCount, r.steadySince = n, now
return false
}
// Steady for a moment, and at least one node present. A count that has been
// zero since start is not a settled fleet, it is one that has not arrived.
if n == 0 {
return false
}
// Half the window, not a quarter of it, and the window is a minute.
//
// Measured on the cluster: after a restart node05 registered in one second
// and the other four arrived seventeen seconds later, because a
// disconnected agent reconnects on a backoff that reaches thirty seconds.
// A five-second steady period declared a fleet of one settled and resumed
// placement while four nodes were still backing off — the exact situation
// this exists to avoid, reached by being impatient about it.
settled := now.Sub(r.steadySince) >= r.cfg.SettleFor/2
if settled {
r.log.Info("fleet settled; placement resuming",
"nodes", n, "waited", now.Sub(r.startedAt).Round(time.Millisecond))
r.startedAt = time.Time{} // never wait again in this process
}
return settled
}
// RecordUsage stamps replicas with when each was last sent a request.
//
// Replicas absent from the map are left alone rather than zeroed: a router that
// has just restarted knows nothing about what happened before it, and treating
// that as "never used" would make every replica look abandoned at once —
// exactly when a fleet is least able to afford acting on that.
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
if len(lastUsed) == 0 {
return
}
r.mu.Lock()
defer r.mu.Unlock()
for id, at := range lastUsed {
rep, ok := r.replicas[id]
if !ok {
continue
}
// Only forwards. Two routers reporting, or a stale snapshot arriving
// late, must not move a replica's last use backwards.
if at.After(rep.LastUsedAt) {
rep.LastUsedAt = at
}
}
r.noteUsage(lastUsed)
}
// IdleFor reports how long a replica has gone without a request, and whether
// that is known at all.
//
// The second return is not a detail. "Idle for an hour" and "nobody has told me
// yet" are different states, and anything that acts on idleness — eviction,
// most obviously — has to refuse to act on the second.
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[id]
if !ok || rep.LastUsedAt.IsZero() {
return 0, false
}
return r.now().Sub(rep.LastUsedAt), true
}
// Want marks an on-demand model as asked for, so the next pass places it.
//
// Called by whatever fields the request — the router, in practice. It returns
// whether the model is one this applies to, so a caller can tell "being loaded"
// from "there is genuinely nothing serving this", which are different answers
// to give a client.
func (r *Reconciler) Want(model string) bool {
m, ok := r.cat.Load().Resolve(model)
if !ok || !m.Scaling.OnDemand() {
return false
}
r.mu.Lock()
r.wanted[m.Name] = r.now()
r.mu.Unlock()
// Immediately, not on the next tick. Someone is waiting on this request,
// and five seconds of ticker is five seconds they spend staring at nothing.
r.Notify()
return true
}
// desiredFor is how many replicas a model should have right now.
//
// For an ordinary model that is its configured minimum. For an on-demand one it
// is one while somebody is asking and none once they stop, which is the whole
// feature: a cluster with more models than GPUs can hold every model in its
// catalog and only the wanted ones in its cards.
func (r *Reconciler) desiredFor(m catalog.Model) int {
if !m.Scaling.OnDemand() {
return m.Scaling.Min
}
r.mu.Lock()
last, asked := r.wanted[m.Name]
r.mu.Unlock()
if !asked {
return 0
}
if r.now().Sub(last) > r.idleTimeout(m) {
return 0
}
return 1
}
// idleTimeout is how long an on-demand model stays up unasked-for.
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
if m.Scaling.IdleTimeout > 0 {
return m.Scaling.IdleTimeout
}
return r.cfg.IdleTimeout
}
// noteUsage keeps an on-demand model alive while it is being used.
//
// Without this a model would be evicted exactly IdleTimeout after the request
// that loaded it, however busy it had been since — the timer would measure
// time since the first request rather than the last.
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
byModel := map[string]time.Time{}
for id, at := range lastUsed {
rep, ok := r.replicas[id]
if !ok {
continue
}
if at.After(byModel[rep.Model]) {
byModel[rep.Model] = at
}
}
for model, at := range byModel {
if cur, ok := r.wanted[model]; ok && at.After(cur) {
r.wanted[model] = at
}
}
}
// expireStuckReplicas fails replicas that never finished coming up.
//
// Every other path out of Starting depends on the agent reporting something. A
// replica whose agent never reports — the start was a duplicate shard id and
// became a no-op, the node went quiet in a way expiry did not catch, the engine
// wedged before its first health check — sits in Starting for as long as the
// process lives, holding its GPUs and counting towards the model's replica
// count while serving nothing.
//
// Found on a real cluster: two replicas sat in Starting for eight minutes past
// a two-minute startup timeout, and nothing would ever have moved them.
func (r *Reconciler) expireStuckReplicas() {
now := r.now()
var stuck []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Starting && rep.State != Placing {
continue
}
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
continue
}
rep.State = Failed
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
stuck = append(stuck, rep)
}
r.mu.Unlock()
for _, rep := range stuck {
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
"replica", rep.ID, "model", rep.Model, "state", "starting",
"timeout", r.startupTimeout(rep))
}
}
// startupTimeout is the profile's, or the configured default when the profile
// does not say — a replica with no deadline at all is the thing this exists to
// prevent.
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
return p.StartupTimeout
}
}
// No profile deadline: a generous default, because the thing being prevented
// is a replica with no deadline at all rather than one with a loose one.
// Loading a large model across nodes genuinely takes tens of minutes.
return 30 * time.Minute
}
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
// replicas of nodes that are gone.
//
// The two-stage delay is deliberate: a brief network blip must not tear down
// replicas that are still serving traffic perfectly well, and rebuilding a
// multi-node replica costs tens of minutes of model loading.
func (r *Reconciler) expireNodes() {
now := r.now()
for _, n := range r.inv.Nodes() {
silent := now.Sub(n.LastSeen)
switch {
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
r.inv.SetState(n.ID, inventory.Unreachable)
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
}
}
}
// drainCordonedNodes moves work off nodes an operator is taking out of service.
//
// Draining a node is not the same as losing one: the engines are healthy and
// still serving, so their replicas are marked Draining rather than Failed. That
// makes the model look short of replicas, a replacement is placed elsewhere,
// and the original is stopped only once the replacement is serving — which is
// the difference between a rolling move and an outage.
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
var toDrain []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
continue
}
for _, s := range rep.Shards {
n, ok := r.inv.Get(s.NodeID)
if ok && n.State == inventory.Draining {
rep.State = Draining
rep.DrainingAt = r.now()
toDrain = append(toDrain, rep)
break
}
}
}
shards := map[string][]Shard{}
for _, rep := range toDrain {
shards[rep.ID] = slices.Clone(rep.Shards)
}
r.mu.Unlock()
for _, rep := range toDrain {
r.log.Info("draining replica off a node being taken out of service",
"replica", rep.ID, "nodes", rep.NodeIDs())
for _, s := range shards[rep.ID] {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
r.mu.Lock()
defer r.mu.Unlock()
for _, rep := range r.replicas {
if rep.State.Terminal() || !rep.OnNode(nodeID) {
continue
}
rep.State = Failed
rep.Err = reason
}
}
// reapFailed releases the resources of terminal replicas and tells the agents
// to clean up whatever might still be running.
func (r *Reconciler) reapFailed(ctx context.Context) {
r.mu.Lock()
var doomed []*Replica
for _, rep := range r.replicas {
if rep.State.Terminal() {
doomed = append(doomed, rep)
}
}
for _, rep := range doomed {
delete(r.replicas, rep.ID)
if rep.State == Failed {
// Carry the failure count on the model, not the replica: the
// replica is about to disappear, and the backoff has to survive it.
r.attempts[rep.Model]++
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
}
}
r.mu.Unlock()
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", retry.Round(time.Second))
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
for _, s := range rep.Shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard of a dead replica failed",
"replica", rep.ID, "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(rep.ID)
}
}
// backoff grows with consecutive failures and is jittered, so a cluster-wide
// problem does not produce synchronized retry storms across every model.
func (r *Reconciler) backoff(attempt int) time.Duration {
if attempt <= 0 {
return 0
}
d := r.cfg.BaseBackoff << min(attempt-1, 16)
if d > r.cfg.MaxBackoff || d <= 0 {
d = r.cfg.MaxBackoff
}
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
return d - d/8 + jitter
}
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// Snapshot under the lock and work from values afterwards. A replica
// pointer stops being safe the moment it is in the map: an agent's status
// report can mutate it from another goroutine at any time.
r.mu.Lock()
var liveIDs, drainingIDs, avoid []string
for _, rep := range r.replicas {
if rep.Model != m.Name {
continue
}
switch {
case rep.State.Counts():
liveIDs = append(liveIDs, rep.ID)
avoid = append(avoid, rep.NodeIDs()...)
case rep.State == Draining:
drainingIDs = append(drainingIDs, rep.ID)
}
}
blockedUntil := r.nextAttempt[m.Name]
r.mu.Unlock()
desired := r.desiredFor(m)
switch {
case len(liveIDs) < desired:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
// model that cannot start — a missing weights path, say — and bury
// the real error in noise.
return
}
r.scaleUp(ctx, m, avoid)
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
}
// A drained replica is stopped once its replacement is serving, or once the
// grace period expires. Stopping it the moment it was drained would cut
// in-flight requests, which is the thing draining exists to avoid.
replaced := len(liveIDs) >= desired
for _, id := range drainingIDs {
r.mu.Lock()
rep, ok := r.replicas[id]
var expired bool
if ok {
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > r.cfg.DrainGrace
}
r.mu.Unlock()
switch {
case !ok:
case replaced:
r.stopReplica(ctx, id)
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", r.cfg.DrainGrace)
r.stopReplica(ctx, id)
}
}
}
// scaleUp places and starts one replica per pass.
//
// One at a time on purpose: placement decisions depend on what is already
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
res := r.sched.Place(m.PlacementProfiles(), avoid)
if res.Placement == nil && len(avoid) > 0 {
// Spreading across failure domains is a preference, not a rule. If it
// is the only thing standing in the way, take the placement anyway:
// a second replica on one node still survives an engine crash.
res = r.sched.Place(m.PlacementProfiles(), nil)
}
r.mu.Lock()
r.lastExplain[m.Name] = res
r.mu.Unlock()
if res.Placement == nil {
// Full, with a model somebody has just asked for and cards held by one
// nobody has asked for in a while. Stopping one is the whole point of
// on-demand loading; without it the feature works until the cluster is
// full, which is the only time it matters.
//
// Not retried here. The replica is stopped, its GPUs are released on
// the pass that reaps it, and the next pass places this model — which
// keeps the stop-then-start ordering in one place rather than two.
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
return
}
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
return
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
r.log.Error("starting replica failed", "model", m.Name, "err", err)
}
}
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
prof, ok := m.ProfileByName(pl.Profile.Name)
if !ok {
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
}
espec, ok := m.EngineProfile(prof.Name)
if !ok {
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
}
r.mu.Lock()
r.seq++
id := replicaID(m.Name, r.seq)
attempt := r.attempts[m.Name]
r.mu.Unlock()
rep := &Replica{
ID: id, Model: m.Name, Profile: prof.Name,
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
WantShards: len(pl.Nodes),
}
// The leader's address and port are fixed before any shard starts: every
// rank needs the same rendezvous point, and a worker that starts first must
// already know where to dial.
leader := pl.Nodes[0]
leaderNode, ok := r.inv.Get(leader.NodeID)
if !ok {
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
}
basePort := r.allocatePorts(pl)
for _, a := range pl.Nodes {
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
rep.Shards = append(rep.Shards, Shard{
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
HTTPPort: basePort, State: supervisor.Pending,
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
// Everything read from rep has to be read now: once it is in the map an
// agent status report can mutate it concurrently, and the race detector is
// right to object.
//
// The leader first, then workers ascending. Both engines put the rendezvous
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
// worker started first has nothing to connect to and spends its startup
// budget retrying.
//
// An earlier version started workers first, on the theory that the head
// should wait for them. That is backwards: vLLM's server blocks until the
// Ray cluster has the GPUs it needs, so the leader can be started
// immediately and will wait on its own.
order := slices.Clone(rep.Shards)
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
nodeIDs := rep.NodeIDs()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
for _, s := range order {
cmd := ShardCommand{
ReplicaID: id,
ShardID: s.ID,
NodeID: s.NodeID,
Spec: engines.ShardSpec{
ReplicaID: id,
ShardID: s.ID,
Rank: s.Rank,
NNodes: len(pl.Nodes),
LeaderAddr: addrOf(leaderNode),
LeaderPort: basePort + 1,
GPUIndices: s.GPUIndices,
HCAs: s.HCAs,
MgmtInterface: r.cfg.MgmtInterface,
HTTPPort: s.HTTPPort,
Profile: espec,
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
Fabric: pl.Fabric,
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
return fmt.Errorf("starting shard %s: %w", s.ID, err)
}
r.markShard(id, s.ID, supervisor.Starting, "")
}
r.log.Info("replica placed",
"replica", id, "model", m.Name, "profile", prof.Name,
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
return nil
}
// replicaID names a replica uniquely for the life of the cluster, not the life
// of this process.
//
// The counter alone is not enough, and the way it fails is ugly. A control
// plane that restarts begins counting again, so the replica it places next is
// called llama-1 — which is also what an engine started before the restart and
// still running on some other node calls itself. Two nodes then claim the same
// shard id, adoption hands the id back and forth between them, and every
// placement lands on a port an orphan is already holding. Observed on a real
// cluster as a bind-conflict loop that never settled.
//
// The random suffix costs nothing and makes the collision impossible.
func replicaID(model string, seq int) string {
var buf [4]byte
if _, err := crypterand.Read(buf[:]); err != nil {
// Randomness failing is not a reason to stop placing replicas; the
// counter alone is still unique within this process.
return fmt.Sprintf("%s-%d", model, seq)
}
return fmt.Sprintf("%s-%x", model, buf)
}
// allocatePorts picks a port shared by every shard of a replica.
//
// The same port on every node keeps the leader address predictable and the
// port bookkeeping trivial; nodes do not share a network namespace, so there is
// no conflict in reusing it.
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
used := map[int]struct{}{}
r.mu.Lock()
for _, rep := range r.replicas {
for _, s := range rep.Shards {
used[s.HTTPPort] = struct{}{}
}
}
// Ports an agent has reported but nothing owns — a stray mid-shutdown, an
// engine whose replica was just reaped. The process is still listening
// whatever the bookkeeping says, and handing its port to a new shard means
// that shard fails to bind and takes the whole replica with it.
for p := range r.reservedPorts {
used[p] = struct{}{}
}
r.mu.Unlock()
// Two ports per replica: HTTP and the distributed init rendezvous.
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
if _, taken := used[p]; !taken {
return p
}
}
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
continue
}
rep.State = Draining
rep.DrainingAt = r.now()
shards := slices.Clone(rep.Shards)
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
return
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
r.mu.Unlock()
for _, s := range shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(id)
r.mu.Lock()
delete(r.replicas, id)
r.mu.Unlock()
}
// Adopt rebuilds replica state from shards an agent reports it is already
// running (plan §6.5).
//
// This is what makes a control plane restart survivable. Without it the
// reconciler sees an empty cluster, places duplicates, and the engines already
// running keep holding GPUs the inventory believes are free — so the duplicates
// fail to allocate, with an error naming the new process rather than the old.
//
// Shards whose model or profile is no longer in the catalog are returned for
// the caller to stop: nothing will ever claim them.
func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) {
defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards)
byReplica := map[string][]AdoptedShard{}
for _, s := range shards {
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
}
r.mu.Lock()
defer r.mu.Unlock()
for replicaID, group := range byReplica {
if existing, known := r.replicas[replicaID]; known {
// The other half of a replica we are still assembling. Merge rather
// than choose: these shards belong to the same replica as the ones
// already adopted, they are simply on a different node whose agent
// registered a moment later. Treating them as a competing claim is
// what tore multi-node replicas apart on every control plane
// restart — one half adopted and failed for being short, the other
// stopped as unclaimed when it finally arrived.
if !existing.AdoptedAt.IsZero() && len(existing.Shards) < existing.WantShards {
r.mergeAdopted(existing, group)
continue
}
// Usually this is an agent reconnecting rather than a control plane
// restart, and its status reports keep the state current.
//
// Not always. Replica ids are derived from the model name and a
// counter, so a control plane that restarts and places before the
// agents reconnect produces a *different* replica under the same
// id, on a freshly allocated port. Skipping the adoption then
// leaves the cluster pointing at a port nothing listens on, while
// the engine that is actually serving is treated as a stray.
//
// The fleet is authoritative about what is running (plan §6.5), so
// what we merely placed loses to what an agent reports — unless our
// copy has a shard that has actually come up, in which case two
// things really are running and the one we did not ask for is the
// stray.
if existing.hasLiveShard() {
// Only what we do not recognise. A reconnecting agent reports
// the shards we already know, and disowning those would tell it
// to stop the very replica that is serving.
known := map[string]struct{}{}
for _, sh := range existing.Shards {
known[sh.ID] = struct{}{}
}
var strays []AdoptedShard
for _, s := range group {
if _, ok := known[s.ShardID]; !ok {
strays = append(strays, s)
}
}
unclaimed = append(unclaimed, strays...)
r.reservePorts(strays)
continue
}
r.log.Warn("a running replica reported by the fleet replaces one we placed",
"replica", replicaID, "placed_endpoint", existing.Endpoint)
for _, s := range existing.Shards {
r.inv.Release(s.ID)
}
delete(r.replicas, replicaID)
}
first := group[0]
m, ok := r.cat.Load().Resolve(first.Model)
if !ok {
unclaimed = append(unclaimed, group...)
r.reservePorts(group)
continue
}
prof, ok := m.ProfileByName(first.Profile)
if !ok {
unclaimed = append(unclaimed, group...)
r.reservePorts(group)
continue
}
rep := &Replica{
ID: replicaID, Model: m.Name, Profile: prof.Name,
Fabric: first.Fabric, Pool: first.Pool,
Weight: prof.Weight, CreatedAt: r.now(),
Endpoint: first.Endpoint,
// From the profile, not from what was reported: the point is to
// notice when what was reported is short.
WantShards: max(prof.Nodes, 1),
AdoptedAt: r.now(),
}
r.mergeAdopted(rep, group)
r.replicas[replicaID] = rep
}
return unclaimed
}
// mergeAdopted folds shards an agent reports into a replica being reassembled.
//
// Called both for the first group, which creates the replica, and for every
// later group belonging to it, so a replica spanning N nodes is built up over N
// registrations. Shard ids already present are skipped: an agent that
// reconnects repeats what it is running, and counting those again would carry a
// replica past its shard count without a second node ever appearing.
func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
have := make(map[string]struct{}, len(rep.Shards))
for _, sh := range rep.Shards {
have[sh.ID] = struct{}{}
}
for _, s := range group {
if _, dup := have[s.ShardID]; dup {
continue
}
have[s.ShardID] = struct{}{}
rep.Shards = append(rep.Shards, Shard{
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
HTTPPort: s.HTTPPort, State: s.State,
})
// Reclaim the GPUs, or the next placement will hand them out twice.
r.inv.Allocate(s.NodeID, inventory.Allocation{
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
})
// The endpoint comes from the leader, which may be in any of the
// groups — the rank 0 node is not necessarily the first to register.
if s.Rank == 0 && s.Endpoint != "" {
rep.Endpoint = s.Endpoint
}
}
slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank })
rep.deriveState(r.now())
r.log.Info("adopted replica from the fleet",
"replica", rep.ID, "model", rep.Model,
"shards", len(rep.Shards), "want", rep.WantShards, "state", rep.State)
}
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
// agent, having just told us everything it is running, did not mention.
//
// The mirror image of adoption, and the half that was missing. Adoption handles
// "the agent has something we do not know about"; this handles "we think the
// agent has something it does not". Both follow from the same rule: the fleet is
// authoritative about what is running (plan §6.5).
//
// Without it, an engine that dies while its agent survives leaves the control
// plane believing a replica is Ready forever. Observed on a real cluster: the
// engine was killed, the agent restarted and registered with nothing running,
// and the replica stayed Ready pointing at a dead port for as long as anyone
// watched. Traffic survived — the breaker took the endpoint out — but the
// cluster ran a third short with no sign of it.
//
// Safe against the obvious race. A registration follows a fresh connection, so
// anything we dispatched over a previous one is not running by definition;
// there is no window where a shard is legitimately started but unreported.
func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) {
if nodeID == "" {
return
}
reported := make(map[string]struct{}, len(running))
for _, s := range running {
reported[s.ShardID] = struct{}{}
}
var lost []string
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State == Stopped {
continue
}
changed := false
for i := range rep.Shards {
sh := &rep.Shards[i]
if sh.NodeID != nodeID || sh.State.Terminal() {
continue
}
if _, ok := reported[sh.ID]; ok {
continue
}
sh.State = supervisor.Failed
sh.Err = "the agent reports it is not running"
r.inv.Release(sh.ID)
changed = true
lost = append(lost, sh.ID)
}
if changed {
rep.deriveState(r.now())
}
}
r.mu.Unlock()
for _, id := range lost {
r.log.Warn("a shard we believed was running is not; the agent does not have it",
"node", nodeID, "shard", id)
}
}
// reservePorts records ports held by shards nothing owns.
//
// They are being stopped, but stopping is not instant and the engine keeps its
// socket until it exits. Handing the port to a new shard in the meantime makes
// that shard fail to bind, which fails its replica, which places another one on
// the same port — a loop that does not end on its own.
//
// Called with the lock held.
func (r *Reconciler) reservePorts(shards []AdoptedShard) {
for _, s := range shards {
if s.HTTPPort > 0 {
r.reservedPorts[s.HTTPPort] = struct{}{}
}
}
}
// AdoptedShard is one running shard an agent reported at registration.
type AdoptedShard struct {
NodeID string
ReplicaID string
ShardID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Model string
Profile string
Pool string
Fabric string
// Endpoint is the leader's base URL, set on the rank 0 shard.
Endpoint string
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
r.Notify()
}
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID != shardID {
continue
}
rep.Shards[i].State = st
if errMsg != "" {
rep.Shards[i].Err = errMsg
}
break
}
prev := rep.State
rep.deriveState(r.now())
if rep.State == Ready && prev != Ready {
// A model that comes up clears its backoff; the next failure starts
// counting from scratch rather than from an old streak.
r.attempts[rep.Model] = 0
delete(r.nextAttempt, rep.Model)
}
}
// Replicas returns a snapshot of every replica, sorted by id.
func (r *Reconciler) Replicas() []Replica {
r.mu.Lock()
defer r.mu.Unlock()
out := make([]Replica, 0, len(r.replicas))
for _, rep := range r.replicas {
clone := *rep
clone.Shards = slices.Clone(rep.Shards)
out = append(out, clone)
}
slices.SortFunc(out, func(a, b Replica) int {
if a.ID < b.ID {
return -1
}
if a.ID > b.ID {
return 1
}
return 0
})
return out
}
// ReadyReplicas returns the replicas the router may send traffic to.
func (r *Reconciler) ReadyReplicas() []Replica {
var out []Replica
for _, rep := range r.Replicas() {
if rep.State == Ready {
out = append(out, rep)
}
}
return out
}
// Explain returns the most recent placement result for a model, which is what
// `ctl explain-placement` prints.
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
r.mu.Lock()
defer r.mu.Unlock()
res, ok := r.lastExplain[model]
return res, ok
}
func addrOf(n inventory.Node) string {
if n.MgmtAddr != "" {
return n.MgmtAddr
}
if n.Hostname != "" {
return n.Hostname
}
return n.ID
}