Files
llmgw/internal/control/reconciler/reconciler.go
T
Ilker ManapandClaude Opus 5 1c57a4e378
CI / test (push) Successful in 1m29s
Let the catalog change while the cluster is running
Adding a model was an operator procedure with downtime: edit a file on the
control plane, restart it, wait out the settling window, watch replicas
churn while adoption catches up. I did that eight times in a day, and one
of the restarts cost two minutes of outage while stalled replicas timed
out. None of it was necessary. The reconciler already compares desired
against actual on every pass, so a new catalog is simply a different
desired.

A Catalog stays immutable — New is the only way to build one and it refuses
an invalid one — and what changes is which immutable catalog is current.
Readers take the pointer and finish their operation against it, so a swap
mid-request is invisible rather than a race. The router is told rather than
asked, the same shape as the endpoint list and for the same reason: a push
happens once per change, a pull would happen once per request.

The router still seeds from configuration as well as being pushed to. A
router with a static endpoint list and no control plane is a supported mode
and has nothing to push it a catalog; without seeding it would resolve no
model at all.

Which turned up a bug that was latent while catalogs were fixed.
reconcileModel only ever visits models that are in the catalog, so a
replica whose model had been removed was visited by nothing: it kept its
GPUs, kept counting, and kept running an engine for a name no client could
ask for. The endpoint publisher skipped such a replica with the comment
"the reconciler will tear it down", which was not true. It is now, and the
comment says what actually happens instead of what was assumed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 01:08:21 +02:00

1179 lines
38 KiB
Go

package reconciler
import (
"context"
crypterand "crypto/rand"
"fmt"
"log/slog"
"math/rand/v2"
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// ShardCommand is everything an agent needs to launch one shard.
type ShardCommand struct {
ReplicaID string
ShardID string
NodeID string
Spec engines.ShardSpec
// StartupTimeout comes from the profile; model loading dominates it.
StartupTimeout time.Duration
// Pool and Fabric are control plane concepts the agent cannot derive. They
// travel with the command so the agent can report them back after a
// control plane restart (plan §6.5).
Pool string
Fabric string
}
// Dispatcher sends commands to agents.
//
// It is an interface so the reconciliation loop can be tested without a network:
// the loop's logic is where the interesting failures are, not the transport.
type Dispatcher interface {
StartShard(ctx context.Context, cmd ShardCommand) error
StopShard(ctx context.Context, nodeID, shardID string) error
DrainShard(ctx context.Context, nodeID, shardID string) error
}
// Config tunes the reconciliation loop.
type Config struct {
// Interval is how often desired and actual are compared when nothing else
// happens. Events wake the loop sooner.
Interval time.Duration
// BaseBackoff is how long to wait before rebuilding a failed replica.
BaseBackoff time.Duration
// MaxBackoff caps the exponential growth.
MaxBackoff time.Duration
// UnreachableAfter is how long without a heartbeat before a node is
// unreachable.
UnreachableAfter time.Duration
// LostAfter is how long unreachable before its replicas are declared dead.
// Longer than UnreachableAfter on purpose: a brief network blip must not
// tear down a replica that is still serving perfectly well.
LostAfter time.Duration
// DrainGrace bounds how long a draining replica waits for its replacement.
// Without a bound, draining a node whose model cannot be placed anywhere
// else would never finish.
DrainGrace time.Duration
// PortRange is the per-node port range for engine HTTP and dist-init.
PortRange [2]int
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
MgmtInterface string
// SettleFor is how long after start to leave placement alone while agents
// reconnect.
//
// A control plane that has just started knows nothing, and the first
// reconcile tick runs before any agent has registered. It sees an empty
// fleet and acts on it: "needs 2 nodes on one fabric, but no node reports a
// fabric" is that, and it is harmless only because adoption catches up.
// What is not harmless is placing on the fraction of the fleet that
// happens to have reported first — after one restart both replicas of a
// model landed on the single node that had registered, when four were
// coming.
//
// Adoption still runs throughout; only placement waits. The window ends
// early once the fleet stops growing, so a small cluster does not sit idle
// for the whole duration — but not too early: an agent that lost its
// connection reconnects on a backoff reaching thirty seconds, so "no new
// node for a few seconds" means nothing.
SettleFor time.Duration
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Interval: 5 * time.Second,
BaseBackoff: 10 * time.Second,
MaxBackoff: 10 * time.Minute,
UnreachableAfter: 15 * time.Second,
LostAfter: 60 * time.Second,
SettleFor: 60 * time.Second,
DrainGrace: 5 * time.Minute,
PortRange: [2]int{20000, 20999},
MgmtInterface: "eth0",
}
}
func (c *Config) normalize() {
d := DefaultConfig()
if c.Interval <= 0 {
c.Interval = d.Interval
}
if c.BaseBackoff <= 0 {
c.BaseBackoff = d.BaseBackoff
}
if c.MaxBackoff < c.BaseBackoff {
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
}
if c.SettleFor < 0 {
c.SettleFor = 0
} else if c.SettleFor == 0 {
c.SettleFor = d.SettleFor
}
if c.UnreachableAfter <= 0 {
c.UnreachableAfter = d.UnreachableAfter
}
if c.LostAfter < c.UnreachableAfter {
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
}
if c.DrainGrace <= 0 {
c.DrainGrace = d.DrainGrace
}
if c.PortRange[1] <= c.PortRange[0] {
c.PortRange = d.PortRange
}
if c.MgmtInterface == "" {
c.MgmtInterface = d.MgmtInterface
}
}
// Reconciler compares the catalog against what is running and closes the gap.
type Reconciler struct {
cfg Config
cat *catalog.Live
inv *inventory.Store
sched *placement.Scheduler
disp Dispatcher
log *slog.Logger
now func() time.Time
mu sync.Mutex
replicas map[string]*Replica
// nextAttempt is the earliest a model may be rebuilt, per model.
nextAttempt map[string]time.Time
// attempts counts consecutive failures per model, for the backoff.
attempts map[string]int
// lastExplain keeps the most recent placement failure per model, so
// `ctl explain-placement` can answer without re-running placement.
lastExplain map[string]placement.Result
seq int
// reservedPorts holds ports the fleet reports as in use but no replica
// claims, so the allocator cannot hand out what something is still bound to.
reservedPorts map[int]struct{}
wake chan struct{}
stop chan struct{}
done chan struct{}
once sync.Once
// startedAt and lastNodeCount track the settling window: placement waits
// until the fleet has stopped growing, or the window expires.
startedAt time.Time
lastNodeCount int
steadySince time.Time
}
// New builds a reconciler.
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
cfg.normalize()
if log == nil {
log = slog.Default()
}
return &Reconciler{
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
sched: placement.New(inv),
now: time.Now,
replicas: map[string]*Replica{},
nextAttempt: map[string]time.Time{},
attempts: map[string]int{},
reservedPorts: map[int]struct{}{},
lastExplain: map[string]placement.Result{},
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
startedAt: time.Now(),
}
}
// SetClock replaces the time source, for tests.
func (r *Reconciler) SetClock(now func() time.Time) {
r.now = now
r.startedAt = now()
}
// Start runs the loop until Close.
func (r *Reconciler) Start(ctx context.Context) {
go func() {
defer close(r.done)
t := time.NewTicker(r.cfg.Interval)
defer t.Stop()
r.Reconcile(ctx)
for {
select {
case <-r.stop:
return
case <-ctx.Done():
return
case <-r.wake:
r.Reconcile(ctx)
case <-t.C:
r.Reconcile(ctx)
}
}
}()
}
// Close stops the loop.
func (r *Reconciler) Close() {
r.once.Do(func() {
close(r.stop)
<-r.done
})
}
// Notify wakes the loop, for when something changed and waiting for the next
// tick would add pointless latency.
func (r *Reconciler) Notify() {
select {
case r.wake <- struct{}{}:
default:
}
}
// Reconcile performs one pass. It is exported so tests can drive it directly
// instead of racing a ticker.
func (r *Reconciler) Reconcile(ctx context.Context) {
r.expireNodes()
r.expireStuckReplicas()
r.drainCordonedNodes(ctx)
r.reapFailed(ctx)
// Everything above is about what is already running and stays correct on a
// partial fleet. Placement is the part that must not act on one.
if !r.fleetHasSettled() {
return
}
for _, m := range r.cat.Load().Models() {
r.reconcileModel(ctx, m)
}
r.stopReplicasOfRemovedModels(ctx)
}
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
//
// reconcileModel only ever visits models that are in the catalog, so a replica
// whose model has been removed was visited by nothing: it kept its GPUs, kept
// counting towards the inventory, and kept running an engine for a name no
// client could ask for. The endpoint publisher already skipped it with the
// comment "the reconciler will tear it down", which was simply not true.
//
// It mattered little while a catalog could only change by restarting the
// process. It matters now that models come and go while the cluster runs, which
// is the whole point of making the catalog live.
//
// Ordered after the model loop so a rebuild elsewhere has already claimed what
// it needs: freeing these GPUs first would let a replacement land on hardware
// that is about to be freed anyway, and churn two nodes instead of one.
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
cat := r.cat.Load()
r.mu.Lock()
var gone []*Replica
for _, rep := range r.replicas {
if _, ok := cat.Resolve(rep.Model); !ok {
gone = append(gone, rep)
}
}
r.mu.Unlock()
for _, rep := range gone {
r.log.Info("stopping a replica whose model left the catalog",
"replica", rep.ID, "model", rep.Model)
r.stopReplica(ctx, rep.ID)
}
}
// fleetHasSettled reports whether placement may run yet.
//
// Two ways out, because either alone is wrong. Waiting for the window to expire
// would make every restart cost that much downtime even when the whole fleet
// reconnected in a second. Waiting only for the count to stop changing would
// hang forever on a cluster whose nodes never all come back.
func (r *Reconciler) fleetHasSettled() bool {
if r.cfg.SettleFor <= 0 {
return true
}
now := r.now()
if now.Sub(r.startedAt) >= r.cfg.SettleFor {
return true
}
n := len(r.inv.Nodes())
if n != r.lastNodeCount {
r.lastNodeCount, r.steadySince = n, now
return false
}
// Steady for a moment, and at least one node present. A count that has been
// zero since start is not a settled fleet, it is one that has not arrived.
if n == 0 {
return false
}
// Half the window, not a quarter of it, and the window is a minute.
//
// Measured on the cluster: after a restart node05 registered in one second
// and the other four arrived seventeen seconds later, because a
// disconnected agent reconnects on a backoff that reaches thirty seconds.
// A five-second steady period declared a fleet of one settled and resumed
// placement while four nodes were still backing off — the exact situation
// this exists to avoid, reached by being impatient about it.
settled := now.Sub(r.steadySince) >= r.cfg.SettleFor/2
if settled {
r.log.Info("fleet settled; placement resuming",
"nodes", n, "waited", now.Sub(r.startedAt).Round(time.Millisecond))
r.startedAt = time.Time{} // never wait again in this process
}
return settled
}
// RecordUsage stamps replicas with when each was last sent a request.
//
// Replicas absent from the map are left alone rather than zeroed: a router that
// has just restarted knows nothing about what happened before it, and treating
// that as "never used" would make every replica look abandoned at once —
// exactly when a fleet is least able to afford acting on that.
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
if len(lastUsed) == 0 {
return
}
r.mu.Lock()
defer r.mu.Unlock()
for id, at := range lastUsed {
rep, ok := r.replicas[id]
if !ok {
continue
}
// Only forwards. Two routers reporting, or a stale snapshot arriving
// late, must not move a replica's last use backwards.
if at.After(rep.LastUsedAt) {
rep.LastUsedAt = at
}
}
}
// IdleFor reports how long a replica has gone without a request, and whether
// that is known at all.
//
// The second return is not a detail. "Idle for an hour" and "nobody has told me
// yet" are different states, and anything that acts on idleness — eviction,
// most obviously — has to refuse to act on the second.
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[id]
if !ok || rep.LastUsedAt.IsZero() {
return 0, false
}
return r.now().Sub(rep.LastUsedAt), true
}
// expireStuckReplicas fails replicas that never finished coming up.
//
// Every other path out of Starting depends on the agent reporting something. A
// replica whose agent never reports — the start was a duplicate shard id and
// became a no-op, the node went quiet in a way expiry did not catch, the engine
// wedged before its first health check — sits in Starting for as long as the
// process lives, holding its GPUs and counting towards the model's replica
// count while serving nothing.
//
// Found on a real cluster: two replicas sat in Starting for eight minutes past
// a two-minute startup timeout, and nothing would ever have moved them.
func (r *Reconciler) expireStuckReplicas() {
now := r.now()
var stuck []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Starting && rep.State != Placing {
continue
}
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
continue
}
rep.State = Failed
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
stuck = append(stuck, rep)
}
r.mu.Unlock()
for _, rep := range stuck {
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
"replica", rep.ID, "model", rep.Model, "state", "starting",
"timeout", r.startupTimeout(rep))
}
}
// startupTimeout is the profile's, or the configured default when the profile
// does not say — a replica with no deadline at all is the thing this exists to
// prevent.
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
return p.StartupTimeout
}
}
// No profile deadline: a generous default, because the thing being prevented
// is a replica with no deadline at all rather than one with a loose one.
// Loading a large model across nodes genuinely takes tens of minutes.
return 30 * time.Minute
}
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
// replicas of nodes that are gone.
//
// The two-stage delay is deliberate: a brief network blip must not tear down
// replicas that are still serving traffic perfectly well, and rebuilding a
// multi-node replica costs tens of minutes of model loading.
func (r *Reconciler) expireNodes() {
now := r.now()
for _, n := range r.inv.Nodes() {
silent := now.Sub(n.LastSeen)
switch {
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
r.inv.SetState(n.ID, inventory.Unreachable)
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
}
}
}
// drainCordonedNodes moves work off nodes an operator is taking out of service.
//
// Draining a node is not the same as losing one: the engines are healthy and
// still serving, so their replicas are marked Draining rather than Failed. That
// makes the model look short of replicas, a replacement is placed elsewhere,
// and the original is stopped only once the replacement is serving — which is
// the difference between a rolling move and an outage.
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
var toDrain []*Replica
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
continue
}
for _, s := range rep.Shards {
n, ok := r.inv.Get(s.NodeID)
if ok && n.State == inventory.Draining {
rep.State = Draining
rep.DrainingAt = r.now()
toDrain = append(toDrain, rep)
break
}
}
}
shards := map[string][]Shard{}
for _, rep := range toDrain {
shards[rep.ID] = slices.Clone(rep.Shards)
}
r.mu.Unlock()
for _, rep := range toDrain {
r.log.Info("draining replica off a node being taken out of service",
"replica", rep.ID, "nodes", rep.NodeIDs())
for _, s := range shards[rep.ID] {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
r.mu.Lock()
defer r.mu.Unlock()
for _, rep := range r.replicas {
if rep.State.Terminal() || !rep.OnNode(nodeID) {
continue
}
rep.State = Failed
rep.Err = reason
}
}
// reapFailed releases the resources of terminal replicas and tells the agents
// to clean up whatever might still be running.
func (r *Reconciler) reapFailed(ctx context.Context) {
r.mu.Lock()
var doomed []*Replica
for _, rep := range r.replicas {
if rep.State.Terminal() {
doomed = append(doomed, rep)
}
}
for _, rep := range doomed {
delete(r.replicas, rep.ID)
if rep.State == Failed {
// Carry the failure count on the model, not the replica: the
// replica is about to disappear, and the backoff has to survive it.
r.attempts[rep.Model]++
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
}
}
r.mu.Unlock()
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", retry.Round(time.Second))
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
for _, s := range rep.Shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard of a dead replica failed",
"replica", rep.ID, "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(rep.ID)
}
}
// backoff grows with consecutive failures and is jittered, so a cluster-wide
// problem does not produce synchronized retry storms across every model.
func (r *Reconciler) backoff(attempt int) time.Duration {
if attempt <= 0 {
return 0
}
d := r.cfg.BaseBackoff << min(attempt-1, 16)
if d > r.cfg.MaxBackoff || d <= 0 {
d = r.cfg.MaxBackoff
}
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
return d - d/8 + jitter
}
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// Snapshot under the lock and work from values afterwards. A replica
// pointer stops being safe the moment it is in the map: an agent's status
// report can mutate it from another goroutine at any time.
r.mu.Lock()
var liveIDs, drainingIDs, avoid []string
for _, rep := range r.replicas {
if rep.Model != m.Name {
continue
}
switch {
case rep.State.Counts():
liveIDs = append(liveIDs, rep.ID)
avoid = append(avoid, rep.NodeIDs()...)
case rep.State == Draining:
drainingIDs = append(drainingIDs, rep.ID)
}
}
blockedUntil := r.nextAttempt[m.Name]
r.mu.Unlock()
desired := m.Scaling.Min
switch {
case len(liveIDs) < desired:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
// model that cannot start — a missing weights path, say — and bury
// the real error in noise.
return
}
r.scaleUp(ctx, m, avoid)
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
}
// A drained replica is stopped once its replacement is serving, or once the
// grace period expires. Stopping it the moment it was drained would cut
// in-flight requests, which is the thing draining exists to avoid.
replaced := len(liveIDs) >= desired
for _, id := range drainingIDs {
r.mu.Lock()
rep, ok := r.replicas[id]
var expired bool
if ok {
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > r.cfg.DrainGrace
}
r.mu.Unlock()
switch {
case !ok:
case replaced:
r.stopReplica(ctx, id)
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", r.cfg.DrainGrace)
r.stopReplica(ctx, id)
}
}
}
// scaleUp places and starts one replica per pass.
//
// One at a time on purpose: placement decisions depend on what is already
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
res := r.sched.Place(m.PlacementProfiles(), avoid)
if res.Placement == nil && len(avoid) > 0 {
// Spreading across failure domains is a preference, not a rule. If it
// is the only thing standing in the way, take the placement anyway:
// a second replica on one node still survives an engine crash.
res = r.sched.Place(m.PlacementProfiles(), nil)
}
r.mu.Lock()
r.lastExplain[m.Name] = res
r.mu.Unlock()
if res.Placement == nil {
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
return
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
r.log.Error("starting replica failed", "model", m.Name, "err", err)
}
}
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
prof, ok := m.ProfileByName(pl.Profile.Name)
if !ok {
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
}
espec, ok := m.EngineProfile(prof.Name)
if !ok {
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
}
r.mu.Lock()
r.seq++
id := replicaID(m.Name, r.seq)
attempt := r.attempts[m.Name]
r.mu.Unlock()
rep := &Replica{
ID: id, Model: m.Name, Profile: prof.Name,
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
WantShards: len(pl.Nodes),
}
// The leader's address and port are fixed before any shard starts: every
// rank needs the same rendezvous point, and a worker that starts first must
// already know where to dial.
leader := pl.Nodes[0]
leaderNode, ok := r.inv.Get(leader.NodeID)
if !ok {
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
}
basePort := r.allocatePorts(pl)
for _, a := range pl.Nodes {
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
rep.Shards = append(rep.Shards, Shard{
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
HTTPPort: basePort, State: supervisor.Pending,
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
// Everything read from rep has to be read now: once it is in the map an
// agent status report can mutate it concurrently, and the race detector is
// right to object.
//
// The leader first, then workers ascending. Both engines put the rendezvous
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
// worker started first has nothing to connect to and spends its startup
// budget retrying.
//
// An earlier version started workers first, on the theory that the head
// should wait for them. That is backwards: vLLM's server blocks until the
// Ray cluster has the GPUs it needs, so the leader can be started
// immediately and will wait on its own.
order := slices.Clone(rep.Shards)
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
nodeIDs := rep.NodeIDs()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
for _, s := range order {
cmd := ShardCommand{
ReplicaID: id,
ShardID: s.ID,
NodeID: s.NodeID,
Spec: engines.ShardSpec{
ReplicaID: id,
ShardID: s.ID,
Rank: s.Rank,
NNodes: len(pl.Nodes),
LeaderAddr: addrOf(leaderNode),
LeaderPort: basePort + 1,
GPUIndices: s.GPUIndices,
HCAs: s.HCAs,
MgmtInterface: r.cfg.MgmtInterface,
HTTPPort: s.HTTPPort,
Profile: espec,
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
Fabric: pl.Fabric,
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
return fmt.Errorf("starting shard %s: %w", s.ID, err)
}
r.markShard(id, s.ID, supervisor.Starting, "")
}
r.log.Info("replica placed",
"replica", id, "model", m.Name, "profile", prof.Name,
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
return nil
}
// replicaID names a replica uniquely for the life of the cluster, not the life
// of this process.
//
// The counter alone is not enough, and the way it fails is ugly. A control
// plane that restarts begins counting again, so the replica it places next is
// called llama-1 — which is also what an engine started before the restart and
// still running on some other node calls itself. Two nodes then claim the same
// shard id, adoption hands the id back and forth between them, and every
// placement lands on a port an orphan is already holding. Observed on a real
// cluster as a bind-conflict loop that never settled.
//
// The random suffix costs nothing and makes the collision impossible.
func replicaID(model string, seq int) string {
var buf [4]byte
if _, err := crypterand.Read(buf[:]); err != nil {
// Randomness failing is not a reason to stop placing replicas; the
// counter alone is still unique within this process.
return fmt.Sprintf("%s-%d", model, seq)
}
return fmt.Sprintf("%s-%x", model, buf)
}
// allocatePorts picks a port shared by every shard of a replica.
//
// The same port on every node keeps the leader address predictable and the
// port bookkeeping trivial; nodes do not share a network namespace, so there is
// no conflict in reusing it.
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
used := map[int]struct{}{}
r.mu.Lock()
for _, rep := range r.replicas {
for _, s := range rep.Shards {
used[s.HTTPPort] = struct{}{}
}
}
// Ports an agent has reported but nothing owns — a stray mid-shutdown, an
// engine whose replica was just reaped. The process is still listening
// whatever the bookkeeping says, and handing its port to a new shard means
// that shard fails to bind and takes the whole replica with it.
for p := range r.reservedPorts {
used[p] = struct{}{}
}
r.mu.Unlock()
// Two ports per replica: HTTP and the distributed init rendezvous.
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
if _, taken := used[p]; !taken {
return p
}
}
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
continue
}
rep.State = Draining
rep.DrainingAt = r.now()
shards := slices.Clone(rep.Shards)
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
return
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
r.mu.Unlock()
for _, s := range shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(id)
r.mu.Lock()
delete(r.replicas, id)
r.mu.Unlock()
}
// Adopt rebuilds replica state from shards an agent reports it is already
// running (plan §6.5).
//
// This is what makes a control plane restart survivable. Without it the
// reconciler sees an empty cluster, places duplicates, and the engines already
// running keep holding GPUs the inventory believes are free — so the duplicates
// fail to allocate, with an error naming the new process rather than the old.
//
// Shards whose model or profile is no longer in the catalog are returned for
// the caller to stop: nothing will ever claim them.
func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) {
defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards)
byReplica := map[string][]AdoptedShard{}
for _, s := range shards {
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
}
r.mu.Lock()
defer r.mu.Unlock()
for replicaID, group := range byReplica {
if existing, known := r.replicas[replicaID]; known {
// The other half of a replica we are still assembling. Merge rather
// than choose: these shards belong to the same replica as the ones
// already adopted, they are simply on a different node whose agent
// registered a moment later. Treating them as a competing claim is
// what tore multi-node replicas apart on every control plane
// restart — one half adopted and failed for being short, the other
// stopped as unclaimed when it finally arrived.
if !existing.AdoptedAt.IsZero() && len(existing.Shards) < existing.WantShards {
r.mergeAdopted(existing, group)
continue
}
// Usually this is an agent reconnecting rather than a control plane
// restart, and its status reports keep the state current.
//
// Not always. Replica ids are derived from the model name and a
// counter, so a control plane that restarts and places before the
// agents reconnect produces a *different* replica under the same
// id, on a freshly allocated port. Skipping the adoption then
// leaves the cluster pointing at a port nothing listens on, while
// the engine that is actually serving is treated as a stray.
//
// The fleet is authoritative about what is running (plan §6.5), so
// what we merely placed loses to what an agent reports — unless our
// copy has a shard that has actually come up, in which case two
// things really are running and the one we did not ask for is the
// stray.
if existing.hasLiveShard() {
// Only what we do not recognise. A reconnecting agent reports
// the shards we already know, and disowning those would tell it
// to stop the very replica that is serving.
known := map[string]struct{}{}
for _, sh := range existing.Shards {
known[sh.ID] = struct{}{}
}
var strays []AdoptedShard
for _, s := range group {
if _, ok := known[s.ShardID]; !ok {
strays = append(strays, s)
}
}
unclaimed = append(unclaimed, strays...)
r.reservePorts(strays)
continue
}
r.log.Warn("a running replica reported by the fleet replaces one we placed",
"replica", replicaID, "placed_endpoint", existing.Endpoint)
for _, s := range existing.Shards {
r.inv.Release(s.ID)
}
delete(r.replicas, replicaID)
}
first := group[0]
m, ok := r.cat.Load().Resolve(first.Model)
if !ok {
unclaimed = append(unclaimed, group...)
r.reservePorts(group)
continue
}
prof, ok := m.ProfileByName(first.Profile)
if !ok {
unclaimed = append(unclaimed, group...)
r.reservePorts(group)
continue
}
rep := &Replica{
ID: replicaID, Model: m.Name, Profile: prof.Name,
Fabric: first.Fabric, Pool: first.Pool,
Weight: prof.Weight, CreatedAt: r.now(),
Endpoint: first.Endpoint,
// From the profile, not from what was reported: the point is to
// notice when what was reported is short.
WantShards: max(prof.Nodes, 1),
AdoptedAt: r.now(),
}
r.mergeAdopted(rep, group)
r.replicas[replicaID] = rep
}
return unclaimed
}
// mergeAdopted folds shards an agent reports into a replica being reassembled.
//
// Called both for the first group, which creates the replica, and for every
// later group belonging to it, so a replica spanning N nodes is built up over N
// registrations. Shard ids already present are skipped: an agent that
// reconnects repeats what it is running, and counting those again would carry a
// replica past its shard count without a second node ever appearing.
func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
have := make(map[string]struct{}, len(rep.Shards))
for _, sh := range rep.Shards {
have[sh.ID] = struct{}{}
}
for _, s := range group {
if _, dup := have[s.ShardID]; dup {
continue
}
have[s.ShardID] = struct{}{}
rep.Shards = append(rep.Shards, Shard{
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
HTTPPort: s.HTTPPort, State: s.State,
})
// Reclaim the GPUs, or the next placement will hand them out twice.
r.inv.Allocate(s.NodeID, inventory.Allocation{
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
})
// The endpoint comes from the leader, which may be in any of the
// groups — the rank 0 node is not necessarily the first to register.
if s.Rank == 0 && s.Endpoint != "" {
rep.Endpoint = s.Endpoint
}
}
slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank })
rep.deriveState()
r.log.Info("adopted replica from the fleet",
"replica", rep.ID, "model", rep.Model,
"shards", len(rep.Shards), "want", rep.WantShards, "state", rep.State)
}
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
// agent, having just told us everything it is running, did not mention.
//
// The mirror image of adoption, and the half that was missing. Adoption handles
// "the agent has something we do not know about"; this handles "we think the
// agent has something it does not". Both follow from the same rule: the fleet is
// authoritative about what is running (plan §6.5).
//
// Without it, an engine that dies while its agent survives leaves the control
// plane believing a replica is Ready forever. Observed on a real cluster: the
// engine was killed, the agent restarted and registered with nothing running,
// and the replica stayed Ready pointing at a dead port for as long as anyone
// watched. Traffic survived — the breaker took the endpoint out — but the
// cluster ran a third short with no sign of it.
//
// Safe against the obvious race. A registration follows a fresh connection, so
// anything we dispatched over a previous one is not running by definition;
// there is no window where a shard is legitimately started but unreported.
func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) {
if nodeID == "" {
return
}
reported := make(map[string]struct{}, len(running))
for _, s := range running {
reported[s.ShardID] = struct{}{}
}
var lost []string
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State == Stopped {
continue
}
changed := false
for i := range rep.Shards {
sh := &rep.Shards[i]
if sh.NodeID != nodeID || sh.State.Terminal() {
continue
}
if _, ok := reported[sh.ID]; ok {
continue
}
sh.State = supervisor.Failed
sh.Err = "the agent reports it is not running"
r.inv.Release(sh.ID)
changed = true
lost = append(lost, sh.ID)
}
if changed {
rep.deriveState()
}
}
r.mu.Unlock()
for _, id := range lost {
r.log.Warn("a shard we believed was running is not; the agent does not have it",
"node", nodeID, "shard", id)
}
}
// reservePorts records ports held by shards nothing owns.
//
// They are being stopped, but stopping is not instant and the engine keeps its
// socket until it exits. Handing the port to a new shard in the meantime makes
// that shard fail to bind, which fails its replica, which places another one on
// the same port — a loop that does not end on its own.
//
// Called with the lock held.
func (r *Reconciler) reservePorts(shards []AdoptedShard) {
for _, s := range shards {
if s.HTTPPort > 0 {
r.reservedPorts[s.HTTPPort] = struct{}{}
}
}
}
// AdoptedShard is one running shard an agent reported at registration.
type AdoptedShard struct {
NodeID string
ReplicaID string
ShardID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Model string
Profile string
Pool string
Fabric string
// Endpoint is the leader's base URL, set on the rank 0 shard.
Endpoint string
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
r.Notify()
}
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID != shardID {
continue
}
rep.Shards[i].State = st
if errMsg != "" {
rep.Shards[i].Err = errMsg
}
break
}
prev := rep.State
rep.deriveState()
if rep.State == Ready && prev != Ready {
// A model that comes up clears its backoff; the next failure starts
// counting from scratch rather than from an old streak.
r.attempts[rep.Model] = 0
delete(r.nextAttempt, rep.Model)
}
}
// Replicas returns a snapshot of every replica, sorted by id.
func (r *Reconciler) Replicas() []Replica {
r.mu.Lock()
defer r.mu.Unlock()
out := make([]Replica, 0, len(r.replicas))
for _, rep := range r.replicas {
clone := *rep
clone.Shards = slices.Clone(rep.Shards)
out = append(out, clone)
}
slices.SortFunc(out, func(a, b Replica) int {
if a.ID < b.ID {
return -1
}
if a.ID > b.ID {
return 1
}
return 0
})
return out
}
// ReadyReplicas returns the replicas the router may send traffic to.
func (r *Reconciler) ReadyReplicas() []Replica {
var out []Replica
for _, rep := range r.Replicas() {
if rep.State == Ready {
out = append(out, rep)
}
}
return out
}
// Explain returns the most recent placement result for a model, which is what
// `ctl explain-placement` prints.
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
r.mu.Lock()
defer r.mu.Unlock()
res, ok := r.lastExplain[model]
return res, ok
}
func addrOf(n inventory.Node) string {
if n.MgmtAddr != "" {
return n.MgmtAddr
}
if n.Hostname != "" {
return n.Hostname
}
return n.ID
}