package reconciler import ( "context" crypterand "crypto/rand" "fmt" "log/slog" "math/rand/v2" "slices" "sync" "sync/atomic" "time" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" "gitea.mynodes.xyz/ilker/llmgw/internal/control/quota" "gitea.mynodes.xyz/ilker/llmgw/internal/notify" "gitea.mynodes.xyz/ilker/llmgw/internal/obs" ) // ShardCommand is everything an agent needs to launch one shard. type ShardCommand struct { ReplicaID string ShardID string NodeID string Spec engines.ShardSpec // StartupTimeout comes from the profile; model loading dominates it. StartupTimeout time.Duration // Pool and Fabric are control plane concepts the agent cannot derive. They // travel with the command so the agent can report them back after a // control plane restart (plan §6.5). Pool string Fabric string // ProfileFP travels for the same reason, and answers a different question: // whether the profile has been edited since this shard was started. ProfileFP string } // Dispatcher sends commands to agents. // // It is an interface so the reconciliation loop can be tested without a network: // the loop's logic is where the interesting failures are, not the transport. type Dispatcher interface { StartShard(ctx context.Context, cmd ShardCommand) error StopShard(ctx context.Context, nodeID, shardID string) error DrainShard(ctx context.Context, nodeID, shardID string) error } // Config tunes the reconciliation loop. type Config struct { // Location is the time zone models' scaling schedules are written in; // nil is the control plane's own. Location *time.Location // Interval is how often desired and actual are compared when nothing else // happens. Events wake the loop sooner. Interval time.Duration // BaseBackoff is how long to wait before rebuilding a failed replica. BaseBackoff time.Duration // MaxBackoff caps the exponential growth. MaxBackoff time.Duration // UnreachableAfter is how long without a heartbeat before a node is // unreachable. UnreachableAfter time.Duration // LostAfter is how long unreachable before its replicas are declared dead. // Longer than UnreachableAfter on purpose: a brief network blip must not // tear down a replica that is still serving perfectly well. LostAfter time.Duration // DrainGrace bounds how long a draining replica waits for its replacement. // Without a bound, draining a node whose model cannot be placed anywhere // else would never finish. DrainGrace time.Duration // PreemptGrace is how long a replica stopped to make room for another // is given to finish what it is serving: a request, not a replacement. PreemptGrace time.Duration // PreemptHold is how long a model that gave up room may not take it back, // so the replica it was given up for is placed first. PreemptHold time.Duration // ScaleUpWaiting is the average number of requests waiting per ready // replica, held for ScaleUpAfter, that adds a replica — up to the model's // max. ScaleDownAfter is how long with nothing waiting before one of the // replicas added that way is taken away again, down to its min. ScaleUpWaiting float64 ScaleUpAfter time.Duration ScaleDownAfter time.Duration // ParkAfter is how many failures in a row stop a model being retried. // // Backoff alone never gives up: it grows to MaxBackoff and keeps trying, // and a model whose weights its engine cannot load — an architecture the // installed llama.cpp does not know — was started 155 times over eight // hours on the live cluster, each time failing the same way, with nothing // to show for it but the log. A parked model is not placed again until its // definition changes or an operator runs `llmgw ctl retry`. ParkAfter int // PortRange is the per-node port range for engine HTTP and dist-init. PortRange [2]int // MgmtInterface is the ethernet interface used for NCCL bootstrap. MgmtInterface string // EngineKey gives each replica the key its engine demands. Derived from // the replica id rather than stored, so a control plane that restarts — // or a second one — computes the same key for a replica it adopts. Nil // starts engines without one. EngineKey func(replicaID string) string // EvictionGrace is how long a replica must have gone unasked-for before it // may be stopped to make room for another model. // // Shorter than IdleTimeout on purpose: a replica idle for five minutes is // not worth reclaiming on its own, and is worth reclaiming when something // else needs the cards. Still minutes rather than seconds, because stopping // one throws away the KV cache prefix routing spent filling. EvictionGrace time.Duration // MinLifetime is how long a replica is safe from eviction after it starts // serving, whatever else wants its cards. // // The thing that stops two models on a cluster with room for one from // taking turns evicting each other. A load costs minutes; oscillating at a // load per cycle is worse for both models than one of them waiting. MinLifetime time.Duration // IdleTimeout is how long an on-demand model stays loaded with nobody // asking for it. A model may override it. IdleTimeout time.Duration // SettleFor is how long after start to leave placement alone while agents // reconnect. // // A control plane that has just started knows nothing, and the first // reconcile tick runs before any agent has registered. It sees an empty // fleet and acts on it: "needs 2 nodes on one fabric, but no node reports a // fabric" is that, and it is harmless only because adoption catches up. // What is not harmless is placing on the fraction of the fleet that // happens to have reported first — after one restart both replicas of a // model landed on the single node that had registered, when four were // coming. // // Adoption still runs throughout; only placement waits. The window ends // early once the fleet stops growing, so a small cluster does not sit idle // for the whole duration — but not too early: an agent that lost its // connection reconnects on a backoff reaching thirty seconds, so "no new // node for a few seconds" means nothing. SettleFor time.Duration } // DefaultConfig returns sensible defaults. func DefaultConfig() Config { return Config{ Interval: 5 * time.Second, BaseBackoff: 10 * time.Second, MaxBackoff: 10 * time.Minute, UnreachableAfter: 15 * time.Second, LostAfter: 60 * time.Second, SettleFor: 60 * time.Second, IdleTimeout: 15 * time.Minute, EvictionGrace: 2 * time.Minute, MinLifetime: 5 * time.Minute, DrainGrace: 5 * time.Minute, PreemptGrace: 30 * time.Second, PreemptHold: 5 * time.Minute, ParkAfter: 5, ScaleUpWaiting: 1, ScaleUpAfter: time.Minute, ScaleDownAfter: 10 * time.Minute, PortRange: [2]int{20000, 20999}, MgmtInterface: "eth0", } } func (c *Config) normalize() { d := DefaultConfig() if c.Interval <= 0 { c.Interval = d.Interval } if c.BaseBackoff <= 0 { c.BaseBackoff = d.BaseBackoff } if c.PreemptGrace <= 0 { c.PreemptGrace = d.PreemptGrace } if c.PreemptHold <= 0 { c.PreemptHold = d.PreemptHold } if c.MaxBackoff < c.BaseBackoff { c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff) } if c.IdleTimeout <= 0 { c.IdleTimeout = d.IdleTimeout } if c.EvictionGrace <= 0 { c.EvictionGrace = d.EvictionGrace } if c.MinLifetime <= 0 { c.MinLifetime = d.MinLifetime } if c.SettleFor < 0 { c.SettleFor = 0 } else if c.SettleFor == 0 { c.SettleFor = d.SettleFor } if c.UnreachableAfter <= 0 { c.UnreachableAfter = d.UnreachableAfter } if c.LostAfter < c.UnreachableAfter { c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter) } if c.DrainGrace <= 0 { c.DrainGrace = d.DrainGrace } if c.ParkAfter <= 0 { c.ParkAfter = d.ParkAfter } if c.ScaleUpWaiting <= 0 { c.ScaleUpWaiting = d.ScaleUpWaiting } if c.ScaleUpAfter <= 0 { c.ScaleUpAfter = d.ScaleUpAfter } if c.ScaleDownAfter <= 0 { c.ScaleDownAfter = d.ScaleDownAfter } if c.PortRange[1] <= c.PortRange[0] { c.PortRange = d.PortRange } if c.MgmtInterface == "" { c.MgmtInterface = d.MgmtInterface } } // Reconciler compares the catalog against what is running and closes the gap. type Reconciler struct { // leader gates every action; nil means act unconditionally. leader Leader cfg Config cat *catalog.Live inv *inventory.Store sched *placement.Scheduler disp Dispatcher log *slog.Logger now func() time.Time // notify tells people about parked models and lost nodes; nil tells // no one. notify atomic.Pointer[notify.Notifier] mu sync.Mutex replicas map[string]*Replica // nextAttempt is the earliest a model may be rebuilt, per model. nextAttempt map[string]time.Time // attempts counts consecutive failures per model, for the backoff. attempts map[string]int // lastExplain keeps the most recent placement failure per model, so // `ctl explain-placement` can answer without re-running placement. lastExplain map[string]placement.Result seq int // reservedPorts holds ports the fleet reports as in use but no replica // claims, so the allocator cannot hand out what something is still bound to. reservedPorts map[int]struct{} // wanted records when an on-demand model was last asked for. // // The whole of on-demand loading, kept here rather than in the catalog // because it is state about this moment and not about what the model is. // A model nobody has wanted has no entry and is simply not running. wanted map[string]time.Time // restartQueue holds, per model, the replicas an operator asked to be // restarted and not yet begun. One is replaced at a time: replacing them // all at once would need the memory of every replica twice over. restartQueue map[string][]string // restartInPlace are the replicas asked to restart by stopping first. restartInPlace map[string]bool // restartRefused says, by replica, why a restart was given up rather // than leave the model without it. restartRefused map[string]string // parked holds models that failed ParkAfter times in a row. parked map[string]parkRecord // extra is how many replicas load has added above a model's base, and // busySince/quietSince when its load last crossed the thresholds. extra map[string]int busySince map[string]time.Time quietSince map[string]time.Time // placementLogged remembers the last "no placement" said per model, so it // is said when it changes rather than on every five-second pass. placementLogged map[string]placementNote // preempting is, per model, the replicas stopped to make room for it and // not gone yet; held is, per model that gave them up, until when it may // not take the room back; preempted counts, per model, replicas it lost. preempting map[string][]string held map[string]hold preempted map[string]int // quotas is the tenancy tree's capacity quotas as last read; nil is none. quotas func() *quota.Policy // measurements are benchmark results by model/profile; nil is none. measurements func() map[string]Measured wake chan struct{} stop chan struct{} done chan struct{} once sync.Once // startedAt and lastNodeCount track the settling window: placement waits // until the fleet has stopped growing, or the window expires. startedAt time.Time lastNodeCount int steadySince time.Time } // New builds a reconciler. func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler { cfg.normalize() if log == nil { log = slog.Default() } return &Reconciler{ cfg: cfg, cat: cat, inv: inv, disp: disp, log: log, sched: placement.New(inv), now: time.Now, replicas: map[string]*Replica{}, nextAttempt: map[string]time.Time{}, attempts: map[string]int{}, reservedPorts: map[int]struct{}{}, wanted: map[string]time.Time{}, restartQueue: map[string][]string{}, restartInPlace: map[string]bool{}, restartRefused: map[string]string{}, parked: map[string]parkRecord{}, extra: map[string]int{}, busySince: map[string]time.Time{}, quietSince: map[string]time.Time{}, placementLogged: map[string]placementNote{}, lastExplain: map[string]placement.Result{}, preempting: map[string][]string{}, held: map[string]hold{}, preempted: map[string]int{}, wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), startedAt: time.Now(), } } // SetClock replaces the time source, for tests. func (r *Reconciler) SetClock(now func() time.Time) { r.now = now r.startedAt = now() } // Leader reports whether this process should be acting on the fleet. type Leader interface{ IsLeader() bool } // SetLeadership gates every action on holding leadership. // // A setter rather than a constructor argument because the thing that decides // leadership needs a store, and the store is wired after the controller. Nil // leaves the reconciler acting unconditionally, which is what a single control // plane wants and what every test expects. func (r *Reconciler) SetLeadership(l Leader) { r.leader = l } // Start runs the loop until Close. func (r *Reconciler) Start(ctx context.Context) { go func() { defer close(r.done) t := time.NewTicker(r.cfg.Interval) defer t.Stop() r.Reconcile(ctx) for { select { case <-r.stop: return case <-ctx.Done(): return case <-r.wake: r.Reconcile(ctx) case <-t.C: r.Reconcile(ctx) } } }() } // Close stops the loop. func (r *Reconciler) Close() { r.once.Do(func() { close(r.stop) <-r.done }) } // Notify wakes the loop, for when something changed and waiting for the next // tick would add pointless latency. func (r *Reconciler) Notify() { select { case r.wake <- struct{}{}: default: } } // Reconcile performs one pass. It is exported so tests can drive it directly // instead of racing a ticker. func (r *Reconciler) Reconcile(ctx context.Context) { // A standby does nothing at all here. Everything below issues commands to // agents — placing, evicting, draining, reaping — and two control planes // doing that against one fleet is how a model ends up with twice the // replicas it asked for, each holding GPUs the other believes are free. // // Unset means leader, so a single control plane and every test behave // exactly as they did before leadership existed. if r.leader != nil && !r.leader.IsLeader() { return } ctx, p := beginPass(ctx) defer p.end() r.expireNodes(ctx) r.expireStuckReplicas() r.drainCordonedNodes(ctx) r.reapFailed(ctx) // Everything above is about what is already running and stays correct on a // partial fleet. Placement is the part that must not act on one. if !r.fleetHasSettled() { return } for _, m := range r.cat.Load().Models() { r.reconcileModel(ctx, m) } r.stopReplicasOfRemovedModels(ctx) } // stopReplicasOfRemovedModels tears down what the catalog no longer asks for. // // reconcileModel only ever visits models that are in the catalog, so a replica // whose model has been removed was visited by nothing: it kept its GPUs, kept // counting towards the inventory, and kept running an engine for a name no // client could ask for. The endpoint publisher already skipped it with the // comment "the reconciler will tear it down", which was simply not true. // // It mattered little while a catalog could only change by restarting the // process. It matters now that models come and go while the cluster runs, which // is the whole point of making the catalog live. // // Ordered after the model loop so a rebuild elsewhere has already claimed what // it needs: freeing these GPUs first would let a replacement land on hardware // that is about to be freed anyway, and churn two nodes instead of one. func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) { cat := r.cat.Load() r.mu.Lock() var gone []*Replica for _, rep := range r.replicas { if _, ok := cat.Resolve(rep.Model); !ok { gone = append(gone, rep) } } r.mu.Unlock() for _, rep := range gone { r.log.Info("stopping a replica whose model left the catalog", "replica", rep.ID, "model", rep.Model) r.stopReplica(ctx, rep.ID, "its model left the catalog") } } // fleetHasSettled reports whether placement may run yet. // // Two ways out, because either alone is wrong. Waiting for the window to expire // would make every restart cost that much downtime even when the whole fleet // reconnected in a second. Waiting only for the count to stop changing would // hang forever on a cluster whose nodes never all come back. func (r *Reconciler) fleetHasSettled() bool { if r.cfg.SettleFor <= 0 { return true } now := r.now() if now.Sub(r.startedAt) >= r.cfg.SettleFor { return true } n := len(r.inv.Nodes()) if n != r.lastNodeCount { r.lastNodeCount, r.steadySince = n, now return false } // Steady for a moment, and at least one node present. A count that has been // zero since start is not a settled fleet, it is one that has not arrived. if n == 0 { return false } // Half the window, not a quarter of it, and the window is a minute. // // Measured on the cluster: after a restart node05 registered in one second // and the other four arrived seventeen seconds later, because a // disconnected agent reconnects on a backoff that reaches thirty seconds. // A five-second steady period declared a fleet of one settled and resumed // placement while four nodes were still backing off — the exact situation // this exists to avoid, reached by being impatient about it. settled := now.Sub(r.steadySince) >= r.cfg.SettleFor/2 if settled { r.log.Info("fleet settled; placement resuming", "nodes", n, "waited", now.Sub(r.startedAt).Round(time.Millisecond)) r.startedAt = time.Time{} // never wait again in this process } return settled } // expireStuckReplicas fails replicas that never finished coming up. // // Every other path out of Starting depends on the agent reporting something. A // replica whose agent never reports — the start was a duplicate shard id and // became a no-op, the node went quiet in a way expiry did not catch, the engine // wedged before its first health check — sits in Starting for as long as the // process lives, holding its GPUs and counting towards the model's replica // count while serving nothing. // // Found on a real cluster: two replicas sat in Starting for eight minutes past // a two-minute startup timeout, and nothing would ever have moved them. func (r *Reconciler) expireStuckReplicas() { now := r.now() var stuck []*Replica r.mu.Lock() for _, rep := range r.replicas { if rep.State != Starting && rep.State != Placing { continue } if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) { continue } rep.State = Failed rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep)) rep.startDone(now) stuck = append(stuck, rep) } r.mu.Unlock() for _, rep := range stuck { r.log.Warn("replica did not come up in time; failing it so it can be replaced", "replica", rep.ID, "model", rep.Model, "state", "starting", "timeout", r.startupTimeout(rep)) } } // startupTimeout is the profile's, or the configured default when the profile // does not say — a replica with no deadline at all is the thing this exists to // prevent. func (r *Reconciler) startupTimeout(rep *Replica) time.Duration { if m, ok := r.cat.Load().Resolve(rep.Model); ok { if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 { return p.StartupTimeout } } // No profile deadline: a generous default, because the thing being prevented // is a replica with no deadline at all rather than one with a loose one. // Loading a large model across nodes genuinely takes tens of minutes. return 30 * time.Minute } // expireNodes moves silent nodes through Unreachable to Lost, and fails the // replicas of nodes that are gone. // // The two-stage delay is deliberate: a brief network blip must not tear down // replicas that are still serving traffic perfectly well, and rebuilding a // multi-node replica costs tens of minutes of model loading. func (r *Reconciler) expireNodes(ctx context.Context) { now := r.now() for _, n := range r.inv.Nodes() { silent := now.Sub(n.LastSeen) switch { case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter: r.inv.SetState(n.ID, inventory.Unreachable) r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second)) _, span := startSpan(ctx, "node unreachable", attribute.String("llmgw.node", n.ID), attribute.Float64("llmgw.silent_seconds", silent.Seconds())) span.End() case n.State == inventory.Unreachable && silent > r.cfg.LostAfter: _, span := startSpan(ctx, "node lost", attribute.String("llmgw.node", n.ID), attribute.Float64("llmgw.silent_seconds", silent.Seconds())) r.inv.SetState(n.ID, inventory.Lost) r.log.Error("node lost; its replicas will be rebuilt elsewhere", "node", n.ID, "silent_for", silent.Round(time.Second)) r.notify.Load().Notify(notify.NodeLost, n.ID, fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)), map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())}) span.SetAttributes(attribute.StringSlice("llmgw.replicas_failed", r.failReplicasOn(n.ID, "node "+n.ID+" was lost"))) span.End() } } } // drainCordonedNodes moves work off nodes an operator is taking out of service. // // Draining a node is not the same as losing one: the engines are healthy and // still serving, so their replicas are marked Draining rather than Failed. That // makes the model look short of replicas, a replacement is placed elsewhere, // and the original is stopped only once the replacement is serving — which is // the difference between a rolling move and an outage. func (r *Reconciler) drainCordonedNodes(ctx context.Context) { var toDrain []*Replica r.mu.Lock() for _, rep := range r.replicas { if rep.State != Ready && rep.State != Starting && rep.State != Placing { continue } for _, s := range rep.Shards { n, ok := r.inv.Get(s.NodeID) if ok && n.State == inventory.Draining { rep.State = Draining rep.DrainingAt = r.now() toDrain = append(toDrain, rep) break } } } shards := map[string][]Shard{} for _, rep := range toDrain { shards[rep.ID] = slices.Clone(rep.Shards) } r.mu.Unlock() for _, rep := range toDrain { r.log.Info("draining replica off a node being taken out of service", "replica", rep.ID, "nodes", rep.NodeIDs()) r.drainShards(ctx, rep.ID, shards[rep.ID], "its node is being drained") } } // failReplicasOn fails every replica on a node, and says which. func (r *Reconciler) failReplicasOn(nodeID, reason string) []string { r.mu.Lock() defer r.mu.Unlock() var ids []string for _, rep := range r.replicas { if rep.State.Terminal() || !rep.OnNode(nodeID) { continue } rep.State = Failed rep.Err = reason rep.startDone(r.now()) ids = append(ids, rep.ID) } slices.Sort(ids) return ids } // reapFailed releases the resources of terminal replicas and tells the agents // to clean up whatever might still be running. func (r *Reconciler) reapFailed(ctx context.Context) { r.mu.Lock() var doomed []*Replica for _, rep := range r.replicas { if rep.State.Terminal() { doomed = append(doomed, rep) } } parkedNow := map[string]bool{} for _, rep := range doomed { delete(r.replicas, rep.ID) rep.startDone(r.now()) if rep.State == Failed { // Carry the failure count on the model, not the replica: the // replica is about to disappear, and the backoff has to survive it. r.attempts[rep.Model]++ r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model])) if n := r.attempts[rep.Model]; n >= r.cfg.ParkAfter { if _, already := r.parked[rep.Model]; !already { m, _ := r.cat.Load().Resolve(rep.Model) r.parked[rep.Model] = parkRecord{ Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m), } parkedNow[rep.ID] = true r.log.Error("model parked: it failed every time it started, and is not retried "+ "until its definition changes or `llmgw ctl retry` is run", "model", rep.Model, "failures", n, "last_error", rep.Err) r.notify.Load().Notify(notify.ModelParked, rep.Model, fmt.Sprintf("model %s is parked: it failed %d times in a row (%s); `llmgw ctl retry %s` tries again", rep.Model, n, rep.Err, rep.Model), map[string]any{"model": rep.Model, "failures": n, "last_error": rep.Err}) } } } } r.mu.Unlock() for _, rep := range doomed { // rep is out of the map now, so no agent report can reach it any more. sctx, span := startSpan(ctx, "reap replica", replicaAttrs(rep)...) span.SetAttributes(attribute.String("llmgw.state", rep.State.String())) if rep.State == Failed { r.mu.Lock() attempts := r.attempts[rep.Model] retry := r.backoff(attempts) r.mu.Unlock() r.log.Error("replica failed", "replica", rep.ID, "model", rep.Model, "err", rep.Err, "retry_in", retry.Round(time.Second)) span.SetAttributes( attribute.String("llmgw.error", rep.Err), attribute.Int("llmgw.failures_in_a_row", attempts), attribute.Float64("llmgw.retry_in_seconds", retry.Seconds()), attribute.Bool("llmgw.parked", parkedNow[rep.ID]), ) } // Stop every shard, including the ones that looked healthy: a partially // alive replica holds GPUs while serving nothing. for _, s := range rep.Shards { if err := r.disp.StopShard(sctx, s.NodeID, s.ID); err != nil { r.log.Warn("stopping a shard of a dead replica failed", "replica", rep.ID, "shard", s.ID, "err", err) } } r.inv.ReleaseReplica(rep.ID) span.End() } } // drainShards tells a replica's nodes to stop routing to its shards, as one // "drain replica" span saying why. func (r *Reconciler) drainShards(ctx context.Context, id string, shards []Shard, why string) { nodes := make([]string, 0, len(shards)) for _, s := range shards { nodes = append(nodes, s.NodeID) } ctx, span := startSpan(ctx, "drain replica", attribute.String("llmgw.replica", id), attribute.StringSlice("llmgw.nodes", nodes), attribute.String("llmgw.reason", why)) defer span.End() for _, s := range shards { if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil { r.log.Warn("draining a shard failed", "shard", s.ID, "err", err) } } } // backoff grows with consecutive failures and is jittered, so a cluster-wide // problem does not produce synchronized retry storms across every model. func (r *Reconciler) backoff(attempt int) time.Duration { if attempt <= 0 { return 0 } d := r.cfg.BaseBackoff << min(attempt-1, 16) if d > r.cfg.MaxBackoff || d <= 0 { d = r.cfg.MaxBackoff } jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter return d - d/8 + jitter } func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) { // Snapshot under the lock and work from values afterwards. A replica // pointer stops being safe the moment it is in the map: an agent's status // report can mutate it from another goroutine at any time. r.mu.Lock() var liveIDs, drainingIDs, retiringIDs, avoid []string readyLive := 0 for _, rep := range r.replicas { if rep.Model != m.Name { continue } switch { case rep.State.Counts() && !rep.RetiringAt.IsZero(): retiringIDs = append(retiringIDs, rep.ID) avoid = append(avoid, rep.NodeIDs()...) case rep.State.Counts(): liveIDs = append(liveIDs, rep.ID) avoid = append(avoid, rep.NodeIDs()...) if rep.State == Ready { readyLive++ } case rep.State == Draining: drainingIDs = append(drainingIDs, rep.ID) } } // The next restart an operator queued, once the last one is done. restarting := "" if len(retiringIDs) == 0 { if id, ok := r.nextRestartLocked(m.Name); ok { r.replicas[id].RetiringAt = r.now() retiringIDs = append(retiringIDs, id) liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id }) readyLive-- restarting = id r.log.Info("restarting a replica: placing its replacement first", "replica", id, "model", m.Name) } } blockedUntil := r.nextAttempt[m.Name] park, isParked := r.parked[m.Name] if isParked && park.fp != modelFingerprint(m) { // Somebody changed the model — a new source, an edited profile — so // the reason it kept failing may be gone. Worth one more streak. delete(r.parked, m.Name) r.attempts[m.Name] = 0 delete(r.nextAttempt, m.Name) isParked, blockedUntil = false, time.Time{} r.log.Info("model unparked: its definition changed", "model", m.Name) } r.mu.Unlock() if restarting != "" { _, span := startSpan(ctx, "restart replica", attribute.String("llmgw.replica", restarting), attribute.String("llmgw.model", m.Name)) span.End() } desired := r.desiredFor(m) outcome := notTried switch { case len(liveIDs) < desired && isParked: // Not placed. Its failure is on every status the operator reads. case len(liveIDs) < desired && r.isHeld(m.Name): // It gave up room to a model of a higher class, which is placed // first; taking it back now would undo the preemption. case len(liveIDs) < desired: if r.now().Before(blockedUntil) { // Still in backoff. Rebuilding immediately would hot-loop on a // model that cannot start — a missing weights path, say — and bury // the real error in noise. return } sctx, span := startSpan(ctx, "scale up", attribute.String("llmgw.model", m.Name), attribute.Int("llmgw.replicas", len(liveIDs)), attribute.Int("llmgw.desired", desired)) outcome = r.scaleUp(sctx, m, avoid) span.SetAttributes(attribute.String("llmgw.outcome", outcome.String())) span.End() case len(liveIDs) > desired: sctx, span := startSpan(ctx, "scale down", attribute.String("llmgw.model", m.Name), attribute.Int("llmgw.replicas", len(liveIDs)), attribute.Int("llmgw.desired", desired)) r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired") span.End() } // A drained replica is stopped once its replacement is serving, or once the // grace period expires. Stopping it the moment it was drained would cut // in-flight requests, which is the thing draining exists to avoid. replaced := len(liveIDs) >= desired for _, id := range drainingIDs { r.mu.Lock() rep, ok := r.replicas[id] var expired bool grace, preemptedFor := r.cfg.DrainGrace, "" if ok { if rep.PreemptedFor != "" { grace, preemptedFor = r.cfg.PreemptGrace, rep.PreemptedFor } expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace } r.mu.Unlock() switch { case !ok: case replaced: r.stopReplica(ctx, id, "drained, and its replacement serves") case expired && preemptedFor != "": r.log.Info("stopping a preempted replica", "replica", id, "for", preemptedFor, "after", grace) r.stopReplica(ctx, id, "preempted for "+preemptedFor+", and its grace is over") case expired: // The replacement never arrived — often because nowhere else can // host it. Draining forever would leave the node undrainable. r.log.Warn("stopping a drained replica without a replacement", "replica", id, "after", grace) r.stopReplica(ctx, id, "drained, and no replacement arrived within "+grace.String()) } } r.retire(ctx, m, retiringIDs, readyLive, len(liveIDs), desired, outcome, isParked) } // scaleUp places and starts one replica per pass. // // One at a time on purpose: placement decisions depend on what is already // allocated, and starting several at once from the same inventory snapshot // would hand them the same GPUs. func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome { _, pspan := startSpan(ctx, "place", attribute.String("llmgw.model", m.Name), attribute.StringSlice("llmgw.avoid", avoid)) profiles := r.orderedProfiles(m) res := r.sched.Place(profiles, avoid) if res.Placement == nil && len(avoid) > 0 { // Spreading across failure domains is a preference, not a rule. If it // is the only thing standing in the way, take the placement anyway: // a second replica on one node still survives an engine crash. res = r.sched.Place(profiles, nil) pspan.SetAttributes(attribute.Bool("llmgw.spread_given_up", res.Placement != nil)) } placementAttrs(pspan, res) pspan.End() r.mu.Lock() r.lastExplain[m.Name] = res r.mu.Unlock() if res.Placement == nil { // Full, with a model somebody has just asked for and cards held by one // nobody has asked for in a while. Stopping one is the whole point of // on-demand loading; without it the feature works until the cluster is // full, which is the only time it matters. // // Not retried here. The replica is stopped, its GPUs are released on // the pass that reaps it, and the next pass places this model — which // keeps the stop-then-start ordering in one place rather than two. if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) { return makingRoom } if r.awaitingPreemption(m.Name) || r.preemptFor(ctx, m, avoid) { return makingRoom } r.notePlacementFailure(m.Name, res.Explain()) return noRoom } if why := r.overLimit(m, res.Placement); why != "" { // Recorded as the placement's answer, so the model reads as not // placeable, and says why, wherever that is shown. r.mu.Lock() r.lastExplain[m.Name] = placement.Result{Rejections: []placement.Rejection{ {Profile: res.Placement.Profile.Name, Reason: why}}} r.mu.Unlock() r.notePlacementFailure(m.Name, why) trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.over_limit", why)) return overLimit } if err := r.startReplica(ctx, m, res.Placement); err != nil { r.log.Error("starting replica failed", "model", m.Name, "err", err) trace.SpanFromContext(ctx).SetStatus(codes.Error, err.Error()) return noRoom } return placed } // scaleOutcome is what one scaleUp did. type scaleOutcome int const ( notTried scaleOutcome = iota // nothing was asked of placement this pass placed // a replica was started makingRoom // room is being made: a preemption, an idle replica stopping noRoom // placement found nowhere, and nothing is making room overLimit // there is room, but the owner's quota does not allow it ) // String names an outcome for a trace. func (o scaleOutcome) String() string { switch o { case placed: return "placed" case makingRoom: return "making room" case noRoom: return "no room" case overLimit: return "over its quota" default: return "not tried" } } // placementAttrs puts a placement's answer on its span: where it went, or // every candidate it turned down and why, which is what `ctl // explain-placement` would have printed at that moment. func placementAttrs(span trace.Span, res placement.Result) { span.SetAttributes(attribute.Int("llmgw.rejections", len(res.Rejections))) pl := res.Placement if pl == nil { explain := res.Explain() if len(explain) > 4096 { explain = explain[:4096] + "…" } span.SetAttributes(attribute.String("llmgw.explain", explain)) span.SetStatus(codes.Error, "fits nowhere") return } nodes := make([]string, 0, len(pl.Nodes)) for _, a := range pl.Nodes { nodes = append(nodes, a.NodeID) } span.SetAttributes( attribute.String("llmgw.profile", pl.Profile.Name), attribute.String("llmgw.engine", pl.Profile.Engine), attribute.String("llmgw.gpu_class", pl.Profile.GPUClass), attribute.String("llmgw.pool", pl.Pool), attribute.String("llmgw.fabric", pl.Fabric), attribute.StringSlice("llmgw.nodes", nodes), ) } func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error { prof, ok := m.ProfileByName(pl.Profile.Name) if !ok { return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name) } espec, ok := m.EngineProfile(prof.Name) if !ok { return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name) } r.mu.Lock() r.seq++ id := replicaID(m.Name, r.seq) attempt := r.attempts[m.Name] r.mu.Unlock() rep := &Replica{ ID: id, Model: m.Name, Profile: prof.Name, ProfileFP: prof.Fingerprint(), Fabric: pl.Fabric, Pool: pl.Pool, State: Placing, Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt, WantShards: len(pl.Nodes), } // The leader's address and port are fixed before any shard starts: every // rank needs the same rendezvous point, and a worker that starts first must // already know where to dial. leader := pl.Nodes[0] leaderNode, ok := r.inv.Get(leader.NodeID) if !ok { return fmt.Errorf("leader node %s disappeared", leader.NodeID) } basePort := r.allocatePorts(pl) for _, a := range pl.Nodes { shardID := fmt.Sprintf("%s-r%d", id, a.Rank) rep.Shards = append(rep.Shards, Shard{ ID: shardID, NodeID: a.NodeID, Rank: a.Rank, GPUIndices: a.GPUIndices, HCAs: a.HCAs, HTTPPort: basePort, State: supervisor.Pending, }) r.inv.Allocate(a.NodeID, inventory.Allocation{ ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices, MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), PowerW: a.PowerW, }) } rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort) // Everything read from rep has to be read now: once it is in the map an // agent status report can mutate it concurrently, and the race detector is // right to object. // // The leader first, then workers ascending. Both engines put the rendezvous // point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a // worker started first has nothing to connect to and spends its startup // budget retrying. // // An earlier version started workers first, on the theory that the head // should wait for them. That is backwards: vLLM's server blocks until the // Ray cluster has the GPUs it needs, so the leader can be started // immediately and will wait on its own. order := slices.Clone(rep.Shards) slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank }) nodeIDs := rep.NodeIDs() // Open until the replica serves, fails or is stopped, and the parent of // every node's part of the start: the agents' spans arrive under it. rctx, rspan := startSpan(ctx, "replica start", append(replicaAttrs(rep), attribute.String("llmgw.engine", prof.Engine), attribute.String("llmgw.gpu_class", prof.GPUClass), attribute.String("llmgw.pool", pl.Pool), attribute.String("llmgw.fabric", pl.Fabric), attribute.String("llmgw.endpoint", rep.Endpoint), attribute.Int("llmgw.attempt", attempt), attribute.String("llmgw.startup_timeout", r.startupTimeout(rep).String()), )...) rep.startSpan, rep.startTrace = rspan, rspan.SpanContext() r.mu.Lock() r.replicas[id] = rep r.mu.Unlock() for _, s := range order { cmd := ShardCommand{ ReplicaID: id, ShardID: s.ID, NodeID: s.NodeID, Spec: engines.ShardSpec{ ReplicaID: id, ShardID: s.ID, Rank: s.Rank, NNodes: len(pl.Nodes), LeaderAddr: addrOf(leaderNode), LeaderPort: basePort + 1, GPUIndices: s.GPUIndices, HCAs: s.HCAs, MgmtInterface: r.cfg.MgmtInterface, HTTPPort: s.HTTPPort, Profile: espec, APIKey: r.engineKey(id), Threads: cpuCores(prof), }, StartupTimeout: prof.StartupTimeout, Pool: pl.Pool, Fabric: pl.Fabric, ProfileFP: prof.Fingerprint(), } // The command, sent: its context travels with it to the agent. sctx, sspan := tracer().Start(rctx, "send start", trace.WithSpanKind(trace.SpanKindProducer), trace.WithAttributes( attribute.String("llmgw.shard", s.ID), attribute.String("llmgw.node", s.NodeID), attribute.Int("llmgw.rank", s.Rank), attribute.IntSlice("llmgw.gpus", s.GPUIndices), attribute.Int("llmgw.port", s.HTTPPort), )) err := r.disp.StartShard(sctx, cmd) obs.End(sspan, err) if err != nil { r.markShard(id, s.ID, supervisor.Failed, err.Error()) return fmt.Errorf("starting shard %s: %w", s.ID, err) } r.markShard(id, s.ID, supervisor.Starting, "") } r.log.Info("replica placed", "replica", id, "model", m.Name, "profile", prof.Name, "pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs) return nil } // replicaID names a replica uniquely for the life of the cluster, not the life // of this process. // // The counter alone is not enough, and the way it fails is ugly. A control // plane that restarts begins counting again, so the replica it places next is // called llama-1 — which is also what an engine started before the restart and // still running on some other node calls itself. Two nodes then claim the same // shard id, adoption hands the id back and forth between them, and every // placement lands on a port an orphan is already holding. Observed on a real // cluster as a bind-conflict loop that never settled. // // The random suffix costs nothing and makes the collision impossible. func replicaID(model string, seq int) string { var buf [4]byte if _, err := crypterand.Read(buf[:]); err != nil { // Randomness failing is not a reason to stop placing replicas; the // counter alone is still unique within this process. return fmt.Sprintf("%s-%d", model, seq) } return fmt.Sprintf("%s-%x", model, buf) } // allocatePorts picks a port shared by every shard of a replica. // // The same port on every node keeps the leader address predictable and the // port bookkeeping trivial; nodes do not share a network namespace, so there is // no conflict in reusing it. func (r *Reconciler) allocatePorts(pl *placement.Placement) int { used := map[int]struct{}{} r.mu.Lock() for _, rep := range r.replicas { for _, s := range rep.Shards { used[s.HTTPPort] = struct{}{} } } // Ports an agent has reported but nothing owns — a stray mid-shutdown, an // engine whose replica was just reaped. The process is still listening // whatever the bookkeeping says, and handing its port to a new shard means // that shard fails to bind and takes the whole replica with it. for p := range r.reservedPorts { used[p] = struct{}{} } r.mu.Unlock() // Two ports per replica: HTTP and the distributed init rendezvous. for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 { if _, taken := used[p]; !taken { return p } } return r.cfg.PortRange[0] } // scaleDown drains replicas; why is for the trace. func (r *Reconciler) scaleDown(ctx context.Context, excess []string, why string) { for _, id := range excess { r.mu.Lock() rep, ok := r.replicas[id] if !ok { r.mu.Unlock() continue } rep.State = Draining rep.DrainingAt = r.now() shards := slices.Clone(rep.Shards) r.mu.Unlock() r.log.Info("draining replica", "replica", id) r.drainShards(ctx, id, shards, why) } } // stopReplica stops a replica's shards and forgets it; why is for the trace. func (r *Reconciler) stopReplica(ctx context.Context, id, why string) { r.mu.Lock() rep, ok := r.replicas[id] if !ok { r.mu.Unlock() return } shards := slices.Clone(rep.Shards) rep.State = Stopped rep.startDone(r.now()) ctx, span := startSpan(ctx, "stop replica", append(replicaAttrs(rep), attribute.String("llmgw.reason", why))...) defer span.End() r.mu.Unlock() for _, s := range shards { if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil { r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err) } } r.inv.ReleaseReplica(id) r.mu.Lock() delete(r.replicas, id) r.mu.Unlock() } // OnFetchProgress records how far a shard's download has got. // // Separate from OnShardStatus because it is not a transition: the shard is // still starting, and treating each progress report as a state change would // reset the timers that decide when a slow start becomes a failed one. func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) { r.mu.Lock() defer r.mu.Unlock() rep, ok := r.replicas[replicaID] if !ok { return } for i := range rep.Shards { if rep.Shards[i].ID != shardID { continue } rep.Shards[i].Fetch = &FetchProgress{ Source: source, BytesDone: done, BytesTotal: total, } } } // OnShardStatus records a status report from an agent. func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) { r.markShard(replicaID, shardID, st, errMsg) r.Notify() } func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) { r.mu.Lock() defer r.mu.Unlock() rep, ok := r.replicas[replicaID] if !ok { return } for i := range rep.Shards { if rep.Shards[i].ID != shardID { continue } rep.Shards[i].State = st if errMsg != "" { rep.Shards[i].Err = errMsg } // A real transition means the download is over, one way or the other. // Left in place it would sit in the listing at ninety-nine percent // under a shard that has been serving for an hour. rep.Shards[i].Fetch = nil break } prev := rep.State rep.deriveState(r.now()) if rep.startSpan != nil { ev := []attribute.KeyValue{ attribute.String("llmgw.shard", shardID), attribute.String("llmgw.state", st.String()), } if errMsg != "" { ev = append(ev, attribute.String("llmgw.error", errMsg)) } rep.startSpan.AddEvent("shard "+st.String(), trace.WithAttributes(ev...)) } if rep.State == Ready || rep.State.Terminal() { rep.startDone(r.now()) } if rep.State == Ready && prev != Ready { // A model that comes up clears its backoff; the next failure starts // counting from scratch rather than from an old streak. r.attempts[rep.Model] = 0 delete(r.nextAttempt, rep.Model) } } // Replicas returns a snapshot of every replica, sorted by id. func (r *Reconciler) Replicas() []Replica { r.mu.Lock() defer r.mu.Unlock() out := make([]Replica, 0, len(r.replicas)) for _, rep := range r.replicas { clone := *rep clone.Shards = slices.Clone(rep.Shards) out = append(out, clone) } slices.SortFunc(out, func(a, b Replica) int { if a.ID < b.ID { return -1 } if a.ID > b.ID { return 1 } return 0 }) return out } // ReadyReplicas returns the replicas the router may send traffic to. func (r *Reconciler) ReadyReplicas() []Replica { var out []Replica for _, rep := range r.Replicas() { if rep.State == Ready { out = append(out, rep) } } return out } // Explain returns the most recent placement result for a model, which is what // `ctl explain-placement` prints. func (r *Reconciler) Explain(model string) (placement.Result, bool) { r.mu.Lock() defer r.mu.Unlock() res, ok := r.lastExplain[model] return res, ok } func addrOf(n inventory.Node) string { if n.MgmtAddr != "" { return n.MgmtAddr } if n.Hostname != "" { return n.Hostname } return n.ID } // profileMemory is what a model's profile declared it needs, or zero. // // Looked up when a replica is adopted after a restart, because the allocation // has to carry the same claim it did before — a control plane that forgot every // claim on restart would place a second large model onto a node already full of // the first. func (r *Reconciler) profileMemory(model, profile string) int { p, _ := r.profileOf(model, profile) return p.MinMemoryGB } // profileWatts is the draw a profile declared for each of its shards, so an // adopted replica counts against its racks' power budgets as a placed one // does; zero when it declared none. func (r *Reconciler) profileWatts(model, profile string) float64 { p, _ := r.profileOf(model, profile) if p.PowerWatts <= 0 { return 0 } return p.PowerWatts / float64(max(p.Nodes, 1)) } // profileOf is a replica's profile as the catalog holds it now. func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) { m, ok := r.cat.Load().Resolve(model) if !ok { return catalog.Profile{}, false } for _, p := range m.Profiles { if p.Name == profile { return p, true } } return catalog.Profile{}, false } // cpuCores is what a profile reserves of a node's CPUs: its cores when it // computes on them, nothing when a GPU does the work. func cpuCores(p catalog.Profile) int { if !p.CPUOnly() { return 0 } return p.CPUCores } func (r *Reconciler) engineKey(replicaID string) string { if r.cfg.EngineKey == nil { return "" } return r.cfg.EngineKey(replicaID) } // SetNotifier gives the reconciler somewhere to say a model was parked or a // node lost. func (r *Reconciler) SetNotifier(n *notify.Notifier) { r.notify.Store(n) }