CI / test (push) Successful in 1m39s
The stale-profile flag could not fire where it matters. The control plane keeps no record of a running replica across its own restart — it rebuilds them from what the agents report — and adoption left the fingerprint empty, so every replica read as unknown afterwards. That is precisely the moment the catalog is most likely to have been edited: change the configuration, restart the server, and nothing would ever say the fleet disagrees with it. The fingerprint now travels with StartShard, the agent keeps it beside the pool and fabric it already echoes back, and adoption takes the replica's version from the agent rather than from the catalog it is about to be compared against. An agent too old to report it sends nothing, which still reads as unknown rather than as matching. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1383 lines
45 KiB
Go
1383 lines
45 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
crypterand "crypto/rand"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
)
|
|
|
|
// ShardCommand is everything an agent needs to launch one shard.
|
|
type ShardCommand struct {
|
|
ReplicaID string
|
|
ShardID string
|
|
NodeID string
|
|
Spec engines.ShardSpec
|
|
// StartupTimeout comes from the profile; model loading dominates it.
|
|
StartupTimeout time.Duration
|
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
|
// travel with the command so the agent can report them back after a
|
|
// control plane restart (plan §6.5).
|
|
Pool string
|
|
Fabric string
|
|
// ProfileFP travels for the same reason, and answers a different question:
|
|
// whether the profile has been edited since this shard was started.
|
|
ProfileFP string
|
|
}
|
|
|
|
// Dispatcher sends commands to agents.
|
|
//
|
|
// It is an interface so the reconciliation loop can be tested without a network:
|
|
// the loop's logic is where the interesting failures are, not the transport.
|
|
type Dispatcher interface {
|
|
StartShard(ctx context.Context, cmd ShardCommand) error
|
|
StopShard(ctx context.Context, nodeID, shardID string) error
|
|
DrainShard(ctx context.Context, nodeID, shardID string) error
|
|
}
|
|
|
|
// Config tunes the reconciliation loop.
|
|
type Config struct {
|
|
// Interval is how often desired and actual are compared when nothing else
|
|
// happens. Events wake the loop sooner.
|
|
Interval time.Duration
|
|
// BaseBackoff is how long to wait before rebuilding a failed replica.
|
|
BaseBackoff time.Duration
|
|
// MaxBackoff caps the exponential growth.
|
|
MaxBackoff time.Duration
|
|
// UnreachableAfter is how long without a heartbeat before a node is
|
|
// unreachable.
|
|
UnreachableAfter time.Duration
|
|
// LostAfter is how long unreachable before its replicas are declared dead.
|
|
// Longer than UnreachableAfter on purpose: a brief network blip must not
|
|
// tear down a replica that is still serving perfectly well.
|
|
LostAfter time.Duration
|
|
// DrainGrace bounds how long a draining replica waits for its replacement.
|
|
// Without a bound, draining a node whose model cannot be placed anywhere
|
|
// else would never finish.
|
|
DrainGrace time.Duration
|
|
// PortRange is the per-node port range for engine HTTP and dist-init.
|
|
PortRange [2]int
|
|
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
|
MgmtInterface string
|
|
// EvictionGrace is how long a replica must have gone unasked-for before it
|
|
// may be stopped to make room for another model.
|
|
//
|
|
// Shorter than IdleTimeout on purpose: a replica idle for five minutes is
|
|
// not worth reclaiming on its own, and is worth reclaiming when something
|
|
// else needs the cards. Still minutes rather than seconds, because stopping
|
|
// one throws away the KV cache prefix routing spent filling.
|
|
EvictionGrace time.Duration
|
|
// MinLifetime is how long a replica is safe from eviction after it starts
|
|
// serving, whatever else wants its cards.
|
|
//
|
|
// The thing that stops two models on a cluster with room for one from
|
|
// taking turns evicting each other. A load costs minutes; oscillating at a
|
|
// load per cycle is worse for both models than one of them waiting.
|
|
MinLifetime time.Duration
|
|
// IdleTimeout is how long an on-demand model stays loaded with nobody
|
|
// asking for it. A model may override it.
|
|
IdleTimeout time.Duration
|
|
// SettleFor is how long after start to leave placement alone while agents
|
|
// reconnect.
|
|
//
|
|
// A control plane that has just started knows nothing, and the first
|
|
// reconcile tick runs before any agent has registered. It sees an empty
|
|
// fleet and acts on it: "needs 2 nodes on one fabric, but no node reports a
|
|
// fabric" is that, and it is harmless only because adoption catches up.
|
|
// What is not harmless is placing on the fraction of the fleet that
|
|
// happens to have reported first — after one restart both replicas of a
|
|
// model landed on the single node that had registered, when four were
|
|
// coming.
|
|
//
|
|
// Adoption still runs throughout; only placement waits. The window ends
|
|
// early once the fleet stops growing, so a small cluster does not sit idle
|
|
// for the whole duration — but not too early: an agent that lost its
|
|
// connection reconnects on a backoff reaching thirty seconds, so "no new
|
|
// node for a few seconds" means nothing.
|
|
SettleFor time.Duration
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
SettleFor: 60 * time.Second,
|
|
IdleTimeout: 15 * time.Minute,
|
|
EvictionGrace: 2 * time.Minute,
|
|
MinLifetime: 5 * time.Minute,
|
|
DrainGrace: 5 * time.Minute,
|
|
PortRange: [2]int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
d := DefaultConfig()
|
|
if c.Interval <= 0 {
|
|
c.Interval = d.Interval
|
|
}
|
|
if c.BaseBackoff <= 0 {
|
|
c.BaseBackoff = d.BaseBackoff
|
|
}
|
|
if c.MaxBackoff < c.BaseBackoff {
|
|
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
|
|
}
|
|
if c.IdleTimeout <= 0 {
|
|
c.IdleTimeout = d.IdleTimeout
|
|
}
|
|
if c.EvictionGrace <= 0 {
|
|
c.EvictionGrace = d.EvictionGrace
|
|
}
|
|
if c.MinLifetime <= 0 {
|
|
c.MinLifetime = d.MinLifetime
|
|
}
|
|
if c.SettleFor < 0 {
|
|
c.SettleFor = 0
|
|
} else if c.SettleFor == 0 {
|
|
c.SettleFor = d.SettleFor
|
|
}
|
|
if c.UnreachableAfter <= 0 {
|
|
c.UnreachableAfter = d.UnreachableAfter
|
|
}
|
|
if c.LostAfter < c.UnreachableAfter {
|
|
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
|
|
}
|
|
if c.DrainGrace <= 0 {
|
|
c.DrainGrace = d.DrainGrace
|
|
}
|
|
if c.PortRange[1] <= c.PortRange[0] {
|
|
c.PortRange = d.PortRange
|
|
}
|
|
if c.MgmtInterface == "" {
|
|
c.MgmtInterface = d.MgmtInterface
|
|
}
|
|
}
|
|
|
|
// Reconciler compares the catalog against what is running and closes the gap.
|
|
type Reconciler struct {
|
|
// leader gates every action; nil means act unconditionally.
|
|
leader Leader
|
|
|
|
cfg Config
|
|
cat *catalog.Live
|
|
inv *inventory.Store
|
|
sched *placement.Scheduler
|
|
disp Dispatcher
|
|
log *slog.Logger
|
|
now func() time.Time
|
|
|
|
mu sync.Mutex
|
|
replicas map[string]*Replica
|
|
// nextAttempt is the earliest a model may be rebuilt, per model.
|
|
nextAttempt map[string]time.Time
|
|
// attempts counts consecutive failures per model, for the backoff.
|
|
attempts map[string]int
|
|
// lastExplain keeps the most recent placement failure per model, so
|
|
// `ctl explain-placement` can answer without re-running placement.
|
|
lastExplain map[string]placement.Result
|
|
seq int
|
|
// reservedPorts holds ports the fleet reports as in use but no replica
|
|
// claims, so the allocator cannot hand out what something is still bound to.
|
|
reservedPorts map[int]struct{}
|
|
// wanted records when an on-demand model was last asked for.
|
|
//
|
|
// The whole of on-demand loading, kept here rather than in the catalog
|
|
// because it is state about this moment and not about what the model is.
|
|
// A model nobody has wanted has no entry and is simply not running.
|
|
wanted map[string]time.Time
|
|
|
|
wake chan struct{}
|
|
stop chan struct{}
|
|
done chan struct{}
|
|
once sync.Once
|
|
|
|
// startedAt and lastNodeCount track the settling window: placement waits
|
|
// until the fleet has stopped growing, or the window expires.
|
|
startedAt time.Time
|
|
lastNodeCount int
|
|
steadySince time.Time
|
|
}
|
|
|
|
// New builds a reconciler.
|
|
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Reconciler{
|
|
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
|
|
sched: placement.New(inv),
|
|
now: time.Now,
|
|
replicas: map[string]*Replica{},
|
|
nextAttempt: map[string]time.Time{},
|
|
attempts: map[string]int{},
|
|
reservedPorts: map[int]struct{}{},
|
|
wanted: map[string]time.Time{},
|
|
lastExplain: map[string]placement.Result{},
|
|
wake: make(chan struct{}, 1),
|
|
stop: make(chan struct{}),
|
|
done: make(chan struct{}),
|
|
startedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
// SetClock replaces the time source, for tests.
|
|
func (r *Reconciler) SetClock(now func() time.Time) {
|
|
r.now = now
|
|
r.startedAt = now()
|
|
}
|
|
|
|
// Leader reports whether this process should be acting on the fleet.
|
|
type Leader interface{ IsLeader() bool }
|
|
|
|
// SetLeadership gates every action on holding leadership.
|
|
//
|
|
// A setter rather than a constructor argument because the thing that decides
|
|
// leadership needs a store, and the store is wired after the controller. Nil
|
|
// leaves the reconciler acting unconditionally, which is what a single control
|
|
// plane wants and what every test expects.
|
|
func (r *Reconciler) SetLeadership(l Leader) { r.leader = l }
|
|
|
|
// Start runs the loop until Close.
|
|
func (r *Reconciler) Start(ctx context.Context) {
|
|
go func() {
|
|
defer close(r.done)
|
|
t := time.NewTicker(r.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
r.Reconcile(ctx)
|
|
for {
|
|
select {
|
|
case <-r.stop:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-r.wake:
|
|
r.Reconcile(ctx)
|
|
case <-t.C:
|
|
r.Reconcile(ctx)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Close stops the loop.
|
|
func (r *Reconciler) Close() {
|
|
r.once.Do(func() {
|
|
close(r.stop)
|
|
<-r.done
|
|
})
|
|
}
|
|
|
|
// Notify wakes the loop, for when something changed and waiting for the next
|
|
// tick would add pointless latency.
|
|
func (r *Reconciler) Notify() {
|
|
select {
|
|
case r.wake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Reconcile performs one pass. It is exported so tests can drive it directly
|
|
// instead of racing a ticker.
|
|
func (r *Reconciler) Reconcile(ctx context.Context) {
|
|
// A standby does nothing at all here. Everything below issues commands to
|
|
// agents — placing, evicting, draining, reaping — and two control planes
|
|
// doing that against one fleet is how a model ends up with twice the
|
|
// replicas it asked for, each holding GPUs the other believes are free.
|
|
//
|
|
// Unset means leader, so a single control plane and every test behave
|
|
// exactly as they did before leadership existed.
|
|
if r.leader != nil && !r.leader.IsLeader() {
|
|
return
|
|
}
|
|
r.expireNodes()
|
|
r.expireStuckReplicas()
|
|
r.drainCordonedNodes(ctx)
|
|
r.reapFailed(ctx)
|
|
|
|
// Everything above is about what is already running and stays correct on a
|
|
// partial fleet. Placement is the part that must not act on one.
|
|
if !r.fleetHasSettled() {
|
|
return
|
|
}
|
|
|
|
for _, m := range r.cat.Load().Models() {
|
|
r.reconcileModel(ctx, m)
|
|
}
|
|
r.stopReplicasOfRemovedModels(ctx)
|
|
}
|
|
|
|
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
|
|
//
|
|
// reconcileModel only ever visits models that are in the catalog, so a replica
|
|
// whose model has been removed was visited by nothing: it kept its GPUs, kept
|
|
// counting towards the inventory, and kept running an engine for a name no
|
|
// client could ask for. The endpoint publisher already skipped it with the
|
|
// comment "the reconciler will tear it down", which was simply not true.
|
|
//
|
|
// It mattered little while a catalog could only change by restarting the
|
|
// process. It matters now that models come and go while the cluster runs, which
|
|
// is the whole point of making the catalog live.
|
|
//
|
|
// Ordered after the model loop so a rebuild elsewhere has already claimed what
|
|
// it needs: freeing these GPUs first would let a replacement land on hardware
|
|
// that is about to be freed anyway, and churn two nodes instead of one.
|
|
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
|
cat := r.cat.Load()
|
|
|
|
r.mu.Lock()
|
|
var gone []*Replica
|
|
for _, rep := range r.replicas {
|
|
if _, ok := cat.Resolve(rep.Model); !ok {
|
|
gone = append(gone, rep)
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range gone {
|
|
r.log.Info("stopping a replica whose model left the catalog",
|
|
"replica", rep.ID, "model", rep.Model)
|
|
r.stopReplica(ctx, rep.ID)
|
|
}
|
|
}
|
|
|
|
// fleetHasSettled reports whether placement may run yet.
|
|
//
|
|
// Two ways out, because either alone is wrong. Waiting for the window to expire
|
|
// would make every restart cost that much downtime even when the whole fleet
|
|
// reconnected in a second. Waiting only for the count to stop changing would
|
|
// hang forever on a cluster whose nodes never all come back.
|
|
func (r *Reconciler) fleetHasSettled() bool {
|
|
if r.cfg.SettleFor <= 0 {
|
|
return true
|
|
}
|
|
now := r.now()
|
|
if now.Sub(r.startedAt) >= r.cfg.SettleFor {
|
|
return true
|
|
}
|
|
|
|
n := len(r.inv.Nodes())
|
|
if n != r.lastNodeCount {
|
|
r.lastNodeCount, r.steadySince = n, now
|
|
return false
|
|
}
|
|
// Steady for a moment, and at least one node present. A count that has been
|
|
// zero since start is not a settled fleet, it is one that has not arrived.
|
|
if n == 0 {
|
|
return false
|
|
}
|
|
// Half the window, not a quarter of it, and the window is a minute.
|
|
//
|
|
// Measured on the cluster: after a restart node05 registered in one second
|
|
// and the other four arrived seventeen seconds later, because a
|
|
// disconnected agent reconnects on a backoff that reaches thirty seconds.
|
|
// A five-second steady period declared a fleet of one settled and resumed
|
|
// placement while four nodes were still backing off — the exact situation
|
|
// this exists to avoid, reached by being impatient about it.
|
|
settled := now.Sub(r.steadySince) >= r.cfg.SettleFor/2
|
|
if settled {
|
|
r.log.Info("fleet settled; placement resuming",
|
|
"nodes", n, "waited", now.Sub(r.startedAt).Round(time.Millisecond))
|
|
r.startedAt = time.Time{} // never wait again in this process
|
|
}
|
|
return settled
|
|
}
|
|
|
|
// RecordUsage stamps replicas with when each was last sent a request.
|
|
//
|
|
// Replicas absent from the map are left alone rather than zeroed: a router that
|
|
// has just restarted knows nothing about what happened before it, and treating
|
|
// that as "never used" would make every replica look abandoned at once —
|
|
// exactly when a fleet is least able to afford acting on that.
|
|
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
|
|
if len(lastUsed) == 0 {
|
|
return
|
|
}
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for id, at := range lastUsed {
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
continue
|
|
}
|
|
// Only forwards. Two routers reporting, or a stale snapshot arriving
|
|
// late, must not move a replica's last use backwards.
|
|
if at.After(rep.LastUsedAt) {
|
|
rep.LastUsedAt = at
|
|
}
|
|
}
|
|
r.noteUsage(lastUsed)
|
|
}
|
|
|
|
// IdleFor reports how long a replica has gone without a request, and whether
|
|
// that is known at all.
|
|
//
|
|
// The second return is not a detail. "Idle for an hour" and "nobody has told me
|
|
// yet" are different states, and anything that acts on idleness — eviction,
|
|
// most obviously — has to refuse to act on the second.
|
|
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok || rep.LastUsedAt.IsZero() {
|
|
return 0, false
|
|
}
|
|
return r.now().Sub(rep.LastUsedAt), true
|
|
}
|
|
|
|
// Want marks an on-demand model as asked for, so the next pass places it.
|
|
//
|
|
// Called by whatever fields the request — the router, in practice. It returns
|
|
// whether the model is one this applies to, so a caller can tell "being loaded"
|
|
// from "there is genuinely nothing serving this", which are different answers
|
|
// to give a client.
|
|
func (r *Reconciler) Want(model string) bool {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok || !m.Scaling.OnDemand() {
|
|
return false
|
|
}
|
|
r.mu.Lock()
|
|
r.wanted[m.Name] = r.now()
|
|
r.mu.Unlock()
|
|
// Immediately, not on the next tick. Someone is waiting on this request,
|
|
// and five seconds of ticker is five seconds they spend staring at nothing.
|
|
r.Notify()
|
|
return true
|
|
}
|
|
|
|
// desiredFor is how many replicas a model should have right now.
|
|
//
|
|
// For an ordinary model that is its configured minimum. For an on-demand one it
|
|
// is one while somebody is asking and none once they stop, which is the whole
|
|
// feature: a cluster with more models than GPUs can hold every model in its
|
|
// catalog and only the wanted ones in its cards.
|
|
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
|
if !m.Scaling.OnDemand() {
|
|
return m.Scaling.Min
|
|
}
|
|
r.mu.Lock()
|
|
last, asked := r.wanted[m.Name]
|
|
r.mu.Unlock()
|
|
if !asked {
|
|
return 0
|
|
}
|
|
if r.now().Sub(last) > r.idleTimeout(m) {
|
|
return 0
|
|
}
|
|
return 1
|
|
}
|
|
|
|
// idleTimeout is how long an on-demand model stays up unasked-for.
|
|
func (r *Reconciler) idleTimeout(m catalog.Model) time.Duration {
|
|
if m.Scaling.IdleTimeout > 0 {
|
|
return m.Scaling.IdleTimeout
|
|
}
|
|
return r.cfg.IdleTimeout
|
|
}
|
|
|
|
// noteUsage keeps an on-demand model alive while it is being used.
|
|
//
|
|
// Without this a model would be evicted exactly IdleTimeout after the request
|
|
// that loaded it, however busy it had been since — the timer would measure
|
|
// time since the first request rather than the last.
|
|
func (r *Reconciler) noteUsage(lastUsed map[string]time.Time) {
|
|
byModel := map[string]time.Time{}
|
|
for id, at := range lastUsed {
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if at.After(byModel[rep.Model]) {
|
|
byModel[rep.Model] = at
|
|
}
|
|
}
|
|
for model, at := range byModel {
|
|
if cur, ok := r.wanted[model]; ok && at.After(cur) {
|
|
r.wanted[model] = at
|
|
}
|
|
}
|
|
}
|
|
|
|
// expireStuckReplicas fails replicas that never finished coming up.
|
|
//
|
|
// Every other path out of Starting depends on the agent reporting something. A
|
|
// replica whose agent never reports — the start was a duplicate shard id and
|
|
// became a no-op, the node went quiet in a way expiry did not catch, the engine
|
|
// wedged before its first health check — sits in Starting for as long as the
|
|
// process lives, holding its GPUs and counting towards the model's replica
|
|
// count while serving nothing.
|
|
//
|
|
// Found on a real cluster: two replicas sat in Starting for eight minutes past
|
|
// a two-minute startup timeout, and nothing would ever have moved them.
|
|
func (r *Reconciler) expireStuckReplicas() {
|
|
now := r.now()
|
|
var stuck []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
|
|
stuck = append(stuck, rep)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range stuck {
|
|
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
|
|
"replica", rep.ID, "model", rep.Model, "state", "starting",
|
|
"timeout", r.startupTimeout(rep))
|
|
}
|
|
}
|
|
|
|
// startupTimeout is the profile's, or the configured default when the profile
|
|
// does not say — a replica with no deadline at all is the thing this exists to
|
|
// prevent.
|
|
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
|
|
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
|
|
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
|
|
return p.StartupTimeout
|
|
}
|
|
}
|
|
// No profile deadline: a generous default, because the thing being prevented
|
|
// is a replica with no deadline at all rather than one with a loose one.
|
|
// Loading a large model across nodes genuinely takes tens of minutes.
|
|
return 30 * time.Minute
|
|
}
|
|
|
|
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
|
|
// replicas of nodes that are gone.
|
|
//
|
|
// The two-stage delay is deliberate: a brief network blip must not tear down
|
|
// replicas that are still serving traffic perfectly well, and rebuilding a
|
|
// multi-node replica costs tens of minutes of model loading.
|
|
func (r *Reconciler) expireNodes() {
|
|
now := r.now()
|
|
for _, n := range r.inv.Nodes() {
|
|
silent := now.Sub(n.LastSeen)
|
|
switch {
|
|
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
|
|
r.inv.SetState(n.ID, inventory.Unreachable)
|
|
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
|
|
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
|
|
r.inv.SetState(n.ID, inventory.Lost)
|
|
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
|
"node", n.ID, "silent_for", silent.Round(time.Second))
|
|
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
|
}
|
|
}
|
|
}
|
|
|
|
// drainCordonedNodes moves work off nodes an operator is taking out of service.
|
|
//
|
|
// Draining a node is not the same as losing one: the engines are healthy and
|
|
// still serving, so their replicas are marked Draining rather than Failed. That
|
|
// makes the model look short of replicas, a replacement is placed elsewhere,
|
|
// and the original is stopped only once the replacement is serving — which is
|
|
// the difference between a rolling move and an outage.
|
|
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
|
|
var toDrain []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
for _, s := range rep.Shards {
|
|
n, ok := r.inv.Get(s.NodeID)
|
|
if ok && n.State == inventory.Draining {
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
toDrain = append(toDrain, rep)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
shards := map[string][]Shard{}
|
|
for _, rep := range toDrain {
|
|
shards[rep.ID] = slices.Clone(rep.Shards)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range toDrain {
|
|
r.log.Info("draining replica off a node being taken out of service",
|
|
"replica", rep.ID, "nodes", rep.NodeIDs())
|
|
for _, s := range shards[rep.ID] {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() || !rep.OnNode(nodeID) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = reason
|
|
}
|
|
}
|
|
|
|
// reapFailed releases the resources of terminal replicas and tells the agents
|
|
// to clean up whatever might still be running.
|
|
func (r *Reconciler) reapFailed(ctx context.Context) {
|
|
r.mu.Lock()
|
|
var doomed []*Replica
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() {
|
|
doomed = append(doomed, rep)
|
|
}
|
|
}
|
|
for _, rep := range doomed {
|
|
delete(r.replicas, rep.ID)
|
|
if rep.State == Failed {
|
|
// Carry the failure count on the model, not the replica: the
|
|
// replica is about to disappear, and the backoff has to survive it.
|
|
r.attempts[rep.Model]++
|
|
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range doomed {
|
|
// rep is out of the map now, so no agent report can reach it any more.
|
|
if rep.State == Failed {
|
|
r.mu.Lock()
|
|
retry := r.backoff(r.attempts[rep.Model])
|
|
r.mu.Unlock()
|
|
r.log.Error("replica failed",
|
|
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
|
|
"retry_in", retry.Round(time.Second))
|
|
}
|
|
// Stop every shard, including the ones that looked healthy: a partially
|
|
// alive replica holds GPUs while serving nothing.
|
|
for _, s := range rep.Shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard of a dead replica failed",
|
|
"replica", rep.ID, "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(rep.ID)
|
|
}
|
|
}
|
|
|
|
// backoff grows with consecutive failures and is jittered, so a cluster-wide
|
|
// problem does not produce synchronized retry storms across every model.
|
|
func (r *Reconciler) backoff(attempt int) time.Duration {
|
|
if attempt <= 0 {
|
|
return 0
|
|
}
|
|
d := r.cfg.BaseBackoff << min(attempt-1, 16)
|
|
if d > r.cfg.MaxBackoff || d <= 0 {
|
|
d = r.cfg.MaxBackoff
|
|
}
|
|
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
|
|
return d - d/8 + jitter
|
|
}
|
|
|
|
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
|
// Snapshot under the lock and work from values afterwards. A replica
|
|
// pointer stops being safe the moment it is in the map: an agent's status
|
|
// report can mutate it from another goroutine at any time.
|
|
r.mu.Lock()
|
|
var liveIDs, drainingIDs, avoid []string
|
|
for _, rep := range r.replicas {
|
|
if rep.Model != m.Name {
|
|
continue
|
|
}
|
|
switch {
|
|
case rep.State.Counts():
|
|
liveIDs = append(liveIDs, rep.ID)
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
case rep.State == Draining:
|
|
drainingIDs = append(drainingIDs, rep.ID)
|
|
}
|
|
}
|
|
blockedUntil := r.nextAttempt[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
desired := r.desiredFor(m)
|
|
switch {
|
|
case len(liveIDs) < desired:
|
|
if r.now().Before(blockedUntil) {
|
|
// Still in backoff. Rebuilding immediately would hot-loop on a
|
|
// model that cannot start — a missing weights path, say — and bury
|
|
// the real error in noise.
|
|
return
|
|
}
|
|
r.scaleUp(ctx, m, avoid)
|
|
case len(liveIDs) > desired:
|
|
r.scaleDown(ctx, liveIDs[desired:])
|
|
}
|
|
|
|
// A drained replica is stopped once its replacement is serving, or once the
|
|
// grace period expires. Stopping it the moment it was drained would cut
|
|
// in-flight requests, which is the thing draining exists to avoid.
|
|
replaced := len(liveIDs) >= desired
|
|
for _, id := range drainingIDs {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
var expired bool
|
|
if ok {
|
|
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > r.cfg.DrainGrace
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
switch {
|
|
case !ok:
|
|
case replaced:
|
|
r.stopReplica(ctx, id)
|
|
case expired:
|
|
// The replacement never arrived — often because nowhere else can
|
|
// host it. Draining forever would leave the node undrainable.
|
|
r.log.Warn("stopping a drained replica without a replacement",
|
|
"replica", id, "after", r.cfg.DrainGrace)
|
|
r.stopReplica(ctx, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// scaleUp places and starts one replica per pass.
|
|
//
|
|
// One at a time on purpose: placement decisions depend on what is already
|
|
// allocated, and starting several at once from the same inventory snapshot
|
|
// would hand them the same GPUs.
|
|
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
|
|
res := r.sched.Place(m.PlacementProfiles(), avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
// Spreading across failure domains is a preference, not a rule. If it
|
|
// is the only thing standing in the way, take the placement anyway:
|
|
// a second replica on one node still survives an engine crash.
|
|
res = r.sched.Place(m.PlacementProfiles(), nil)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.lastExplain[m.Name] = res
|
|
r.mu.Unlock()
|
|
|
|
if res.Placement == nil {
|
|
// Full, with a model somebody has just asked for and cards held by one
|
|
// nobody has asked for in a while. Stopping one is the whole point of
|
|
// on-demand loading; without it the feature works until the cluster is
|
|
// full, which is the only time it matters.
|
|
//
|
|
// Not retried here. The replica is stopped, its GPUs are released on
|
|
// the pass that reaps it, and the next pass places this model — which
|
|
// keeps the stop-then-start ordering in one place rather than two.
|
|
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
|
|
return
|
|
}
|
|
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
|
|
return
|
|
}
|
|
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
|
r.log.Error("starting replica failed", "model", m.Name, "err", err)
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
|
|
prof, ok := m.ProfileByName(pl.Profile.Name)
|
|
if !ok {
|
|
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
|
|
}
|
|
espec, ok := m.EngineProfile(prof.Name)
|
|
if !ok {
|
|
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.seq++
|
|
id := replicaID(m.Name, r.seq)
|
|
attempt := r.attempts[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
rep := &Replica{
|
|
ID: id, Model: m.Name, Profile: prof.Name, ProfileFP: prof.Fingerprint(),
|
|
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
|
|
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
|
|
WantShards: len(pl.Nodes),
|
|
}
|
|
|
|
// The leader's address and port are fixed before any shard starts: every
|
|
// rank needs the same rendezvous point, and a worker that starts first must
|
|
// already know where to dial.
|
|
leader := pl.Nodes[0]
|
|
leaderNode, ok := r.inv.Get(leader.NodeID)
|
|
if !ok {
|
|
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
|
|
}
|
|
basePort := r.allocatePorts(pl)
|
|
|
|
for _, a := range pl.Nodes {
|
|
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
|
|
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
|
|
HTTPPort: basePort, State: supervisor.Pending,
|
|
})
|
|
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
|
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
|
MemoryGB: prof.MinMemoryGB,
|
|
})
|
|
}
|
|
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
|
|
|
// Everything read from rep has to be read now: once it is in the map an
|
|
// agent status report can mutate it concurrently, and the race detector is
|
|
// right to object.
|
|
//
|
|
// The leader first, then workers ascending. Both engines put the rendezvous
|
|
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
|
|
// worker started first has nothing to connect to and spends its startup
|
|
// budget retrying.
|
|
//
|
|
// An earlier version started workers first, on the theory that the head
|
|
// should wait for them. That is backwards: vLLM's server blocks until the
|
|
// Ray cluster has the GPUs it needs, so the leader can be started
|
|
// immediately and will wait on its own.
|
|
order := slices.Clone(rep.Shards)
|
|
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
|
|
nodeIDs := rep.NodeIDs()
|
|
|
|
r.mu.Lock()
|
|
r.replicas[id] = rep
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range order {
|
|
cmd := ShardCommand{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
NodeID: s.NodeID,
|
|
Spec: engines.ShardSpec{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
Rank: s.Rank,
|
|
NNodes: len(pl.Nodes),
|
|
LeaderAddr: addrOf(leaderNode),
|
|
LeaderPort: basePort + 1,
|
|
GPUIndices: s.GPUIndices,
|
|
HCAs: s.HCAs,
|
|
MgmtInterface: r.cfg.MgmtInterface,
|
|
HTTPPort: s.HTTPPort,
|
|
Profile: espec,
|
|
},
|
|
StartupTimeout: prof.StartupTimeout,
|
|
Pool: pl.Pool,
|
|
Fabric: pl.Fabric,
|
|
ProfileFP: prof.Fingerprint(),
|
|
}
|
|
if err := r.disp.StartShard(ctx, cmd); err != nil {
|
|
r.markShard(id, s.ID, supervisor.Failed, err.Error())
|
|
return fmt.Errorf("starting shard %s: %w", s.ID, err)
|
|
}
|
|
r.markShard(id, s.ID, supervisor.Starting, "")
|
|
}
|
|
|
|
r.log.Info("replica placed",
|
|
"replica", id, "model", m.Name, "profile", prof.Name,
|
|
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
|
|
return nil
|
|
}
|
|
|
|
// replicaID names a replica uniquely for the life of the cluster, not the life
|
|
// of this process.
|
|
//
|
|
// The counter alone is not enough, and the way it fails is ugly. A control
|
|
// plane that restarts begins counting again, so the replica it places next is
|
|
// called llama-1 — which is also what an engine started before the restart and
|
|
// still running on some other node calls itself. Two nodes then claim the same
|
|
// shard id, adoption hands the id back and forth between them, and every
|
|
// placement lands on a port an orphan is already holding. Observed on a real
|
|
// cluster as a bind-conflict loop that never settled.
|
|
//
|
|
// The random suffix costs nothing and makes the collision impossible.
|
|
func replicaID(model string, seq int) string {
|
|
var buf [4]byte
|
|
if _, err := crypterand.Read(buf[:]); err != nil {
|
|
// Randomness failing is not a reason to stop placing replicas; the
|
|
// counter alone is still unique within this process.
|
|
return fmt.Sprintf("%s-%d", model, seq)
|
|
}
|
|
return fmt.Sprintf("%s-%x", model, buf)
|
|
}
|
|
|
|
// allocatePorts picks a port shared by every shard of a replica.
|
|
//
|
|
// The same port on every node keeps the leader address predictable and the
|
|
// port bookkeeping trivial; nodes do not share a network namespace, so there is
|
|
// no conflict in reusing it.
|
|
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
|
|
used := map[int]struct{}{}
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
for _, s := range rep.Shards {
|
|
used[s.HTTPPort] = struct{}{}
|
|
}
|
|
}
|
|
// Ports an agent has reported but nothing owns — a stray mid-shutdown, an
|
|
// engine whose replica was just reaped. The process is still listening
|
|
// whatever the bookkeeping says, and handing its port to a new shard means
|
|
// that shard fails to bind and takes the whole replica with it.
|
|
for p := range r.reservedPorts {
|
|
used[p] = struct{}{}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
// Two ports per replica: HTTP and the distributed init rendezvous.
|
|
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
|
|
if _, taken := used[p]; !taken {
|
|
return p
|
|
}
|
|
}
|
|
return r.cfg.PortRange[0]
|
|
}
|
|
|
|
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
|
for _, id := range excess {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
shards := slices.Clone(rep.Shards)
|
|
r.mu.Unlock()
|
|
|
|
r.log.Info("draining replica", "replica", id)
|
|
for _, s := range shards {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
shards := slices.Clone(rep.Shards)
|
|
rep.State = Stopped
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(id)
|
|
|
|
r.mu.Lock()
|
|
delete(r.replicas, id)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// Adopt rebuilds replica state from shards an agent reports it is already
|
|
// running (plan §6.5).
|
|
//
|
|
// This is what makes a control plane restart survivable. Without it the
|
|
// reconciler sees an empty cluster, places duplicates, and the engines already
|
|
// running keep holding GPUs the inventory believes are free — so the duplicates
|
|
// fail to allocate, with an error naming the new process rather than the old.
|
|
//
|
|
// Shards whose model or profile is no longer in the catalog are returned for
|
|
// the caller to stop: nothing will ever claim them.
|
|
func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
|
defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards)
|
|
|
|
byReplica := map[string][]AdoptedShard{}
|
|
for _, s := range shards {
|
|
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
for replicaID, group := range byReplica {
|
|
if existing, known := r.replicas[replicaID]; known {
|
|
// The other half of a replica we are still assembling. Merge rather
|
|
// than choose: these shards belong to the same replica as the ones
|
|
// already adopted, they are simply on a different node whose agent
|
|
// registered a moment later. Treating them as a competing claim is
|
|
// what tore multi-node replicas apart on every control plane
|
|
// restart — one half adopted and failed for being short, the other
|
|
// stopped as unclaimed when it finally arrived.
|
|
if !existing.AdoptedAt.IsZero() && len(existing.Shards) < existing.WantShards {
|
|
r.mergeAdopted(existing, group)
|
|
continue
|
|
}
|
|
// Usually this is an agent reconnecting rather than a control plane
|
|
// restart, and its status reports keep the state current.
|
|
//
|
|
// Not always. Replica ids are derived from the model name and a
|
|
// counter, so a control plane that restarts and places before the
|
|
// agents reconnect produces a *different* replica under the same
|
|
// id, on a freshly allocated port. Skipping the adoption then
|
|
// leaves the cluster pointing at a port nothing listens on, while
|
|
// the engine that is actually serving is treated as a stray.
|
|
//
|
|
// The fleet is authoritative about what is running (plan §6.5), so
|
|
// what we merely placed loses to what an agent reports — unless our
|
|
// copy has a shard that has actually come up, in which case two
|
|
// things really are running and the one we did not ask for is the
|
|
// stray.
|
|
if existing.hasLiveShard() {
|
|
// Only what we do not recognise. A reconnecting agent reports
|
|
// the shards we already know, and disowning those would tell it
|
|
// to stop the very replica that is serving.
|
|
known := map[string]struct{}{}
|
|
for _, sh := range existing.Shards {
|
|
known[sh.ID] = struct{}{}
|
|
}
|
|
var strays []AdoptedShard
|
|
for _, s := range group {
|
|
if _, ok := known[s.ShardID]; !ok {
|
|
strays = append(strays, s)
|
|
}
|
|
}
|
|
unclaimed = append(unclaimed, strays...)
|
|
r.reservePorts(strays)
|
|
continue
|
|
}
|
|
r.log.Warn("a running replica reported by the fleet replaces one we placed",
|
|
"replica", replicaID, "placed_endpoint", existing.Endpoint)
|
|
for _, s := range existing.Shards {
|
|
r.inv.Release(s.ID)
|
|
}
|
|
delete(r.replicas, replicaID)
|
|
}
|
|
|
|
first := group[0]
|
|
m, ok := r.cat.Load().Resolve(first.Model)
|
|
if !ok {
|
|
unclaimed = append(unclaimed, group...)
|
|
r.reservePorts(group)
|
|
continue
|
|
}
|
|
prof, ok := m.ProfileByName(first.Profile)
|
|
if !ok {
|
|
unclaimed = append(unclaimed, group...)
|
|
r.reservePorts(group)
|
|
continue
|
|
}
|
|
|
|
rep := &Replica{
|
|
ID: replicaID, Model: m.Name, Profile: prof.Name,
|
|
Fabric: first.Fabric, Pool: first.Pool,
|
|
// From the agent, which was told it when the shard started. Empty
|
|
// from an agent too old to report it, which reads as unknown.
|
|
ProfileFP: first.ProfileFP,
|
|
Weight: prof.Weight, CreatedAt: r.now(),
|
|
Endpoint: first.Endpoint,
|
|
// From the profile, not from what was reported: the point is to
|
|
// notice when what was reported is short.
|
|
WantShards: max(prof.Nodes, 1),
|
|
AdoptedAt: r.now(),
|
|
}
|
|
r.mergeAdopted(rep, group)
|
|
r.replicas[replicaID] = rep
|
|
|
|
}
|
|
return unclaimed
|
|
}
|
|
|
|
// mergeAdopted folds shards an agent reports into a replica being reassembled.
|
|
//
|
|
// Called both for the first group, which creates the replica, and for every
|
|
// later group belonging to it, so a replica spanning N nodes is built up over N
|
|
// registrations. Shard ids already present are skipped: an agent that
|
|
// reconnects repeats what it is running, and counting those again would carry a
|
|
// replica past its shard count without a second node ever appearing.
|
|
func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
|
have := make(map[string]struct{}, len(rep.Shards))
|
|
for _, sh := range rep.Shards {
|
|
have[sh.ID] = struct{}{}
|
|
}
|
|
for _, s := range group {
|
|
if _, dup := have[s.ShardID]; dup {
|
|
continue
|
|
}
|
|
have[s.ShardID] = struct{}{}
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
|
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
|
HTTPPort: s.HTTPPort, State: s.State,
|
|
})
|
|
// Reclaim the GPUs and the memory, or the next placement hands them
|
|
// out twice. The memory comes from the catalog rather than the agent:
|
|
// what a profile asked for is not something a running process reports.
|
|
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
|
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
|
|
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
|
|
})
|
|
// The endpoint comes from the leader, which may be in any of the
|
|
// groups — the rank 0 node is not necessarily the first to register.
|
|
if s.Rank == 0 && s.Endpoint != "" {
|
|
rep.Endpoint = s.Endpoint
|
|
}
|
|
}
|
|
slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank })
|
|
rep.deriveState(r.now())
|
|
|
|
r.log.Info("adopted replica from the fleet",
|
|
"replica", rep.ID, "model", rep.Model,
|
|
"shards", len(rep.Shards), "want", rep.WantShards, "state", rep.State)
|
|
}
|
|
|
|
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
|
|
// agent, having just told us everything it is running, did not mention.
|
|
//
|
|
// The mirror image of adoption, and the half that was missing. Adoption handles
|
|
// "the agent has something we do not know about"; this handles "we think the
|
|
// agent has something it does not". Both follow from the same rule: the fleet is
|
|
// authoritative about what is running (plan §6.5).
|
|
//
|
|
// Without it, an engine that dies while its agent survives leaves the control
|
|
// plane believing a replica is Ready forever. Observed on a real cluster: the
|
|
// engine was killed, the agent restarted and registered with nothing running,
|
|
// and the replica stayed Ready pointing at a dead port for as long as anyone
|
|
// watched. Traffic survived — the breaker took the endpoint out — but the
|
|
// cluster ran a third short with no sign of it.
|
|
//
|
|
// Safe against the obvious race. A registration follows a fresh connection, so
|
|
// anything we dispatched over a previous one is not running by definition;
|
|
// there is no window where a shard is legitimately started but unreported.
|
|
func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) {
|
|
if nodeID == "" {
|
|
return
|
|
}
|
|
reported := make(map[string]struct{}, len(running))
|
|
for _, s := range running {
|
|
reported[s.ShardID] = struct{}{}
|
|
}
|
|
|
|
var lost []string
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State == Stopped {
|
|
continue
|
|
}
|
|
changed := false
|
|
for i := range rep.Shards {
|
|
sh := &rep.Shards[i]
|
|
if sh.NodeID != nodeID || sh.State.Terminal() {
|
|
continue
|
|
}
|
|
if _, ok := reported[sh.ID]; ok {
|
|
continue
|
|
}
|
|
sh.State = supervisor.Failed
|
|
sh.Err = "the agent reports it is not running"
|
|
r.inv.Release(sh.ID)
|
|
changed = true
|
|
lost = append(lost, sh.ID)
|
|
}
|
|
if changed {
|
|
rep.deriveState(r.now())
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, id := range lost {
|
|
r.log.Warn("a shard we believed was running is not; the agent does not have it",
|
|
"node", nodeID, "shard", id)
|
|
}
|
|
}
|
|
|
|
// reservePorts records ports held by shards nothing owns.
|
|
//
|
|
// They are being stopped, but stopping is not instant and the engine keeps its
|
|
// socket until it exits. Handing the port to a new shard in the meantime makes
|
|
// that shard fail to bind, which fails its replica, which places another one on
|
|
// the same port — a loop that does not end on its own.
|
|
//
|
|
// Called with the lock held.
|
|
func (r *Reconciler) reservePorts(shards []AdoptedShard) {
|
|
for _, s := range shards {
|
|
if s.HTTPPort > 0 {
|
|
r.reservedPorts[s.HTTPPort] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
|
|
// AdoptedShard is one running shard an agent reported at registration.
|
|
type AdoptedShard struct {
|
|
NodeID string
|
|
ReplicaID string
|
|
ShardID string
|
|
Rank int
|
|
GPUIndices []int
|
|
HCAs []string
|
|
HTTPPort int
|
|
State supervisor.State
|
|
Model string
|
|
Profile string
|
|
Pool string
|
|
Fabric string
|
|
ProfileFP string
|
|
// Endpoint is the leader's base URL, set on the rank 0 shard.
|
|
Endpoint string
|
|
}
|
|
|
|
// OnFetchProgress records how far a shard's download has got.
|
|
//
|
|
// Separate from OnShardStatus because it is not a transition: the shard is
|
|
// still starting, and treating each progress report as a state change would
|
|
// reset the timers that decide when a slow start becomes a failed one.
|
|
func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID != shardID {
|
|
continue
|
|
}
|
|
rep.Shards[i].Fetch = &FetchProgress{
|
|
Source: source, BytesDone: done, BytesTotal: total,
|
|
}
|
|
}
|
|
}
|
|
|
|
// OnShardStatus records a status report from an agent.
|
|
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.markShard(replicaID, shardID, st, errMsg)
|
|
r.Notify()
|
|
}
|
|
|
|
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID != shardID {
|
|
continue
|
|
}
|
|
rep.Shards[i].State = st
|
|
if errMsg != "" {
|
|
rep.Shards[i].Err = errMsg
|
|
}
|
|
// A real transition means the download is over, one way or the other.
|
|
// Left in place it would sit in the listing at ninety-nine percent
|
|
// under a shard that has been serving for an hour.
|
|
rep.Shards[i].Fetch = nil
|
|
break
|
|
}
|
|
|
|
prev := rep.State
|
|
rep.deriveState(r.now())
|
|
if rep.State == Ready && prev != Ready {
|
|
// A model that comes up clears its backoff; the next failure starts
|
|
// counting from scratch rather than from an old streak.
|
|
r.attempts[rep.Model] = 0
|
|
delete(r.nextAttempt, rep.Model)
|
|
}
|
|
}
|
|
|
|
// Replicas returns a snapshot of every replica, sorted by id.
|
|
func (r *Reconciler) Replicas() []Replica {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
out := make([]Replica, 0, len(r.replicas))
|
|
for _, rep := range r.replicas {
|
|
clone := *rep
|
|
clone.Shards = slices.Clone(rep.Shards)
|
|
out = append(out, clone)
|
|
}
|
|
slices.SortFunc(out, func(a, b Replica) int {
|
|
if a.ID < b.ID {
|
|
return -1
|
|
}
|
|
if a.ID > b.ID {
|
|
return 1
|
|
}
|
|
return 0
|
|
})
|
|
return out
|
|
}
|
|
|
|
// ReadyReplicas returns the replicas the router may send traffic to.
|
|
func (r *Reconciler) ReadyReplicas() []Replica {
|
|
var out []Replica
|
|
for _, rep := range r.Replicas() {
|
|
if rep.State == Ready {
|
|
out = append(out, rep)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Explain returns the most recent placement result for a model, which is what
|
|
// `ctl explain-placement` prints.
|
|
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
res, ok := r.lastExplain[model]
|
|
return res, ok
|
|
}
|
|
|
|
func addrOf(n inventory.Node) string {
|
|
if n.MgmtAddr != "" {
|
|
return n.MgmtAddr
|
|
}
|
|
if n.Hostname != "" {
|
|
return n.Hostname
|
|
}
|
|
return n.ID
|
|
}
|
|
|
|
// profileMemory is what a model's profile declared it needs, or zero.
|
|
//
|
|
// Looked up when a replica is adopted after a restart, because the allocation
|
|
// has to carry the same claim it did before — a control plane that forgot every
|
|
// claim on restart would place a second large model onto a node already full of
|
|
// the first.
|
|
func (r *Reconciler) profileMemory(model, profile string) int {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return 0
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == profile {
|
|
return p.MinMemoryGB
|
|
}
|
|
}
|
|
return 0
|
|
}
|