Issue #3 asked for commands to stop and start the system, and 05aa21c gave
only one machine's services. Now:
- llmgw stop sets serving.stopped in the cluster's settings, live: the
reconciler wants every model at no replicas, freeing every GPU, and the
gateway answers every POST under /v1/ with 503 system_stopped. The control
planes, agents and mirror keep running, and no model's definition changes.
- llmgw start clears it; every model is placed again as it asks.
- llmgw restart has every agent restart over its stream (RestartAgent), then
the other control planes over ssh, standbys before a leader, then this
one; engines keep serving.
- llmgw status is the cluster's summary, saying when it is stopped.
One machine's services moved to llmgw service status|start|stop|restart.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1487 lines
51 KiB
Go
1487 lines
51 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
crypterand "crypto/rand"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"slices"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
|
)
|
|
|
|
// ShardCommand is everything an agent needs to launch one shard.
|
|
type ShardCommand struct {
|
|
ReplicaID string
|
|
ShardID string
|
|
NodeID string
|
|
Spec engines.ShardSpec
|
|
// StartupTimeout comes from the profile; model loading dominates it.
|
|
StartupTimeout time.Duration
|
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
|
// travel with the command so the agent can report them back after a
|
|
// control plane restart (plan §6.5).
|
|
Pool string
|
|
Fabric string
|
|
// ProfileFP travels for the same reason, and answers a different question:
|
|
// whether the profile has been edited since this shard was started.
|
|
ProfileFP string
|
|
}
|
|
|
|
// Dispatcher sends commands to agents.
|
|
//
|
|
// It is an interface so the reconciliation loop can be tested without a network:
|
|
// the loop's logic is where the interesting failures are, not the transport.
|
|
type Dispatcher interface {
|
|
StartShard(ctx context.Context, cmd ShardCommand) error
|
|
StopShard(ctx context.Context, nodeID, shardID string) error
|
|
DrainShard(ctx context.Context, nodeID, shardID string) error
|
|
}
|
|
|
|
// Config tunes the reconciliation loop.
|
|
type Config struct {
|
|
// Location is the time zone models' scaling schedules are written in;
|
|
// nil is the control plane's own.
|
|
Location *time.Location
|
|
// Interval is how often desired and actual are compared when nothing else
|
|
// happens. Events wake the loop sooner.
|
|
Interval time.Duration
|
|
// BaseBackoff is how long to wait before rebuilding a failed replica.
|
|
BaseBackoff time.Duration
|
|
// MaxBackoff caps the exponential growth.
|
|
MaxBackoff time.Duration
|
|
// UnreachableAfter is how long without a heartbeat before a node is
|
|
// unreachable.
|
|
UnreachableAfter time.Duration
|
|
// LostAfter is how long unreachable before its replicas are declared dead.
|
|
// Longer than UnreachableAfter on purpose: a brief network blip must not
|
|
// tear down a replica that is still serving perfectly well.
|
|
LostAfter time.Duration
|
|
// DrainGrace bounds how long a draining replica waits for its replacement.
|
|
// Without a bound, draining a node whose model cannot be placed anywhere
|
|
// else would never finish.
|
|
DrainGrace time.Duration
|
|
// PreemptGrace is how long a replica stopped to make room for another
|
|
// is given to finish what it is serving: a request, not a replacement.
|
|
PreemptGrace time.Duration
|
|
// PreemptHold is how long a model that gave up room may not take it back,
|
|
// so the replica it was given up for is placed first.
|
|
PreemptHold time.Duration
|
|
// ScaleUpWaiting is the average number of requests waiting per ready
|
|
// replica, held for ScaleUpAfter, that adds a replica — up to the model's
|
|
// max. ScaleDownAfter is how long with nothing waiting before one of the
|
|
// replicas added that way is taken away again, down to its min.
|
|
ScaleUpWaiting float64
|
|
ScaleUpAfter time.Duration
|
|
ScaleDownAfter time.Duration
|
|
// ParkAfter is how many failures in a row stop a model being retried.
|
|
//
|
|
// Backoff alone never gives up: it grows to MaxBackoff and keeps trying,
|
|
// and a model whose weights its engine cannot load — an architecture the
|
|
// installed llama.cpp does not know — was started 155 times over eight
|
|
// hours on the live cluster, each time failing the same way, with nothing
|
|
// to show for it but the log. A parked model is not placed again until its
|
|
// definition changes or an operator runs `llmgw ctl retry`.
|
|
ParkAfter int
|
|
// PortRange is the per-node port range for engine HTTP and dist-init.
|
|
PortRange [2]int
|
|
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
|
MgmtInterface string
|
|
// EngineKey gives each replica the key its engine demands. Derived from
|
|
// the replica id rather than stored, so a control plane that restarts —
|
|
// or a second one — computes the same key for a replica it adopts. Nil
|
|
// starts engines without one.
|
|
EngineKey func(replicaID string) string
|
|
// EvictionGrace is how long a replica must have gone unasked-for before it
|
|
// may be stopped to make room for another model.
|
|
//
|
|
// Shorter than IdleTimeout on purpose: a replica idle for five minutes is
|
|
// not worth reclaiming on its own, and is worth reclaiming when something
|
|
// else needs the cards. Still minutes rather than seconds, because stopping
|
|
// one throws away the KV cache prefix routing spent filling.
|
|
EvictionGrace time.Duration
|
|
// MinLifetime is how long a replica is safe from eviction after it starts
|
|
// serving, whatever else wants its cards.
|
|
//
|
|
// The thing that stops two models on a cluster with room for one from
|
|
// taking turns evicting each other. A load costs minutes; oscillating at a
|
|
// load per cycle is worse for both models than one of them waiting.
|
|
MinLifetime time.Duration
|
|
// IdleTimeout is how long an on-demand model stays loaded with nobody
|
|
// asking for it. A model may override it.
|
|
IdleTimeout time.Duration
|
|
// SettleFor is how long after start to leave placement alone while agents
|
|
// reconnect.
|
|
//
|
|
// A control plane that has just started knows nothing, and the first
|
|
// reconcile tick runs before any agent has registered. It sees an empty
|
|
// fleet and acts on it: "needs 2 nodes on one fabric, but no node reports a
|
|
// fabric" is that, and it is harmless only because adoption catches up.
|
|
// What is not harmless is placing on the fraction of the fleet that
|
|
// happens to have reported first — after one restart both replicas of a
|
|
// model landed on the single node that had registered, when four were
|
|
// coming.
|
|
//
|
|
// Adoption still runs throughout; only placement waits. The window ends
|
|
// early once the fleet stops growing, so a small cluster does not sit idle
|
|
// for the whole duration — but not too early: an agent that lost its
|
|
// connection reconnects on a backoff reaching thirty seconds, so "no new
|
|
// node for a few seconds" means nothing.
|
|
SettleFor time.Duration
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
SettleFor: 60 * time.Second,
|
|
IdleTimeout: 15 * time.Minute,
|
|
EvictionGrace: 2 * time.Minute,
|
|
MinLifetime: 5 * time.Minute,
|
|
DrainGrace: 5 * time.Minute,
|
|
PreemptGrace: 30 * time.Second,
|
|
PreemptHold: 5 * time.Minute,
|
|
ParkAfter: 5,
|
|
ScaleUpWaiting: 1,
|
|
ScaleUpAfter: time.Minute,
|
|
ScaleDownAfter: 10 * time.Minute,
|
|
PortRange: [2]int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
d := DefaultConfig()
|
|
if c.Interval <= 0 {
|
|
c.Interval = d.Interval
|
|
}
|
|
if c.BaseBackoff <= 0 {
|
|
c.BaseBackoff = d.BaseBackoff
|
|
}
|
|
if c.PreemptGrace <= 0 {
|
|
c.PreemptGrace = d.PreemptGrace
|
|
}
|
|
if c.PreemptHold <= 0 {
|
|
c.PreemptHold = d.PreemptHold
|
|
}
|
|
if c.MaxBackoff < c.BaseBackoff {
|
|
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
|
|
}
|
|
if c.IdleTimeout <= 0 {
|
|
c.IdleTimeout = d.IdleTimeout
|
|
}
|
|
if c.EvictionGrace <= 0 {
|
|
c.EvictionGrace = d.EvictionGrace
|
|
}
|
|
if c.MinLifetime <= 0 {
|
|
c.MinLifetime = d.MinLifetime
|
|
}
|
|
if c.SettleFor < 0 {
|
|
c.SettleFor = 0
|
|
} else if c.SettleFor == 0 {
|
|
c.SettleFor = d.SettleFor
|
|
}
|
|
if c.UnreachableAfter <= 0 {
|
|
c.UnreachableAfter = d.UnreachableAfter
|
|
}
|
|
if c.LostAfter < c.UnreachableAfter {
|
|
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
|
|
}
|
|
if c.DrainGrace <= 0 {
|
|
c.DrainGrace = d.DrainGrace
|
|
}
|
|
if c.ParkAfter <= 0 {
|
|
c.ParkAfter = d.ParkAfter
|
|
}
|
|
if c.ScaleUpWaiting <= 0 {
|
|
c.ScaleUpWaiting = d.ScaleUpWaiting
|
|
}
|
|
if c.ScaleUpAfter <= 0 {
|
|
c.ScaleUpAfter = d.ScaleUpAfter
|
|
}
|
|
if c.ScaleDownAfter <= 0 {
|
|
c.ScaleDownAfter = d.ScaleDownAfter
|
|
}
|
|
if c.PortRange[1] <= c.PortRange[0] {
|
|
c.PortRange = d.PortRange
|
|
}
|
|
if c.MgmtInterface == "" {
|
|
c.MgmtInterface = d.MgmtInterface
|
|
}
|
|
}
|
|
|
|
// Reconciler compares the catalog against what is running and closes the gap.
|
|
type Reconciler struct {
|
|
// leader gates every action; nil means act unconditionally.
|
|
leader Leader
|
|
|
|
cfg Config
|
|
cat *catalog.Live
|
|
inv *inventory.Store
|
|
sched *placement.Scheduler
|
|
disp Dispatcher
|
|
log *slog.Logger
|
|
now func() time.Time
|
|
// notify tells people about parked models and lost nodes; nil tells
|
|
// no one.
|
|
notify atomic.Pointer[notify.Notifier]
|
|
|
|
mu sync.Mutex
|
|
replicas map[string]*Replica
|
|
// nextAttempt is the earliest a model may be rebuilt, per model.
|
|
nextAttempt map[string]time.Time
|
|
// attempts counts consecutive failures per model, for the backoff.
|
|
attempts map[string]int
|
|
// lastExplain keeps the most recent placement failure per model, so
|
|
// `ctl explain-placement` can answer without re-running placement.
|
|
lastExplain map[string]placement.Result
|
|
seq int
|
|
// reservedPorts holds ports the fleet reports as in use but no replica
|
|
// claims, so the allocator cannot hand out what something is still bound to.
|
|
reservedPorts map[int]struct{}
|
|
// wanted records when an on-demand model was last asked for.
|
|
//
|
|
// The whole of on-demand loading, kept here rather than in the catalog
|
|
// because it is state about this moment and not about what the model is.
|
|
// A model nobody has wanted has no entry and is simply not running.
|
|
wanted map[string]time.Time
|
|
// restartQueue holds, per model, the replicas an operator asked to be
|
|
// restarted and not yet begun. One is replaced at a time: replacing them
|
|
// all at once would need the memory of every replica twice over.
|
|
restartQueue map[string][]string
|
|
// restartInPlace are the replicas asked to restart by stopping first.
|
|
restartInPlace map[string]bool
|
|
// restartPace is, per model, how many of its replicas may be restarting
|
|
// at once: one unless an operator said otherwise.
|
|
restartPace map[string]RestartPace
|
|
// restartRefused says, by replica, why a restart was given up rather
|
|
// than leave the model without it.
|
|
restartRefused map[string]string
|
|
// parked holds models that failed ParkAfter times in a row.
|
|
parked map[string]parkRecord
|
|
// extra is how many replicas load has added above a model's base, and
|
|
// busySince/quietSince when its load last crossed the thresholds.
|
|
extra map[string]int
|
|
busySince map[string]time.Time
|
|
quietSince map[string]time.Time
|
|
// placementLogged remembers the last "no placement" said per model, so it
|
|
// is said when it changes rather than on every five-second pass.
|
|
placementLogged map[string]placementNote
|
|
// preempting is, per model, the replicas stopped to make room for it and
|
|
// not gone yet; held is, per model that gave them up, until when it may
|
|
// not take the room back; preempted counts, per model, replicas it lost.
|
|
preempting map[string][]string
|
|
held map[string]hold
|
|
preempted map[string]int
|
|
// quotas is the tenancy tree's capacity quotas as last read; nil is none.
|
|
quotas func() *quota.Policy
|
|
// measurements are benchmark results by model/profile; nil is none.
|
|
measurements func() map[string]Measured
|
|
|
|
wake chan struct{}
|
|
stop chan struct{}
|
|
done chan struct{}
|
|
once sync.Once
|
|
|
|
// startedAt and lastNodeCount track the settling window: placement waits
|
|
// until the fleet has stopped growing, or the window expires.
|
|
startedAt time.Time
|
|
lastNodeCount int
|
|
steadySince time.Time
|
|
// stopped is the operators having stopped the service: every model is
|
|
// wanted at no replicas until they start it again.
|
|
stopped atomic.Bool
|
|
// stoodBy is whether the last pass found this process a standby: one
|
|
// that then takes over settles again, as at start.
|
|
stoodBy bool
|
|
}
|
|
|
|
// New builds a reconciler.
|
|
func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Reconciler{
|
|
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
|
|
sched: placement.New(inv),
|
|
now: time.Now,
|
|
replicas: map[string]*Replica{},
|
|
nextAttempt: map[string]time.Time{},
|
|
attempts: map[string]int{},
|
|
reservedPorts: map[int]struct{}{},
|
|
wanted: map[string]time.Time{},
|
|
restartQueue: map[string][]string{},
|
|
restartInPlace: map[string]bool{},
|
|
restartPace: map[string]RestartPace{},
|
|
restartRefused: map[string]string{},
|
|
parked: map[string]parkRecord{},
|
|
extra: map[string]int{},
|
|
busySince: map[string]time.Time{},
|
|
quietSince: map[string]time.Time{},
|
|
placementLogged: map[string]placementNote{},
|
|
lastExplain: map[string]placement.Result{},
|
|
preempting: map[string][]string{},
|
|
held: map[string]hold{},
|
|
preempted: map[string]int{},
|
|
wake: make(chan struct{}, 1),
|
|
stop: make(chan struct{}),
|
|
done: make(chan struct{}),
|
|
startedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
// SetClock replaces the time source, for tests.
|
|
func (r *Reconciler) SetClock(now func() time.Time) {
|
|
r.now = now
|
|
r.startedAt = now()
|
|
}
|
|
|
|
// Leader reports whether this process should be acting on the fleet.
|
|
type Leader interface{ IsLeader() bool }
|
|
|
|
// SetLeadership gates every action on holding leadership.
|
|
//
|
|
// A setter rather than a constructor argument because the thing that decides
|
|
// leadership needs a store, and the store is wired after the controller. Nil
|
|
// leaves the reconciler acting unconditionally, which is what a single control
|
|
// plane wants and what every test expects.
|
|
func (r *Reconciler) SetLeadership(l Leader) { r.leader = l }
|
|
|
|
// Start runs the loop until Close.
|
|
func (r *Reconciler) Start(ctx context.Context) {
|
|
go func() {
|
|
defer close(r.done)
|
|
t := time.NewTicker(r.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
r.Reconcile(ctx)
|
|
for {
|
|
select {
|
|
case <-r.stop:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-r.wake:
|
|
r.Reconcile(ctx)
|
|
case <-t.C:
|
|
r.Reconcile(ctx)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Close stops the loop.
|
|
func (r *Reconciler) Close() {
|
|
r.once.Do(func() {
|
|
close(r.stop)
|
|
<-r.done
|
|
})
|
|
}
|
|
|
|
// Notify wakes the loop, for when something changed and waiting for the next
|
|
// tick would add pointless latency.
|
|
func (r *Reconciler) Notify() {
|
|
select {
|
|
case r.wake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Reconcile performs one pass. It is exported so tests can drive it directly
|
|
// instead of racing a ticker.
|
|
func (r *Reconciler) Reconcile(ctx context.Context) {
|
|
// A standby does nothing at all here. Everything below issues commands to
|
|
// agents — placing, evicting, draining, reaping — and two control planes
|
|
// doing that against one fleet is how a model ends up with twice the
|
|
// replicas it asked for, each holding GPUs the other believes are free.
|
|
//
|
|
// Unset means leader, so a single control plane and every test behave
|
|
// exactly as they did before leadership existed.
|
|
if r.leader != nil && !r.leader.IsLeader() {
|
|
r.stoodBy = true
|
|
return
|
|
}
|
|
// Taking over from another control plane is a start as far as the fleet
|
|
// is concerned: a standby refuses agents, so it knows none of what runs,
|
|
// and placing before they have reconnected and said starts a second
|
|
// copy of every replica — then drains the healthy one it finds next.
|
|
// Measured on the cluster: a failover replaced the guardrails' judge and
|
|
// a model's replica within twenty seconds, for nothing.
|
|
// From the moment it leads, not from when the process started: a
|
|
// standby a minute old had its window run out while it was refusing the
|
|
// very agents the window waits for.
|
|
if r.stoodBy {
|
|
r.stoodBy = false
|
|
r.resettle()
|
|
}
|
|
ctx, p := beginPass(ctx)
|
|
defer p.end()
|
|
r.expireNodes(ctx)
|
|
r.expireStuckReplicas()
|
|
r.drainCordonedNodes(ctx)
|
|
r.reapFailed(ctx)
|
|
|
|
// Everything above is about what is already running and stays correct on a
|
|
// partial fleet. Placement is the part that must not act on one.
|
|
if !r.fleetHasSettled() {
|
|
return
|
|
}
|
|
|
|
for _, m := range r.cat.Load().Models() {
|
|
r.reconcileModel(ctx, m)
|
|
}
|
|
r.stopReplicasOfRemovedModels(ctx)
|
|
}
|
|
|
|
// stopReplicasOfRemovedModels tears down what the catalog no longer asks for.
|
|
//
|
|
// reconcileModel only ever visits models that are in the catalog, so a replica
|
|
// whose model has been removed was visited by nothing: it kept its GPUs, kept
|
|
// counting towards the inventory, and kept running an engine for a name no
|
|
// client could ask for. The endpoint publisher already skipped it with the
|
|
// comment "the reconciler will tear it down", which was simply not true.
|
|
//
|
|
// It mattered little while a catalog could only change by restarting the
|
|
// process. It matters now that models come and go while the cluster runs, which
|
|
// is the whole point of making the catalog live.
|
|
//
|
|
// Ordered after the model loop so a rebuild elsewhere has already claimed what
|
|
// it needs: freeing these GPUs first would let a replacement land on hardware
|
|
// that is about to be freed anyway, and churn two nodes instead of one.
|
|
func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
|
cat := r.cat.Load()
|
|
|
|
r.mu.Lock()
|
|
var gone []*Replica
|
|
for _, rep := range r.replicas {
|
|
if _, ok := cat.Resolve(rep.Model); !ok {
|
|
gone = append(gone, rep)
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range gone {
|
|
r.log.Info("stopping a replica whose model left the catalog",
|
|
"replica", rep.ID, "model", rep.Model)
|
|
r.stopReplica(ctx, rep.ID, "its model left the catalog")
|
|
}
|
|
}
|
|
|
|
// SetStopped stops every model, or lets each run as its catalog entry
|
|
// asks again, from the next pass.
|
|
func (r *Reconciler) SetStopped(stopped bool) {
|
|
if r.stopped.Swap(stopped) != stopped {
|
|
r.Notify()
|
|
}
|
|
}
|
|
|
|
// Stopped is whether the operators have stopped the service.
|
|
func (r *Reconciler) Stopped() bool { return r.stopped.Load() }
|
|
|
|
// resettle opens the settling window again from now, for a control plane
|
|
// that has just become the leader.
|
|
func (r *Reconciler) resettle() {
|
|
// Like the rest of the window, touched only by Reconcile's own pass.
|
|
r.startedAt, r.lastNodeCount, r.steadySince = r.now(), -1, r.now()
|
|
r.log.Info("leading now: placement waits for the fleet to report what runs", "for up to", r.cfg.SettleFor)
|
|
}
|
|
|
|
// fleetHasSettled reports whether placement may run yet.
|
|
//
|
|
// Two ways out, because either alone is wrong. Waiting for the window to expire
|
|
// would make every restart cost that much downtime even when the whole fleet
|
|
// reconnected in a second. Waiting only for the count to stop changing would
|
|
// hang forever on a cluster whose nodes never all come back.
|
|
func (r *Reconciler) fleetHasSettled() bool {
|
|
if r.cfg.SettleFor <= 0 {
|
|
return true
|
|
}
|
|
now := r.now()
|
|
if now.Sub(r.startedAt) >= r.cfg.SettleFor {
|
|
return true
|
|
}
|
|
|
|
n := len(r.inv.Nodes())
|
|
if n != r.lastNodeCount {
|
|
r.lastNodeCount, r.steadySince = n, now
|
|
return false
|
|
}
|
|
// Steady for a moment, and at least one node present. A count that has been
|
|
// zero since start is not a settled fleet, it is one that has not arrived.
|
|
if n == 0 {
|
|
return false
|
|
}
|
|
// Half the window, not a quarter of it, and the window is a minute.
|
|
//
|
|
// Measured on the cluster: after a restart node05 registered in one second
|
|
// and the other four arrived seventeen seconds later, because a
|
|
// disconnected agent reconnects on a backoff that reaches thirty seconds.
|
|
// A five-second steady period declared a fleet of one settled and resumed
|
|
// placement while four nodes were still backing off — the exact situation
|
|
// this exists to avoid, reached by being impatient about it.
|
|
settled := now.Sub(r.steadySince) >= r.cfg.SettleFor/2
|
|
if settled {
|
|
r.log.Info("fleet settled; placement resuming",
|
|
"nodes", n, "waited", now.Sub(r.startedAt).Round(time.Millisecond))
|
|
r.startedAt = time.Time{} // never wait again in this process
|
|
}
|
|
return settled
|
|
}
|
|
|
|
// expireStuckReplicas fails replicas that never finished coming up.
|
|
//
|
|
// Every other path out of Starting depends on the agent reporting something. A
|
|
// replica whose agent never reports — the start was a duplicate shard id and
|
|
// became a no-op, the node went quiet in a way expiry did not catch, the engine
|
|
// wedged before its first health check — sits in Starting for as long as the
|
|
// process lives, holding its GPUs and counting towards the model's replica
|
|
// count while serving nothing.
|
|
//
|
|
// Found on a real cluster: two replicas sat in Starting for eight minutes past
|
|
// a two-minute startup timeout, and nothing would ever have moved them.
|
|
func (r *Reconciler) expireStuckReplicas() {
|
|
now := r.now()
|
|
var stuck []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
|
|
rep.startDone(now)
|
|
stuck = append(stuck, rep)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range stuck {
|
|
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
|
|
"replica", rep.ID, "model", rep.Model, "state", "starting",
|
|
"timeout", r.startupTimeout(rep))
|
|
}
|
|
}
|
|
|
|
// startupTimeout is the profile's, or the configured default when the profile
|
|
// does not say — a replica with no deadline at all is the thing this exists to
|
|
// prevent.
|
|
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
|
|
if m, ok := r.cat.Load().Resolve(rep.Model); ok {
|
|
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
|
|
return p.StartupTimeout
|
|
}
|
|
}
|
|
// No profile deadline: a generous default, because the thing being prevented
|
|
// is a replica with no deadline at all rather than one with a loose one.
|
|
// Loading a large model across nodes genuinely takes tens of minutes.
|
|
return 30 * time.Minute
|
|
}
|
|
|
|
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
|
|
// replicas of nodes that are gone.
|
|
//
|
|
// The two-stage delay is deliberate: a brief network blip must not tear down
|
|
// replicas that are still serving traffic perfectly well, and rebuilding a
|
|
// multi-node replica costs tens of minutes of model loading.
|
|
func (r *Reconciler) expireNodes(ctx context.Context) {
|
|
now := r.now()
|
|
for _, n := range r.inv.Nodes() {
|
|
silent := now.Sub(n.LastSeen)
|
|
switch {
|
|
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
|
|
r.inv.SetState(n.ID, inventory.Unreachable)
|
|
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
|
|
_, span := startSpan(ctx, "node unreachable", attribute.String("llmgw.node", n.ID),
|
|
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
|
|
span.End()
|
|
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
|
|
_, span := startSpan(ctx, "node lost", attribute.String("llmgw.node", n.ID),
|
|
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
|
|
r.inv.SetState(n.ID, inventory.Lost)
|
|
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
|
"node", n.ID, "silent_for", silent.Round(time.Second))
|
|
r.notify.Load().Notify(notify.NodeLost, n.ID,
|
|
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
|
|
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
|
|
span.SetAttributes(attribute.StringSlice("llmgw.replicas_failed",
|
|
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")))
|
|
span.End()
|
|
}
|
|
}
|
|
}
|
|
|
|
// drainCordonedNodes moves work off nodes an operator is taking out of service.
|
|
//
|
|
// Draining a node is not the same as losing one: the engines are healthy and
|
|
// still serving, so their replicas are marked Draining rather than Failed. That
|
|
// makes the model look short of replicas, a replacement is placed elsewhere,
|
|
// and the original is stopped only once the replacement is serving — which is
|
|
// the difference between a rolling move and an outage.
|
|
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
|
|
var toDrain []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
for _, s := range rep.Shards {
|
|
n, ok := r.inv.Get(s.NodeID)
|
|
if ok && n.State == inventory.Draining {
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
toDrain = append(toDrain, rep)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
shards := map[string][]Shard{}
|
|
for _, rep := range toDrain {
|
|
shards[rep.ID] = slices.Clone(rep.Shards)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range toDrain {
|
|
r.log.Info("draining replica off a node being taken out of service",
|
|
"replica", rep.ID, "nodes", rep.NodeIDs())
|
|
r.drainShards(ctx, rep.ID, shards[rep.ID], "its node is being drained")
|
|
}
|
|
}
|
|
|
|
// failReplicasOn fails every replica on a node, and says which.
|
|
func (r *Reconciler) failReplicasOn(nodeID, reason string) []string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
var ids []string
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() || !rep.OnNode(nodeID) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = reason
|
|
rep.startDone(r.now())
|
|
ids = append(ids, rep.ID)
|
|
}
|
|
slices.Sort(ids)
|
|
return ids
|
|
}
|
|
|
|
// reapFailed releases the resources of terminal replicas and tells the agents
|
|
// to clean up whatever might still be running.
|
|
func (r *Reconciler) reapFailed(ctx context.Context) {
|
|
r.mu.Lock()
|
|
var doomed []*Replica
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() {
|
|
doomed = append(doomed, rep)
|
|
}
|
|
}
|
|
parkedNow := map[string]bool{}
|
|
for _, rep := range doomed {
|
|
delete(r.replicas, rep.ID)
|
|
rep.startDone(r.now())
|
|
if rep.State == Failed {
|
|
// Carry the failure count on the model, not the replica: the
|
|
// replica is about to disappear, and the backoff has to survive it.
|
|
r.attempts[rep.Model]++
|
|
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
|
|
if n := r.attempts[rep.Model]; n >= r.cfg.ParkAfter {
|
|
if _, already := r.parked[rep.Model]; !already {
|
|
m, _ := r.cat.Load().Resolve(rep.Model)
|
|
r.parked[rep.Model] = parkRecord{
|
|
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
|
|
}
|
|
parkedNow[rep.ID] = true
|
|
r.log.Error("model parked: it failed every time it started, and is not retried "+
|
|
"until its definition changes or `llmgw ctl retry` is run",
|
|
"model", rep.Model, "failures", n, "last_error", rep.Err)
|
|
r.notify.Load().Notify(notify.ModelParked, rep.Model,
|
|
fmt.Sprintf("model %s is parked: it failed %d times in a row (%s); `llmgw ctl retry %s` tries again",
|
|
rep.Model, n, rep.Err, rep.Model),
|
|
map[string]any{"model": rep.Model, "failures": n, "last_error": rep.Err})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range doomed {
|
|
// rep is out of the map now, so no agent report can reach it any more.
|
|
sctx, span := startSpan(ctx, "reap replica", replicaAttrs(rep)...)
|
|
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
|
|
if rep.State == Failed {
|
|
r.mu.Lock()
|
|
attempts := r.attempts[rep.Model]
|
|
retry := r.backoff(attempts)
|
|
r.mu.Unlock()
|
|
r.log.Error("replica failed",
|
|
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
|
|
"retry_in", retry.Round(time.Second))
|
|
span.SetAttributes(
|
|
attribute.String("llmgw.error", rep.Err),
|
|
attribute.Int("llmgw.failures_in_a_row", attempts),
|
|
attribute.Float64("llmgw.retry_in_seconds", retry.Seconds()),
|
|
attribute.Bool("llmgw.parked", parkedNow[rep.ID]),
|
|
)
|
|
}
|
|
// Stop every shard, including the ones that looked healthy: a partially
|
|
// alive replica holds GPUs while serving nothing.
|
|
for _, s := range rep.Shards {
|
|
if err := r.disp.StopShard(sctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard of a dead replica failed",
|
|
"replica", rep.ID, "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(rep.ID)
|
|
span.End()
|
|
}
|
|
}
|
|
|
|
// drainShards tells a replica's nodes to stop routing to its shards, as one
|
|
// "drain replica" span saying why.
|
|
func (r *Reconciler) drainShards(ctx context.Context, id string, shards []Shard, why string) {
|
|
nodes := make([]string, 0, len(shards))
|
|
for _, s := range shards {
|
|
nodes = append(nodes, s.NodeID)
|
|
}
|
|
ctx, span := startSpan(ctx, "drain replica",
|
|
attribute.String("llmgw.replica", id),
|
|
attribute.StringSlice("llmgw.nodes", nodes),
|
|
attribute.String("llmgw.reason", why))
|
|
defer span.End()
|
|
for _, s := range shards {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// backoff grows with consecutive failures and is jittered, so a cluster-wide
|
|
// problem does not produce synchronized retry storms across every model.
|
|
func (r *Reconciler) backoff(attempt int) time.Duration {
|
|
if attempt <= 0 {
|
|
return 0
|
|
}
|
|
d := r.cfg.BaseBackoff << min(attempt-1, 16)
|
|
if d > r.cfg.MaxBackoff || d <= 0 {
|
|
d = r.cfg.MaxBackoff
|
|
}
|
|
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
|
|
return d - d/8 + jitter
|
|
}
|
|
|
|
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
|
// Snapshot under the lock and work from values afterwards. A replica
|
|
// pointer stops being safe the moment it is in the map: an agent's status
|
|
// report can mutate it from another goroutine at any time.
|
|
r.mu.Lock()
|
|
var liveIDs, drainingIDs, retiringIDs, avoid []string
|
|
readyLive := 0
|
|
for _, rep := range r.replicas {
|
|
if rep.Model != m.Name {
|
|
continue
|
|
}
|
|
switch {
|
|
case rep.State.Counts() && !rep.RetiringAt.IsZero():
|
|
retiringIDs = append(retiringIDs, rep.ID)
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
case rep.State.Counts():
|
|
liveIDs = append(liveIDs, rep.ID)
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
if rep.State == Ready {
|
|
readyLive++
|
|
}
|
|
case rep.State == Draining:
|
|
drainingIDs = append(drainingIDs, rep.ID)
|
|
}
|
|
}
|
|
// The next restarts an operator queued, as many at once as the pace
|
|
// allows: one by default, each begun once the last is done.
|
|
restarting := ""
|
|
pace := r.restartPace[m.Name]
|
|
inPlaceNow := 0
|
|
for _, id := range retiringIDs {
|
|
if r.restartInPlace[id] {
|
|
inPlaceNow++
|
|
}
|
|
}
|
|
for len(retiringIDs) < pace.width() {
|
|
id, ok := r.nextRestartLocked(m.Name)
|
|
if !ok {
|
|
break
|
|
}
|
|
// Up to MaxUnavailable of them stop first; the rest are surged.
|
|
if inPlaceNow < pace.MaxUnavailable {
|
|
r.restartInPlace[id] = true
|
|
inPlaceNow++
|
|
}
|
|
r.replicas[id].RetiringAt = r.now()
|
|
retiringIDs = append(retiringIDs, id)
|
|
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
|
|
readyLive--
|
|
restarting = id
|
|
r.log.Info("restarting a replica", "replica", id, "model", m.Name,
|
|
"in_place", r.restartInPlace[id], "at_once", pace.width())
|
|
}
|
|
if _, queued := r.restartQueue[m.Name]; !queued && len(retiringIDs) == 0 {
|
|
delete(r.restartPace, m.Name)
|
|
}
|
|
blockedUntil := r.nextAttempt[m.Name]
|
|
park, isParked := r.parked[m.Name]
|
|
if isParked && park.fp != modelFingerprint(m) {
|
|
// Somebody changed the model — a new source, an edited profile — so
|
|
// the reason it kept failing may be gone. Worth one more streak.
|
|
delete(r.parked, m.Name)
|
|
r.attempts[m.Name] = 0
|
|
delete(r.nextAttempt, m.Name)
|
|
isParked, blockedUntil = false, time.Time{}
|
|
r.log.Info("model unparked: its definition changed", "model", m.Name)
|
|
}
|
|
r.mu.Unlock()
|
|
if restarting != "" {
|
|
_, span := startSpan(ctx, "restart replica",
|
|
attribute.String("llmgw.replica", restarting), attribute.String("llmgw.model", m.Name))
|
|
span.End()
|
|
}
|
|
|
|
desired := r.desiredFor(m)
|
|
outcome := notTried
|
|
switch {
|
|
case len(liveIDs) < desired && isParked:
|
|
// Not placed. Its failure is on every status the operator reads.
|
|
case len(liveIDs) < desired && r.isHeld(m.Name):
|
|
// It gave up room to a model of a higher class, which is placed
|
|
// first; taking it back now would undo the preemption.
|
|
case len(liveIDs) < desired:
|
|
if r.now().Before(blockedUntil) {
|
|
// Still in backoff. Rebuilding immediately would hot-loop on a
|
|
// model that cannot start — a missing weights path, say — and bury
|
|
// the real error in noise.
|
|
return
|
|
}
|
|
sctx, span := startSpan(ctx, "scale up",
|
|
attribute.String("llmgw.model", m.Name),
|
|
attribute.Int("llmgw.replicas", len(liveIDs)),
|
|
attribute.Int("llmgw.desired", desired))
|
|
outcome = r.scaleUp(sctx, m, avoid)
|
|
span.SetAttributes(attribute.String("llmgw.outcome", outcome.String()))
|
|
span.End()
|
|
case len(liveIDs) > desired:
|
|
sctx, span := startSpan(ctx, "scale down",
|
|
attribute.String("llmgw.model", m.Name),
|
|
attribute.Int("llmgw.replicas", len(liveIDs)),
|
|
attribute.Int("llmgw.desired", desired))
|
|
r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired")
|
|
span.End()
|
|
}
|
|
|
|
// A drained replica is stopped once its replacement is serving, or once the
|
|
// grace period expires. Stopping it the moment it was drained would cut
|
|
// in-flight requests, which is the thing draining exists to avoid.
|
|
replaced := len(liveIDs) >= desired
|
|
for _, id := range drainingIDs {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
var expired bool
|
|
grace, preemptedFor := r.cfg.DrainGrace, ""
|
|
if ok {
|
|
if rep.PreemptedFor != "" {
|
|
grace, preemptedFor = r.cfg.PreemptGrace, rep.PreemptedFor
|
|
}
|
|
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
switch {
|
|
case !ok:
|
|
case replaced:
|
|
r.stopReplica(ctx, id, "drained, and its replacement serves")
|
|
case expired && preemptedFor != "":
|
|
r.log.Info("stopping a preempted replica", "replica", id,
|
|
"for", preemptedFor, "after", grace)
|
|
r.stopReplica(ctx, id, "preempted for "+preemptedFor+", and its grace is over")
|
|
case expired:
|
|
// The replacement never arrived — often because nowhere else can
|
|
// host it. Draining forever would leave the node undrainable.
|
|
r.log.Warn("stopping a drained replica without a replacement",
|
|
"replica", id, "after", grace)
|
|
r.stopReplica(ctx, id, "drained, and no replacement arrived within "+grace.String())
|
|
}
|
|
}
|
|
|
|
r.retire(ctx, m, retiringIDs, readyLive, len(liveIDs), desired, outcome, isParked)
|
|
}
|
|
|
|
// scaleUp places and starts one replica per pass.
|
|
//
|
|
// One at a time on purpose: placement decisions depend on what is already
|
|
// allocated, and starting several at once from the same inventory snapshot
|
|
// would hand them the same GPUs.
|
|
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome {
|
|
_, pspan := startSpan(ctx, "place",
|
|
attribute.String("llmgw.model", m.Name),
|
|
attribute.StringSlice("llmgw.avoid", avoid))
|
|
profiles := r.orderedProfiles(m)
|
|
res := r.sched.Place(profiles, avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
// Spreading across failure domains is a preference, not a rule. If it
|
|
// is the only thing standing in the way, take the placement anyway:
|
|
// a second replica on one node still survives an engine crash.
|
|
res = r.sched.Place(profiles, nil)
|
|
pspan.SetAttributes(attribute.Bool("llmgw.spread_given_up", res.Placement != nil))
|
|
}
|
|
placementAttrs(pspan, res)
|
|
pspan.End()
|
|
|
|
r.mu.Lock()
|
|
r.lastExplain[m.Name] = res
|
|
r.mu.Unlock()
|
|
|
|
if res.Placement == nil {
|
|
// Full, with a model somebody has just asked for and cards held by one
|
|
// nobody has asked for in a while. Stopping one is the whole point of
|
|
// on-demand loading; without it the feature works until the cluster is
|
|
// full, which is the only time it matters.
|
|
//
|
|
// Not retried here. The replica is stopped, its GPUs are released on
|
|
// the pass that reaps it, and the next pass places this model — which
|
|
// keeps the stop-then-start ordering in one place rather than two.
|
|
if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) {
|
|
return makingRoom
|
|
}
|
|
if r.awaitingPreemption(m.Name) || r.preemptFor(ctx, m, avoid) {
|
|
return makingRoom
|
|
}
|
|
r.notePlacementFailure(m.Name, res.Explain())
|
|
return noRoom
|
|
}
|
|
if why := r.overLimit(m, res.Placement); why != "" {
|
|
// Recorded as the placement's answer, so the model reads as not
|
|
// placeable, and says why, wherever that is shown.
|
|
r.mu.Lock()
|
|
r.lastExplain[m.Name] = placement.Result{Rejections: []placement.Rejection{
|
|
{Profile: res.Placement.Profile.Name, Reason: why}}}
|
|
r.mu.Unlock()
|
|
r.notePlacementFailure(m.Name, why)
|
|
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.over_limit", why))
|
|
return overLimit
|
|
}
|
|
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
|
r.log.Error("starting replica failed", "model", m.Name, "err", err)
|
|
trace.SpanFromContext(ctx).SetStatus(codes.Error, err.Error())
|
|
return noRoom
|
|
}
|
|
return placed
|
|
}
|
|
|
|
// scaleOutcome is what one scaleUp did.
|
|
type scaleOutcome int
|
|
|
|
const (
|
|
notTried scaleOutcome = iota // nothing was asked of placement this pass
|
|
placed // a replica was started
|
|
makingRoom // room is being made: a preemption, an idle replica stopping
|
|
noRoom // placement found nowhere, and nothing is making room
|
|
overLimit // there is room, but the owner's quota does not allow it
|
|
)
|
|
|
|
// String names an outcome for a trace.
|
|
func (o scaleOutcome) String() string {
|
|
switch o {
|
|
case placed:
|
|
return "placed"
|
|
case makingRoom:
|
|
return "making room"
|
|
case noRoom:
|
|
return "no room"
|
|
case overLimit:
|
|
return "over its quota"
|
|
default:
|
|
return "not tried"
|
|
}
|
|
}
|
|
|
|
// placementAttrs puts a placement's answer on its span: where it went, or
|
|
// every candidate it turned down and why, which is what `ctl
|
|
// explain-placement` would have printed at that moment.
|
|
func placementAttrs(span trace.Span, res placement.Result) {
|
|
span.SetAttributes(attribute.Int("llmgw.rejections", len(res.Rejections)))
|
|
pl := res.Placement
|
|
if pl == nil {
|
|
explain := res.Explain()
|
|
if len(explain) > 4096 {
|
|
explain = explain[:4096] + "…"
|
|
}
|
|
span.SetAttributes(attribute.String("llmgw.explain", explain))
|
|
span.SetStatus(codes.Error, "fits nowhere")
|
|
return
|
|
}
|
|
nodes := make([]string, 0, len(pl.Nodes))
|
|
for _, a := range pl.Nodes {
|
|
nodes = append(nodes, a.NodeID)
|
|
}
|
|
span.SetAttributes(
|
|
attribute.String("llmgw.profile", pl.Profile.Name),
|
|
attribute.String("llmgw.engine", pl.Profile.Engine),
|
|
attribute.String("llmgw.gpu_class", pl.Profile.GPUClass),
|
|
attribute.String("llmgw.pool", pl.Pool),
|
|
attribute.String("llmgw.fabric", pl.Fabric),
|
|
attribute.StringSlice("llmgw.nodes", nodes),
|
|
)
|
|
}
|
|
|
|
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
|
|
prof, ok := m.ProfileByName(pl.Profile.Name)
|
|
if !ok {
|
|
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
|
|
}
|
|
espec, ok := m.EngineProfile(prof.Name)
|
|
if !ok {
|
|
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.seq++
|
|
id := replicaID(m.Name, r.seq)
|
|
attempt := r.attempts[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
rep := &Replica{
|
|
ID: id, Model: m.Name, Profile: prof.Name, ProfileFP: prof.Fingerprint(),
|
|
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
|
|
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
|
|
WantShards: len(pl.Nodes),
|
|
}
|
|
|
|
// The leader's address and port are fixed before any shard starts: every
|
|
// rank needs the same rendezvous point, and a worker that starts first must
|
|
// already know where to dial.
|
|
leader := pl.Nodes[0]
|
|
leaderNode, ok := r.inv.Get(leader.NodeID)
|
|
if !ok {
|
|
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
|
|
}
|
|
basePort := r.allocatePorts(pl)
|
|
|
|
for _, a := range pl.Nodes {
|
|
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
|
|
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
|
|
HTTPPort: basePort, State: supervisor.Pending,
|
|
})
|
|
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
|
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
|
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), PowerW: a.PowerW,
|
|
})
|
|
}
|
|
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
|
|
|
// Everything read from rep has to be read now: once it is in the map an
|
|
// agent status report can mutate it concurrently, and the race detector is
|
|
// right to object.
|
|
//
|
|
// The leader first, then workers ascending. Both engines put the rendezvous
|
|
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
|
|
// worker started first has nothing to connect to and spends its startup
|
|
// budget retrying.
|
|
//
|
|
// An earlier version started workers first, on the theory that the head
|
|
// should wait for them. That is backwards: vLLM's server blocks until the
|
|
// Ray cluster has the GPUs it needs, so the leader can be started
|
|
// immediately and will wait on its own.
|
|
order := slices.Clone(rep.Shards)
|
|
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
|
|
nodeIDs := rep.NodeIDs()
|
|
|
|
// Open until the replica serves, fails or is stopped, and the parent of
|
|
// every node's part of the start: the agents' spans arrive under it.
|
|
rctx, rspan := startSpan(ctx, "replica start", append(replicaAttrs(rep),
|
|
attribute.String("llmgw.engine", prof.Engine),
|
|
attribute.String("llmgw.gpu_class", prof.GPUClass),
|
|
attribute.String("llmgw.pool", pl.Pool),
|
|
attribute.String("llmgw.fabric", pl.Fabric),
|
|
attribute.String("llmgw.endpoint", rep.Endpoint),
|
|
attribute.Int("llmgw.attempt", attempt),
|
|
attribute.String("llmgw.startup_timeout", r.startupTimeout(rep).String()),
|
|
)...)
|
|
rep.startSpan, rep.startTrace = rspan, rspan.SpanContext()
|
|
|
|
r.mu.Lock()
|
|
r.replicas[id] = rep
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range order {
|
|
cmd := ShardCommand{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
NodeID: s.NodeID,
|
|
Spec: engines.ShardSpec{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
Rank: s.Rank,
|
|
NNodes: len(pl.Nodes),
|
|
LeaderAddr: addrOf(leaderNode),
|
|
LeaderPort: basePort + 1,
|
|
GPUIndices: s.GPUIndices,
|
|
HCAs: s.HCAs,
|
|
MgmtInterface: r.cfg.MgmtInterface,
|
|
HTTPPort: s.HTTPPort,
|
|
Profile: espec,
|
|
APIKey: r.engineKey(id),
|
|
Threads: cpuCores(prof),
|
|
},
|
|
StartupTimeout: prof.StartupTimeout,
|
|
Pool: pl.Pool,
|
|
Fabric: pl.Fabric,
|
|
ProfileFP: prof.Fingerprint(),
|
|
}
|
|
// The command, sent: its context travels with it to the agent.
|
|
sctx, sspan := tracer().Start(rctx, "send start",
|
|
trace.WithSpanKind(trace.SpanKindProducer),
|
|
trace.WithAttributes(
|
|
attribute.String("llmgw.shard", s.ID),
|
|
attribute.String("llmgw.node", s.NodeID),
|
|
attribute.Int("llmgw.rank", s.Rank),
|
|
attribute.IntSlice("llmgw.gpus", s.GPUIndices),
|
|
attribute.Int("llmgw.port", s.HTTPPort),
|
|
))
|
|
err := r.disp.StartShard(sctx, cmd)
|
|
obs.End(sspan, err)
|
|
if err != nil {
|
|
r.markShard(id, s.ID, supervisor.Failed, err.Error())
|
|
return fmt.Errorf("starting shard %s: %w", s.ID, err)
|
|
}
|
|
r.markShard(id, s.ID, supervisor.Starting, "")
|
|
}
|
|
|
|
r.log.Info("replica placed",
|
|
"replica", id, "model", m.Name, "profile", prof.Name,
|
|
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
|
|
return nil
|
|
}
|
|
|
|
// replicaID names a replica uniquely for the life of the cluster, not the life
|
|
// of this process.
|
|
//
|
|
// The counter alone is not enough, and the way it fails is ugly. A control
|
|
// plane that restarts begins counting again, so the replica it places next is
|
|
// called llama-1 — which is also what an engine started before the restart and
|
|
// still running on some other node calls itself. Two nodes then claim the same
|
|
// shard id, adoption hands the id back and forth between them, and every
|
|
// placement lands on a port an orphan is already holding. Observed on a real
|
|
// cluster as a bind-conflict loop that never settled.
|
|
//
|
|
// The random suffix costs nothing and makes the collision impossible.
|
|
func replicaID(model string, seq int) string {
|
|
var buf [4]byte
|
|
if _, err := crypterand.Read(buf[:]); err != nil {
|
|
// Randomness failing is not a reason to stop placing replicas; the
|
|
// counter alone is still unique within this process.
|
|
return fmt.Sprintf("%s-%d", model, seq)
|
|
}
|
|
return fmt.Sprintf("%s-%x", model, buf)
|
|
}
|
|
|
|
// allocatePorts picks a port shared by every shard of a replica.
|
|
//
|
|
// The same port on every node keeps the leader address predictable and the
|
|
// port bookkeeping trivial; nodes do not share a network namespace, so there is
|
|
// no conflict in reusing it.
|
|
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
|
|
used := map[int]struct{}{}
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
for _, s := range rep.Shards {
|
|
used[s.HTTPPort] = struct{}{}
|
|
}
|
|
}
|
|
// Ports an agent has reported but nothing owns — a stray mid-shutdown, an
|
|
// engine whose replica was just reaped. The process is still listening
|
|
// whatever the bookkeeping says, and handing its port to a new shard means
|
|
// that shard fails to bind and takes the whole replica with it.
|
|
for p := range r.reservedPorts {
|
|
used[p] = struct{}{}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
// Two ports per replica: HTTP and the distributed init rendezvous.
|
|
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
|
|
if _, taken := used[p]; !taken {
|
|
return p
|
|
}
|
|
}
|
|
return r.cfg.PortRange[0]
|
|
}
|
|
|
|
// scaleDown drains replicas; why is for the trace.
|
|
func (r *Reconciler) scaleDown(ctx context.Context, excess []string, why string) {
|
|
for _, id := range excess {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
shards := slices.Clone(rep.Shards)
|
|
r.mu.Unlock()
|
|
|
|
r.log.Info("draining replica", "replica", id)
|
|
r.drainShards(ctx, id, shards, why)
|
|
}
|
|
}
|
|
|
|
// stopReplica stops a replica's shards and forgets it; why is for the trace.
|
|
func (r *Reconciler) stopReplica(ctx context.Context, id, why string) {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
shards := slices.Clone(rep.Shards)
|
|
rep.State = Stopped
|
|
rep.startDone(r.now())
|
|
ctx, span := startSpan(ctx, "stop replica", append(replicaAttrs(rep),
|
|
attribute.String("llmgw.reason", why))...)
|
|
defer span.End()
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(id)
|
|
|
|
r.mu.Lock()
|
|
delete(r.replicas, id)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// OnFetchProgress records how far a shard's download has got.
|
|
//
|
|
// Separate from OnShardStatus because it is not a transition: the shard is
|
|
// still starting, and treating each progress report as a state change would
|
|
// reset the timers that decide when a slow start becomes a failed one.
|
|
func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, total int64) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID != shardID {
|
|
continue
|
|
}
|
|
rep.Shards[i].Fetch = &FetchProgress{
|
|
Source: source, BytesDone: done, BytesTotal: total,
|
|
}
|
|
}
|
|
}
|
|
|
|
// OnShardCompute records where a shard's engine said it put the model.
|
|
func (r *Reconciler) OnShardCompute(replicaID, shardID string, c Compute) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID == shardID {
|
|
cc := c
|
|
rep.Shards[i].Compute = &cc
|
|
}
|
|
}
|
|
}
|
|
|
|
// OnShardStatus records a status report from an agent.
|
|
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.markShard(replicaID, shardID, st, errMsg)
|
|
r.Notify()
|
|
}
|
|
|
|
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID != shardID {
|
|
continue
|
|
}
|
|
rep.Shards[i].State = st
|
|
if errMsg != "" {
|
|
rep.Shards[i].Err = errMsg
|
|
}
|
|
// A real transition means the download is over, one way or the other.
|
|
// Left in place it would sit in the listing at ninety-nine percent
|
|
// under a shard that has been serving for an hour.
|
|
rep.Shards[i].Fetch = nil
|
|
break
|
|
}
|
|
|
|
prev := rep.State
|
|
rep.deriveState(r.now())
|
|
if rep.startSpan != nil {
|
|
ev := []attribute.KeyValue{
|
|
attribute.String("llmgw.shard", shardID), attribute.String("llmgw.state", st.String()),
|
|
}
|
|
if errMsg != "" {
|
|
ev = append(ev, attribute.String("llmgw.error", errMsg))
|
|
}
|
|
rep.startSpan.AddEvent("shard "+st.String(), trace.WithAttributes(ev...))
|
|
}
|
|
if rep.State == Ready || rep.State.Terminal() {
|
|
rep.startDone(r.now())
|
|
}
|
|
if rep.State == Ready && prev != Ready {
|
|
// A model that comes up clears its backoff; the next failure starts
|
|
// counting from scratch rather than from an old streak.
|
|
r.attempts[rep.Model] = 0
|
|
delete(r.nextAttempt, rep.Model)
|
|
}
|
|
}
|
|
|
|
// Replicas returns a snapshot of every replica, sorted by id.
|
|
func (r *Reconciler) Replicas() []Replica {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
out := make([]Replica, 0, len(r.replicas))
|
|
for _, rep := range r.replicas {
|
|
clone := *rep
|
|
clone.Shards = slices.Clone(rep.Shards)
|
|
out = append(out, clone)
|
|
}
|
|
slices.SortFunc(out, func(a, b Replica) int {
|
|
if a.ID < b.ID {
|
|
return -1
|
|
}
|
|
if a.ID > b.ID {
|
|
return 1
|
|
}
|
|
return 0
|
|
})
|
|
return out
|
|
}
|
|
|
|
// ReadyReplicas returns the replicas the router may send traffic to.
|
|
func (r *Reconciler) ReadyReplicas() []Replica {
|
|
var out []Replica
|
|
for _, rep := range r.Replicas() {
|
|
if rep.State == Ready {
|
|
out = append(out, rep)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Explain returns the most recent placement result for a model, which is what
|
|
// `ctl explain-placement` prints.
|
|
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
res, ok := r.lastExplain[model]
|
|
return res, ok
|
|
}
|
|
|
|
func addrOf(n inventory.Node) string {
|
|
if n.MgmtAddr != "" {
|
|
return n.MgmtAddr
|
|
}
|
|
if n.Hostname != "" {
|
|
return n.Hostname
|
|
}
|
|
return n.ID
|
|
}
|
|
|
|
// profileMemory is what a model's profile declared it needs, or zero.
|
|
//
|
|
// Looked up when a replica is adopted after a restart, because the allocation
|
|
// has to carry the same claim it did before — a control plane that forgot every
|
|
// claim on restart would place a second large model onto a node already full of
|
|
// the first.
|
|
func (r *Reconciler) profileMemory(model, profile string) int {
|
|
p, _ := r.profileOf(model, profile)
|
|
return p.MinMemoryGB
|
|
}
|
|
|
|
// profileWatts is the draw a profile declared for each of its shards, so an
|
|
// adopted replica counts against its racks' power budgets as a placed one
|
|
// does; zero when it declared none.
|
|
func (r *Reconciler) profileWatts(model, profile string) float64 {
|
|
p, _ := r.profileOf(model, profile)
|
|
if p.PowerWatts <= 0 {
|
|
return 0
|
|
}
|
|
return p.PowerWatts / float64(max(p.Nodes, 1))
|
|
}
|
|
|
|
// profileOf is a replica's profile as the catalog holds it now.
|
|
func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return catalog.Profile{}, false
|
|
}
|
|
for _, p := range m.Profiles {
|
|
if p.Name == profile {
|
|
return p, true
|
|
}
|
|
}
|
|
return catalog.Profile{}, false
|
|
}
|
|
|
|
// cpuCores is what a profile reserves of a node's CPUs: its cores when it
|
|
// computes on them, nothing when a GPU does the work.
|
|
func cpuCores(p catalog.Profile) int {
|
|
if !p.CPUOnly() {
|
|
return 0
|
|
}
|
|
return p.CPUCores
|
|
}
|
|
|
|
func (r *Reconciler) engineKey(replicaID string) string {
|
|
if r.cfg.EngineKey == nil {
|
|
return ""
|
|
}
|
|
return r.cfg.EngineKey(replicaID)
|
|
}
|
|
|
|
// SetNotifier gives the reconciler somewhere to say a model was parked or a
|
|
// node lost.
|
|
func (r *Reconciler) SetNotifier(n *notify.Notifier) {
|
|
r.notify.Store(n)
|
|
}
|