The previous commit's adoption fix made a deeper problem visible instead of
fixing it. Replica ids were the model name and a counter, and the counter
restarted with the process — so a control plane that came back placed a
replica called llama-1 while an engine started before the restart, still
running on a different node, also called itself llama-1.
Two nodes then claimed the same shard id. Adoption handed it back and forth
between them, every placement landed on a port an orphan was still bound to,
and the engine failed to bind, which failed the replica, which placed another
one on the same port. On the real cluster that loop ran for minutes:
couldn't bind HTTP server socket, hostname: 0.0.0.0, port: 20002
Ids now carry four random bytes, which costs nothing and makes the collision
impossible. The "already known" branch in adoption goes back to meaning what
it says: an agent reconnecting, not a name clash across restarts.
Ports a stray still holds are also reserved now. Stopping is not instant and
an engine keeps its socket until it exits, so the allocator has to know about
sockets no replica claims — otherwise the same loop reappears in the window
between telling an agent to stop something and it actually stopping.
Two of the tests written for the previous commit had quietly become vacuous
under the new ids, asserting things about a literal llama-1 that no longer
exists. They capture the id now.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
910 lines
28 KiB
Go
910 lines
28 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
crypterand "crypto/rand"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
)
|
|
|
|
// ShardCommand is everything an agent needs to launch one shard.
|
|
type ShardCommand struct {
|
|
ReplicaID string
|
|
ShardID string
|
|
NodeID string
|
|
Spec engines.ShardSpec
|
|
// StartupTimeout comes from the profile; model loading dominates it.
|
|
StartupTimeout time.Duration
|
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
|
// travel with the command so the agent can report them back after a
|
|
// control plane restart (plan §6.5).
|
|
Pool string
|
|
Fabric string
|
|
}
|
|
|
|
// Dispatcher sends commands to agents.
|
|
//
|
|
// It is an interface so the reconciliation loop can be tested without a network:
|
|
// the loop's logic is where the interesting failures are, not the transport.
|
|
type Dispatcher interface {
|
|
StartShard(ctx context.Context, cmd ShardCommand) error
|
|
StopShard(ctx context.Context, nodeID, shardID string) error
|
|
DrainShard(ctx context.Context, nodeID, shardID string) error
|
|
}
|
|
|
|
// Config tunes the reconciliation loop.
|
|
type Config struct {
|
|
// Interval is how often desired and actual are compared when nothing else
|
|
// happens. Events wake the loop sooner.
|
|
Interval time.Duration
|
|
// BaseBackoff is how long to wait before rebuilding a failed replica.
|
|
BaseBackoff time.Duration
|
|
// MaxBackoff caps the exponential growth.
|
|
MaxBackoff time.Duration
|
|
// UnreachableAfter is how long without a heartbeat before a node is
|
|
// unreachable.
|
|
UnreachableAfter time.Duration
|
|
// LostAfter is how long unreachable before its replicas are declared dead.
|
|
// Longer than UnreachableAfter on purpose: a brief network blip must not
|
|
// tear down a replica that is still serving perfectly well.
|
|
LostAfter time.Duration
|
|
// DrainGrace bounds how long a draining replica waits for its replacement.
|
|
// Without a bound, draining a node whose model cannot be placed anywhere
|
|
// else would never finish.
|
|
DrainGrace time.Duration
|
|
// PortRange is the per-node port range for engine HTTP and dist-init.
|
|
PortRange [2]int
|
|
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
|
MgmtInterface string
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
DrainGrace: 5 * time.Minute,
|
|
PortRange: [2]int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
d := DefaultConfig()
|
|
if c.Interval <= 0 {
|
|
c.Interval = d.Interval
|
|
}
|
|
if c.BaseBackoff <= 0 {
|
|
c.BaseBackoff = d.BaseBackoff
|
|
}
|
|
if c.MaxBackoff < c.BaseBackoff {
|
|
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
|
|
}
|
|
if c.UnreachableAfter <= 0 {
|
|
c.UnreachableAfter = d.UnreachableAfter
|
|
}
|
|
if c.LostAfter < c.UnreachableAfter {
|
|
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
|
|
}
|
|
if c.DrainGrace <= 0 {
|
|
c.DrainGrace = d.DrainGrace
|
|
}
|
|
if c.PortRange[1] <= c.PortRange[0] {
|
|
c.PortRange = d.PortRange
|
|
}
|
|
if c.MgmtInterface == "" {
|
|
c.MgmtInterface = d.MgmtInterface
|
|
}
|
|
}
|
|
|
|
// Reconciler compares the catalog against what is running and closes the gap.
|
|
type Reconciler struct {
|
|
cfg Config
|
|
cat *catalog.Catalog
|
|
inv *inventory.Store
|
|
sched *placement.Scheduler
|
|
disp Dispatcher
|
|
log *slog.Logger
|
|
now func() time.Time
|
|
|
|
mu sync.Mutex
|
|
replicas map[string]*Replica
|
|
// nextAttempt is the earliest a model may be rebuilt, per model.
|
|
nextAttempt map[string]time.Time
|
|
// attempts counts consecutive failures per model, for the backoff.
|
|
attempts map[string]int
|
|
// lastExplain keeps the most recent placement failure per model, so
|
|
// `ctl explain-placement` can answer without re-running placement.
|
|
lastExplain map[string]placement.Result
|
|
seq int
|
|
// reservedPorts holds ports the fleet reports as in use but no replica
|
|
// claims, so the allocator cannot hand out what something is still bound to.
|
|
reservedPorts map[int]struct{}
|
|
|
|
wake chan struct{}
|
|
stop chan struct{}
|
|
done chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
// New builds a reconciler.
|
|
func New(cfg Config, cat *catalog.Catalog, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Reconciler{
|
|
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
|
|
sched: placement.New(inv),
|
|
now: time.Now,
|
|
replicas: map[string]*Replica{},
|
|
nextAttempt: map[string]time.Time{},
|
|
attempts: map[string]int{},
|
|
reservedPorts: map[int]struct{}{},
|
|
lastExplain: map[string]placement.Result{},
|
|
wake: make(chan struct{}, 1),
|
|
stop: make(chan struct{}),
|
|
done: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
// SetClock replaces the time source, for tests.
|
|
func (r *Reconciler) SetClock(now func() time.Time) { r.now = now }
|
|
|
|
// Start runs the loop until Close.
|
|
func (r *Reconciler) Start(ctx context.Context) {
|
|
go func() {
|
|
defer close(r.done)
|
|
t := time.NewTicker(r.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
r.Reconcile(ctx)
|
|
for {
|
|
select {
|
|
case <-r.stop:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-r.wake:
|
|
r.Reconcile(ctx)
|
|
case <-t.C:
|
|
r.Reconcile(ctx)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Close stops the loop.
|
|
func (r *Reconciler) Close() {
|
|
r.once.Do(func() {
|
|
close(r.stop)
|
|
<-r.done
|
|
})
|
|
}
|
|
|
|
// Notify wakes the loop, for when something changed and waiting for the next
|
|
// tick would add pointless latency.
|
|
func (r *Reconciler) Notify() {
|
|
select {
|
|
case r.wake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Reconcile performs one pass. It is exported so tests can drive it directly
|
|
// instead of racing a ticker.
|
|
func (r *Reconciler) Reconcile(ctx context.Context) {
|
|
r.expireNodes()
|
|
r.expireStuckReplicas()
|
|
r.drainCordonedNodes(ctx)
|
|
r.reapFailed(ctx)
|
|
|
|
for _, m := range r.cat.Models() {
|
|
r.reconcileModel(ctx, m)
|
|
}
|
|
}
|
|
|
|
// expireStuckReplicas fails replicas that never finished coming up.
|
|
//
|
|
// Every other path out of Starting depends on the agent reporting something. A
|
|
// replica whose agent never reports — the start was a duplicate shard id and
|
|
// became a no-op, the node went quiet in a way expiry did not catch, the engine
|
|
// wedged before its first health check — sits in Starting for as long as the
|
|
// process lives, holding its GPUs and counting towards the model's replica
|
|
// count while serving nothing.
|
|
//
|
|
// Found on a real cluster: two replicas sat in Starting for eight minutes past
|
|
// a two-minute startup timeout, and nothing would ever have moved them.
|
|
func (r *Reconciler) expireStuckReplicas() {
|
|
now := r.now()
|
|
var stuck []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
|
|
stuck = append(stuck, rep)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range stuck {
|
|
r.log.Warn("replica did not come up in time; failing it so it can be replaced",
|
|
"replica", rep.ID, "model", rep.Model, "state", "starting",
|
|
"timeout", r.startupTimeout(rep))
|
|
}
|
|
}
|
|
|
|
// startupTimeout is the profile's, or the configured default when the profile
|
|
// does not say — a replica with no deadline at all is the thing this exists to
|
|
// prevent.
|
|
func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
|
|
if m, ok := r.cat.Resolve(rep.Model); ok {
|
|
if p, ok := m.ProfileByName(rep.Profile); ok && p.StartupTimeout > 0 {
|
|
return p.StartupTimeout
|
|
}
|
|
}
|
|
// No profile deadline: a generous default, because the thing being prevented
|
|
// is a replica with no deadline at all rather than one with a loose one.
|
|
// Loading a large model across nodes genuinely takes tens of minutes.
|
|
return 30 * time.Minute
|
|
}
|
|
|
|
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
|
|
// replicas of nodes that are gone.
|
|
//
|
|
// The two-stage delay is deliberate: a brief network blip must not tear down
|
|
// replicas that are still serving traffic perfectly well, and rebuilding a
|
|
// multi-node replica costs tens of minutes of model loading.
|
|
func (r *Reconciler) expireNodes() {
|
|
now := r.now()
|
|
for _, n := range r.inv.Nodes() {
|
|
silent := now.Sub(n.LastSeen)
|
|
switch {
|
|
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
|
|
r.inv.SetState(n.ID, inventory.Unreachable)
|
|
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
|
|
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
|
|
r.inv.SetState(n.ID, inventory.Lost)
|
|
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
|
"node", n.ID, "silent_for", silent.Round(time.Second))
|
|
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
|
}
|
|
}
|
|
}
|
|
|
|
// drainCordonedNodes moves work off nodes an operator is taking out of service.
|
|
//
|
|
// Draining a node is not the same as losing one: the engines are healthy and
|
|
// still serving, so their replicas are marked Draining rather than Failed. That
|
|
// makes the model look short of replicas, a replacement is placed elsewhere,
|
|
// and the original is stopped only once the replacement is serving — which is
|
|
// the difference between a rolling move and an outage.
|
|
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
|
|
var toDrain []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
for _, s := range rep.Shards {
|
|
n, ok := r.inv.Get(s.NodeID)
|
|
if ok && n.State == inventory.Draining {
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
toDrain = append(toDrain, rep)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
shards := map[string][]Shard{}
|
|
for _, rep := range toDrain {
|
|
shards[rep.ID] = slices.Clone(rep.Shards)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range toDrain {
|
|
r.log.Info("draining replica off a node being taken out of service",
|
|
"replica", rep.ID, "nodes", rep.NodeIDs())
|
|
for _, s := range shards[rep.ID] {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() || !rep.OnNode(nodeID) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = reason
|
|
}
|
|
}
|
|
|
|
// reapFailed releases the resources of terminal replicas and tells the agents
|
|
// to clean up whatever might still be running.
|
|
func (r *Reconciler) reapFailed(ctx context.Context) {
|
|
r.mu.Lock()
|
|
var doomed []*Replica
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() {
|
|
doomed = append(doomed, rep)
|
|
}
|
|
}
|
|
for _, rep := range doomed {
|
|
delete(r.replicas, rep.ID)
|
|
if rep.State == Failed {
|
|
// Carry the failure count on the model, not the replica: the
|
|
// replica is about to disappear, and the backoff has to survive it.
|
|
r.attempts[rep.Model]++
|
|
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range doomed {
|
|
// rep is out of the map now, so no agent report can reach it any more.
|
|
if rep.State == Failed {
|
|
r.mu.Lock()
|
|
retry := r.backoff(r.attempts[rep.Model])
|
|
r.mu.Unlock()
|
|
r.log.Error("replica failed",
|
|
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
|
|
"retry_in", retry.Round(time.Second))
|
|
}
|
|
// Stop every shard, including the ones that looked healthy: a partially
|
|
// alive replica holds GPUs while serving nothing.
|
|
for _, s := range rep.Shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard of a dead replica failed",
|
|
"replica", rep.ID, "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(rep.ID)
|
|
}
|
|
}
|
|
|
|
// backoff grows with consecutive failures and is jittered, so a cluster-wide
|
|
// problem does not produce synchronized retry storms across every model.
|
|
func (r *Reconciler) backoff(attempt int) time.Duration {
|
|
if attempt <= 0 {
|
|
return 0
|
|
}
|
|
d := r.cfg.BaseBackoff << min(attempt-1, 16)
|
|
if d > r.cfg.MaxBackoff || d <= 0 {
|
|
d = r.cfg.MaxBackoff
|
|
}
|
|
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
|
|
return d - d/8 + jitter
|
|
}
|
|
|
|
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
|
// Snapshot under the lock and work from values afterwards. A replica
|
|
// pointer stops being safe the moment it is in the map: an agent's status
|
|
// report can mutate it from another goroutine at any time.
|
|
r.mu.Lock()
|
|
var liveIDs, drainingIDs, avoid []string
|
|
for _, rep := range r.replicas {
|
|
if rep.Model != m.Name {
|
|
continue
|
|
}
|
|
switch {
|
|
case rep.State.Counts():
|
|
liveIDs = append(liveIDs, rep.ID)
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
case rep.State == Draining:
|
|
drainingIDs = append(drainingIDs, rep.ID)
|
|
}
|
|
}
|
|
blockedUntil := r.nextAttempt[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
desired := m.Scaling.Min
|
|
switch {
|
|
case len(liveIDs) < desired:
|
|
if r.now().Before(blockedUntil) {
|
|
// Still in backoff. Rebuilding immediately would hot-loop on a
|
|
// model that cannot start — a missing weights path, say — and bury
|
|
// the real error in noise.
|
|
return
|
|
}
|
|
r.scaleUp(ctx, m, avoid)
|
|
case len(liveIDs) > desired:
|
|
r.scaleDown(ctx, liveIDs[desired:])
|
|
}
|
|
|
|
// A drained replica is stopped once its replacement is serving, or once the
|
|
// grace period expires. Stopping it the moment it was drained would cut
|
|
// in-flight requests, which is the thing draining exists to avoid.
|
|
replaced := len(liveIDs) >= desired
|
|
for _, id := range drainingIDs {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
var expired bool
|
|
if ok {
|
|
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > r.cfg.DrainGrace
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
switch {
|
|
case !ok:
|
|
case replaced:
|
|
r.stopReplica(ctx, id)
|
|
case expired:
|
|
// The replacement never arrived — often because nowhere else can
|
|
// host it. Draining forever would leave the node undrainable.
|
|
r.log.Warn("stopping a drained replica without a replacement",
|
|
"replica", id, "after", r.cfg.DrainGrace)
|
|
r.stopReplica(ctx, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// scaleUp places and starts one replica per pass.
|
|
//
|
|
// One at a time on purpose: placement decisions depend on what is already
|
|
// allocated, and starting several at once from the same inventory snapshot
|
|
// would hand them the same GPUs.
|
|
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
|
|
res := r.sched.Place(m.PlacementProfiles(), avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
// Spreading across failure domains is a preference, not a rule. If it
|
|
// is the only thing standing in the way, take the placement anyway:
|
|
// a second replica on one node still survives an engine crash.
|
|
res = r.sched.Place(m.PlacementProfiles(), nil)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.lastExplain[m.Name] = res
|
|
r.mu.Unlock()
|
|
|
|
if res.Placement == nil {
|
|
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
|
|
return
|
|
}
|
|
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
|
r.log.Error("starting replica failed", "model", m.Name, "err", err)
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
|
|
prof, ok := m.ProfileByName(pl.Profile.Name)
|
|
if !ok {
|
|
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
|
|
}
|
|
espec, ok := m.EngineProfile(prof.Name)
|
|
if !ok {
|
|
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.seq++
|
|
id := replicaID(m.Name, r.seq)
|
|
attempt := r.attempts[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
rep := &Replica{
|
|
ID: id, Model: m.Name, Profile: prof.Name,
|
|
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
|
|
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
|
|
}
|
|
|
|
// The leader's address and port are fixed before any shard starts: every
|
|
// rank needs the same rendezvous point, and a worker that starts first must
|
|
// already know where to dial.
|
|
leader := pl.Nodes[0]
|
|
leaderNode, ok := r.inv.Get(leader.NodeID)
|
|
if !ok {
|
|
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
|
|
}
|
|
basePort := r.allocatePorts(pl)
|
|
|
|
for _, a := range pl.Nodes {
|
|
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
|
|
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
|
|
HTTPPort: basePort, State: supervisor.Pending,
|
|
})
|
|
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
|
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
|
})
|
|
}
|
|
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
|
|
|
// Everything read from rep has to be read now: once it is in the map an
|
|
// agent status report can mutate it concurrently, and the race detector is
|
|
// right to object.
|
|
//
|
|
// The leader first, then workers ascending. Both engines put the rendezvous
|
|
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
|
|
// worker started first has nothing to connect to and spends its startup
|
|
// budget retrying.
|
|
//
|
|
// An earlier version started workers first, on the theory that the head
|
|
// should wait for them. That is backwards: vLLM's server blocks until the
|
|
// Ray cluster has the GPUs it needs, so the leader can be started
|
|
// immediately and will wait on its own.
|
|
order := slices.Clone(rep.Shards)
|
|
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
|
|
nodeIDs := rep.NodeIDs()
|
|
|
|
r.mu.Lock()
|
|
r.replicas[id] = rep
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range order {
|
|
cmd := ShardCommand{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
NodeID: s.NodeID,
|
|
Spec: engines.ShardSpec{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
Rank: s.Rank,
|
|
NNodes: len(pl.Nodes),
|
|
LeaderAddr: addrOf(leaderNode),
|
|
LeaderPort: basePort + 1,
|
|
GPUIndices: s.GPUIndices,
|
|
HCAs: s.HCAs,
|
|
MgmtInterface: r.cfg.MgmtInterface,
|
|
HTTPPort: s.HTTPPort,
|
|
Profile: espec,
|
|
},
|
|
StartupTimeout: prof.StartupTimeout,
|
|
Pool: pl.Pool,
|
|
Fabric: pl.Fabric,
|
|
}
|
|
if err := r.disp.StartShard(ctx, cmd); err != nil {
|
|
r.markShard(id, s.ID, supervisor.Failed, err.Error())
|
|
return fmt.Errorf("starting shard %s: %w", s.ID, err)
|
|
}
|
|
r.markShard(id, s.ID, supervisor.Starting, "")
|
|
}
|
|
|
|
r.log.Info("replica placed",
|
|
"replica", id, "model", m.Name, "profile", prof.Name,
|
|
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
|
|
return nil
|
|
}
|
|
|
|
// replicaID names a replica uniquely for the life of the cluster, not the life
|
|
// of this process.
|
|
//
|
|
// The counter alone is not enough, and the way it fails is ugly. A control
|
|
// plane that restarts begins counting again, so the replica it places next is
|
|
// called llama-1 — which is also what an engine started before the restart and
|
|
// still running on some other node calls itself. Two nodes then claim the same
|
|
// shard id, adoption hands the id back and forth between them, and every
|
|
// placement lands on a port an orphan is already holding. Observed on a real
|
|
// cluster as a bind-conflict loop that never settled.
|
|
//
|
|
// The random suffix costs nothing and makes the collision impossible.
|
|
func replicaID(model string, seq int) string {
|
|
var buf [4]byte
|
|
if _, err := crypterand.Read(buf[:]); err != nil {
|
|
// Randomness failing is not a reason to stop placing replicas; the
|
|
// counter alone is still unique within this process.
|
|
return fmt.Sprintf("%s-%d", model, seq)
|
|
}
|
|
return fmt.Sprintf("%s-%x", model, buf)
|
|
}
|
|
|
|
// allocatePorts picks a port shared by every shard of a replica.
|
|
//
|
|
// The same port on every node keeps the leader address predictable and the
|
|
// port bookkeeping trivial; nodes do not share a network namespace, so there is
|
|
// no conflict in reusing it.
|
|
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
|
|
used := map[int]struct{}{}
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
for _, s := range rep.Shards {
|
|
used[s.HTTPPort] = struct{}{}
|
|
}
|
|
}
|
|
// Ports an agent has reported but nothing owns — a stray mid-shutdown, an
|
|
// engine whose replica was just reaped. The process is still listening
|
|
// whatever the bookkeeping says, and handing its port to a new shard means
|
|
// that shard fails to bind and takes the whole replica with it.
|
|
for p := range r.reservedPorts {
|
|
used[p] = struct{}{}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
// Two ports per replica: HTTP and the distributed init rendezvous.
|
|
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
|
|
if _, taken := used[p]; !taken {
|
|
return p
|
|
}
|
|
}
|
|
return r.cfg.PortRange[0]
|
|
}
|
|
|
|
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
|
for _, id := range excess {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
shards := slices.Clone(rep.Shards)
|
|
r.mu.Unlock()
|
|
|
|
r.log.Info("draining replica", "replica", id)
|
|
for _, s := range shards {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
shards := slices.Clone(rep.Shards)
|
|
rep.State = Stopped
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(id)
|
|
|
|
r.mu.Lock()
|
|
delete(r.replicas, id)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// Adopt rebuilds replica state from shards an agent reports it is already
|
|
// running (plan §6.5).
|
|
//
|
|
// This is what makes a control plane restart survivable. Without it the
|
|
// reconciler sees an empty cluster, places duplicates, and the engines already
|
|
// running keep holding GPUs the inventory believes are free — so the duplicates
|
|
// fail to allocate, with an error naming the new process rather than the old.
|
|
//
|
|
// Shards whose model or profile is no longer in the catalog are returned for
|
|
// the caller to stop: nothing will ever claim them.
|
|
func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
|
byReplica := map[string][]AdoptedShard{}
|
|
for _, s := range shards {
|
|
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
for replicaID, group := range byReplica {
|
|
if existing, known := r.replicas[replicaID]; known {
|
|
// Usually this is an agent reconnecting rather than a control plane
|
|
// restart, and its status reports keep the state current.
|
|
//
|
|
// Not always. Replica ids are derived from the model name and a
|
|
// counter, so a control plane that restarts and places before the
|
|
// agents reconnect produces a *different* replica under the same
|
|
// id, on a freshly allocated port. Skipping the adoption then
|
|
// leaves the cluster pointing at a port nothing listens on, while
|
|
// the engine that is actually serving is treated as a stray.
|
|
//
|
|
// The fleet is authoritative about what is running (plan §6.5), so
|
|
// what we merely placed loses to what an agent reports — unless our
|
|
// copy has a shard that has actually come up, in which case two
|
|
// things really are running and the one we did not ask for is the
|
|
// stray.
|
|
if existing.hasLiveShard() {
|
|
unclaimed = append(unclaimed, group...)
|
|
r.reservePorts(group)
|
|
continue
|
|
}
|
|
r.log.Warn("a running replica reported by the fleet replaces one we placed",
|
|
"replica", replicaID, "placed_endpoint", existing.Endpoint)
|
|
for _, s := range existing.Shards {
|
|
r.inv.Release(s.ID)
|
|
}
|
|
delete(r.replicas, replicaID)
|
|
}
|
|
|
|
first := group[0]
|
|
m, ok := r.cat.Resolve(first.Model)
|
|
if !ok {
|
|
unclaimed = append(unclaimed, group...)
|
|
r.reservePorts(group)
|
|
continue
|
|
}
|
|
prof, ok := m.ProfileByName(first.Profile)
|
|
if !ok {
|
|
unclaimed = append(unclaimed, group...)
|
|
r.reservePorts(group)
|
|
continue
|
|
}
|
|
|
|
rep := &Replica{
|
|
ID: replicaID, Model: m.Name, Profile: prof.Name,
|
|
Fabric: first.Fabric, Pool: first.Pool,
|
|
Weight: prof.Weight, CreatedAt: r.now(),
|
|
Endpoint: first.Endpoint,
|
|
}
|
|
for _, s := range group {
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
|
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
|
HTTPPort: s.HTTPPort, State: s.State,
|
|
})
|
|
// Reclaim the GPUs, or the next placement will hand them out twice.
|
|
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
|
ShardID: s.ShardID, ReplicaID: replicaID, GPUIndices: s.GPUIndices,
|
|
})
|
|
}
|
|
// The endpoint comes from the leader; a group missing rank 0 is a
|
|
// partial replica and will be failed by deriveState below.
|
|
for _, s := range group {
|
|
if s.Rank == 0 && s.Endpoint != "" {
|
|
rep.Endpoint = s.Endpoint
|
|
}
|
|
}
|
|
rep.deriveState()
|
|
r.replicas[replicaID] = rep
|
|
|
|
r.log.Info("adopted replica from the fleet",
|
|
"replica", replicaID, "model", m.Name, "shards", len(group), "state", rep.State)
|
|
}
|
|
return unclaimed
|
|
}
|
|
|
|
// reservePorts records ports held by shards nothing owns.
|
|
//
|
|
// They are being stopped, but stopping is not instant and the engine keeps its
|
|
// socket until it exits. Handing the port to a new shard in the meantime makes
|
|
// that shard fail to bind, which fails its replica, which places another one on
|
|
// the same port — a loop that does not end on its own.
|
|
//
|
|
// Called with the lock held.
|
|
func (r *Reconciler) reservePorts(shards []AdoptedShard) {
|
|
for _, s := range shards {
|
|
if s.HTTPPort > 0 {
|
|
r.reservedPorts[s.HTTPPort] = struct{}{}
|
|
}
|
|
}
|
|
}
|
|
|
|
// AdoptedShard is one running shard an agent reported at registration.
|
|
type AdoptedShard struct {
|
|
NodeID string
|
|
ReplicaID string
|
|
ShardID string
|
|
Rank int
|
|
GPUIndices []int
|
|
HCAs []string
|
|
HTTPPort int
|
|
State supervisor.State
|
|
Model string
|
|
Profile string
|
|
Pool string
|
|
Fabric string
|
|
// Endpoint is the leader's base URL, set on the rank 0 shard.
|
|
Endpoint string
|
|
}
|
|
|
|
// OnShardStatus records a status report from an agent.
|
|
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.markShard(replicaID, shardID, st, errMsg)
|
|
r.Notify()
|
|
}
|
|
|
|
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID != shardID {
|
|
continue
|
|
}
|
|
rep.Shards[i].State = st
|
|
if errMsg != "" {
|
|
rep.Shards[i].Err = errMsg
|
|
}
|
|
break
|
|
}
|
|
|
|
prev := rep.State
|
|
rep.deriveState()
|
|
if rep.State == Ready && prev != Ready {
|
|
// A model that comes up clears its backoff; the next failure starts
|
|
// counting from scratch rather than from an old streak.
|
|
r.attempts[rep.Model] = 0
|
|
delete(r.nextAttempt, rep.Model)
|
|
}
|
|
}
|
|
|
|
// Replicas returns a snapshot of every replica, sorted by id.
|
|
func (r *Reconciler) Replicas() []Replica {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
out := make([]Replica, 0, len(r.replicas))
|
|
for _, rep := range r.replicas {
|
|
clone := *rep
|
|
clone.Shards = slices.Clone(rep.Shards)
|
|
out = append(out, clone)
|
|
}
|
|
slices.SortFunc(out, func(a, b Replica) int {
|
|
if a.ID < b.ID {
|
|
return -1
|
|
}
|
|
if a.ID > b.ID {
|
|
return 1
|
|
}
|
|
return 0
|
|
})
|
|
return out
|
|
}
|
|
|
|
// ReadyReplicas returns the replicas the router may send traffic to.
|
|
func (r *Reconciler) ReadyReplicas() []Replica {
|
|
var out []Replica
|
|
for _, rep := range r.Replicas() {
|
|
if rep.State == Ready {
|
|
out = append(out, rep)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Explain returns the most recent placement result for a model, which is what
|
|
// `ctl explain-placement` prints.
|
|
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
res, ok := r.lastExplain[model]
|
|
return res, ok
|
|
}
|
|
|
|
func addrOf(n inventory.Node) string {
|
|
if n.MgmtAddr != "" {
|
|
return n.MgmtAddr
|
|
}
|
|
if n.Hostname != "" {
|
|
return n.Hostname
|
|
}
|
|
return n.ID
|
|
}
|