ctl talks to a management API on the admin listener, not the client-facing one. The two have different audiences: one serves inference to whoever holds an API key, the other lets a caller cordon nodes and read the cluster's shape. - nodes surfaces unhealthy GPUs and why they are excluded. That is usually the answer when a node looks idle but nothing will schedule onto it, and it was invisible before. - explain-placement prints every candidate considered and why it was rejected. - replicas shows the shards a replica is made of, with per-shard errors. - cordon, uncordon and drain are one endpoint with different targets, because the difference between cordon and drain is exactly what an operator needs to be sure of before running either. Draining had to be fixed first to make ctl drain worth having. A drained replica was stopped on the next pass, which cuts in-flight requests — the thing draining exists to avoid. Now a node marked Draining has its replicas drained, the model falls below its desired count, a replacement is placed elsewhere, and the original is stopped only once the replacement is serving. A grace period bounds it, because draining the only node that can host a model would otherwise never finish and the node could never be taken out of service. Verified against a two-node cluster of real binaries: draining moved a replica to the surviving node and traffic kept returning 200 throughout. ctl logs is deliberately absent. It needs a request/response pair the agent protocol does not have, and the failure path already ships an engine log tail with every status report — so the case that matters most is covered, and reading a healthy shard's output on demand is not worth a protocol change yet. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
783 lines
22 KiB
Go
783 lines
22 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
)
|
|
|
|
// ShardCommand is everything an agent needs to launch one shard.
|
|
type ShardCommand struct {
|
|
ReplicaID string
|
|
ShardID string
|
|
NodeID string
|
|
Spec engines.ShardSpec
|
|
// StartupTimeout comes from the profile; model loading dominates it.
|
|
StartupTimeout time.Duration
|
|
// Pool and Fabric are control plane concepts the agent cannot derive. They
|
|
// travel with the command so the agent can report them back after a
|
|
// control plane restart (plan §6.5).
|
|
Pool string
|
|
Fabric string
|
|
}
|
|
|
|
// Dispatcher sends commands to agents.
|
|
//
|
|
// It is an interface so the reconciliation loop can be tested without a network:
|
|
// the loop's logic is where the interesting failures are, not the transport.
|
|
type Dispatcher interface {
|
|
StartShard(ctx context.Context, cmd ShardCommand) error
|
|
StopShard(ctx context.Context, nodeID, shardID string) error
|
|
DrainShard(ctx context.Context, nodeID, shardID string) error
|
|
}
|
|
|
|
// Config tunes the reconciliation loop.
|
|
type Config struct {
|
|
// Interval is how often desired and actual are compared when nothing else
|
|
// happens. Events wake the loop sooner.
|
|
Interval time.Duration
|
|
// BaseBackoff is how long to wait before rebuilding a failed replica.
|
|
BaseBackoff time.Duration
|
|
// MaxBackoff caps the exponential growth.
|
|
MaxBackoff time.Duration
|
|
// UnreachableAfter is how long without a heartbeat before a node is
|
|
// unreachable.
|
|
UnreachableAfter time.Duration
|
|
// LostAfter is how long unreachable before its replicas are declared dead.
|
|
// Longer than UnreachableAfter on purpose: a brief network blip must not
|
|
// tear down a replica that is still serving perfectly well.
|
|
LostAfter time.Duration
|
|
// DrainGrace bounds how long a draining replica waits for its replacement.
|
|
// Without a bound, draining a node whose model cannot be placed anywhere
|
|
// else would never finish.
|
|
DrainGrace time.Duration
|
|
// PortRange is the per-node port range for engine HTTP and dist-init.
|
|
PortRange [2]int
|
|
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
|
|
MgmtInterface string
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
DrainGrace: 5 * time.Minute,
|
|
PortRange: [2]int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
d := DefaultConfig()
|
|
if c.Interval <= 0 {
|
|
c.Interval = d.Interval
|
|
}
|
|
if c.BaseBackoff <= 0 {
|
|
c.BaseBackoff = d.BaseBackoff
|
|
}
|
|
if c.MaxBackoff < c.BaseBackoff {
|
|
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
|
|
}
|
|
if c.UnreachableAfter <= 0 {
|
|
c.UnreachableAfter = d.UnreachableAfter
|
|
}
|
|
if c.LostAfter < c.UnreachableAfter {
|
|
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
|
|
}
|
|
if c.DrainGrace <= 0 {
|
|
c.DrainGrace = d.DrainGrace
|
|
}
|
|
if c.PortRange[1] <= c.PortRange[0] {
|
|
c.PortRange = d.PortRange
|
|
}
|
|
if c.MgmtInterface == "" {
|
|
c.MgmtInterface = d.MgmtInterface
|
|
}
|
|
}
|
|
|
|
// Reconciler compares the catalog against what is running and closes the gap.
|
|
type Reconciler struct {
|
|
cfg Config
|
|
cat *catalog.Catalog
|
|
inv *inventory.Store
|
|
sched *placement.Scheduler
|
|
disp Dispatcher
|
|
log *slog.Logger
|
|
now func() time.Time
|
|
|
|
mu sync.Mutex
|
|
replicas map[string]*Replica
|
|
// nextAttempt is the earliest a model may be rebuilt, per model.
|
|
nextAttempt map[string]time.Time
|
|
// attempts counts consecutive failures per model, for the backoff.
|
|
attempts map[string]int
|
|
// lastExplain keeps the most recent placement failure per model, so
|
|
// `ctl explain-placement` can answer without re-running placement.
|
|
lastExplain map[string]placement.Result
|
|
seq int
|
|
|
|
wake chan struct{}
|
|
stop chan struct{}
|
|
done chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
// New builds a reconciler.
|
|
func New(cfg Config, cat *catalog.Catalog, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Reconciler{
|
|
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
|
|
sched: placement.New(inv),
|
|
now: time.Now,
|
|
replicas: map[string]*Replica{},
|
|
nextAttempt: map[string]time.Time{},
|
|
attempts: map[string]int{},
|
|
lastExplain: map[string]placement.Result{},
|
|
wake: make(chan struct{}, 1),
|
|
stop: make(chan struct{}),
|
|
done: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
// SetClock replaces the time source, for tests.
|
|
func (r *Reconciler) SetClock(now func() time.Time) { r.now = now }
|
|
|
|
// Start runs the loop until Close.
|
|
func (r *Reconciler) Start(ctx context.Context) {
|
|
go func() {
|
|
defer close(r.done)
|
|
t := time.NewTicker(r.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
r.Reconcile(ctx)
|
|
for {
|
|
select {
|
|
case <-r.stop:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case <-r.wake:
|
|
r.Reconcile(ctx)
|
|
case <-t.C:
|
|
r.Reconcile(ctx)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Close stops the loop.
|
|
func (r *Reconciler) Close() {
|
|
r.once.Do(func() {
|
|
close(r.stop)
|
|
<-r.done
|
|
})
|
|
}
|
|
|
|
// Notify wakes the loop, for when something changed and waiting for the next
|
|
// tick would add pointless latency.
|
|
func (r *Reconciler) Notify() {
|
|
select {
|
|
case r.wake <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// Reconcile performs one pass. It is exported so tests can drive it directly
|
|
// instead of racing a ticker.
|
|
func (r *Reconciler) Reconcile(ctx context.Context) {
|
|
r.expireNodes()
|
|
r.drainCordonedNodes(ctx)
|
|
r.reapFailed(ctx)
|
|
|
|
for _, m := range r.cat.Models() {
|
|
r.reconcileModel(ctx, m)
|
|
}
|
|
}
|
|
|
|
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
|
|
// replicas of nodes that are gone.
|
|
//
|
|
// The two-stage delay is deliberate: a brief network blip must not tear down
|
|
// replicas that are still serving traffic perfectly well, and rebuilding a
|
|
// multi-node replica costs tens of minutes of model loading.
|
|
func (r *Reconciler) expireNodes() {
|
|
now := r.now()
|
|
for _, n := range r.inv.Nodes() {
|
|
silent := now.Sub(n.LastSeen)
|
|
switch {
|
|
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
|
|
r.inv.SetState(n.ID, inventory.Unreachable)
|
|
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
|
|
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
|
|
r.inv.SetState(n.ID, inventory.Lost)
|
|
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
|
"node", n.ID, "silent_for", silent.Round(time.Second))
|
|
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
|
}
|
|
}
|
|
}
|
|
|
|
// drainCordonedNodes moves work off nodes an operator is taking out of service.
|
|
//
|
|
// Draining a node is not the same as losing one: the engines are healthy and
|
|
// still serving, so their replicas are marked Draining rather than Failed. That
|
|
// makes the model look short of replicas, a replacement is placed elsewhere,
|
|
// and the original is stopped only once the replacement is serving — which is
|
|
// the difference between a rolling move and an outage.
|
|
func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
|
|
var toDrain []*Replica
|
|
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State != Ready && rep.State != Starting && rep.State != Placing {
|
|
continue
|
|
}
|
|
for _, s := range rep.Shards {
|
|
n, ok := r.inv.Get(s.NodeID)
|
|
if ok && n.State == inventory.Draining {
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
toDrain = append(toDrain, rep)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
shards := map[string][]Shard{}
|
|
for _, rep := range toDrain {
|
|
shards[rep.ID] = slices.Clone(rep.Shards)
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range toDrain {
|
|
r.log.Info("draining replica off a node being taken out of service",
|
|
"replica", rep.ID, "nodes", rep.NodeIDs())
|
|
for _, s := range shards[rep.ID] {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() || !rep.OnNode(nodeID) {
|
|
continue
|
|
}
|
|
rep.State = Failed
|
|
rep.Err = reason
|
|
}
|
|
}
|
|
|
|
// reapFailed releases the resources of terminal replicas and tells the agents
|
|
// to clean up whatever might still be running.
|
|
func (r *Reconciler) reapFailed(ctx context.Context) {
|
|
r.mu.Lock()
|
|
var doomed []*Replica
|
|
for _, rep := range r.replicas {
|
|
if rep.State.Terminal() {
|
|
doomed = append(doomed, rep)
|
|
}
|
|
}
|
|
for _, rep := range doomed {
|
|
delete(r.replicas, rep.ID)
|
|
if rep.State == Failed {
|
|
// Carry the failure count on the model, not the replica: the
|
|
// replica is about to disappear, and the backoff has to survive it.
|
|
r.attempts[rep.Model]++
|
|
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
for _, rep := range doomed {
|
|
// rep is out of the map now, so no agent report can reach it any more.
|
|
if rep.State == Failed {
|
|
r.mu.Lock()
|
|
retry := r.backoff(r.attempts[rep.Model])
|
|
r.mu.Unlock()
|
|
r.log.Error("replica failed",
|
|
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
|
|
"retry_in", retry.Round(time.Second))
|
|
}
|
|
// Stop every shard, including the ones that looked healthy: a partially
|
|
// alive replica holds GPUs while serving nothing.
|
|
for _, s := range rep.Shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard of a dead replica failed",
|
|
"replica", rep.ID, "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(rep.ID)
|
|
}
|
|
}
|
|
|
|
// backoff grows with consecutive failures and is jittered, so a cluster-wide
|
|
// problem does not produce synchronized retry storms across every model.
|
|
func (r *Reconciler) backoff(attempt int) time.Duration {
|
|
if attempt <= 0 {
|
|
return 0
|
|
}
|
|
d := r.cfg.BaseBackoff << min(attempt-1, 16)
|
|
if d > r.cfg.MaxBackoff || d <= 0 {
|
|
d = r.cfg.MaxBackoff
|
|
}
|
|
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
|
|
return d - d/8 + jitter
|
|
}
|
|
|
|
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
|
// Snapshot under the lock and work from values afterwards. A replica
|
|
// pointer stops being safe the moment it is in the map: an agent's status
|
|
// report can mutate it from another goroutine at any time.
|
|
r.mu.Lock()
|
|
var liveIDs, drainingIDs, avoid []string
|
|
for _, rep := range r.replicas {
|
|
if rep.Model != m.Name {
|
|
continue
|
|
}
|
|
switch {
|
|
case rep.State.Counts():
|
|
liveIDs = append(liveIDs, rep.ID)
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
case rep.State == Draining:
|
|
drainingIDs = append(drainingIDs, rep.ID)
|
|
}
|
|
}
|
|
blockedUntil := r.nextAttempt[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
desired := m.Scaling.Min
|
|
switch {
|
|
case len(liveIDs) < desired:
|
|
if r.now().Before(blockedUntil) {
|
|
// Still in backoff. Rebuilding immediately would hot-loop on a
|
|
// model that cannot start — a missing weights path, say — and bury
|
|
// the real error in noise.
|
|
return
|
|
}
|
|
r.scaleUp(ctx, m, avoid)
|
|
case len(liveIDs) > desired:
|
|
r.scaleDown(ctx, liveIDs[desired:])
|
|
}
|
|
|
|
// A drained replica is stopped once its replacement is serving, or once the
|
|
// grace period expires. Stopping it the moment it was drained would cut
|
|
// in-flight requests, which is the thing draining exists to avoid.
|
|
replaced := len(liveIDs) >= desired
|
|
for _, id := range drainingIDs {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
var expired bool
|
|
if ok {
|
|
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > r.cfg.DrainGrace
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
switch {
|
|
case !ok:
|
|
case replaced:
|
|
r.stopReplica(ctx, id)
|
|
case expired:
|
|
// The replacement never arrived — often because nowhere else can
|
|
// host it. Draining forever would leave the node undrainable.
|
|
r.log.Warn("stopping a drained replica without a replacement",
|
|
"replica", id, "after", r.cfg.DrainGrace)
|
|
r.stopReplica(ctx, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// scaleUp places and starts one replica per pass.
|
|
//
|
|
// One at a time on purpose: placement decisions depend on what is already
|
|
// allocated, and starting several at once from the same inventory snapshot
|
|
// would hand them the same GPUs.
|
|
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
|
|
res := r.sched.Place(m.PlacementProfiles(), avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
// Spreading across failure domains is a preference, not a rule. If it
|
|
// is the only thing standing in the way, take the placement anyway:
|
|
// a second replica on one node still survives an engine crash.
|
|
res = r.sched.Place(m.PlacementProfiles(), nil)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.lastExplain[m.Name] = res
|
|
r.mu.Unlock()
|
|
|
|
if res.Placement == nil {
|
|
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
|
|
return
|
|
}
|
|
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
|
r.log.Error("starting replica failed", "model", m.Name, "err", err)
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
|
|
prof, ok := m.ProfileByName(pl.Profile.Name)
|
|
if !ok {
|
|
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
|
|
}
|
|
espec, ok := m.EngineProfile(prof.Name)
|
|
if !ok {
|
|
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
r.seq++
|
|
id := fmt.Sprintf("%s-%d", m.Name, r.seq)
|
|
attempt := r.attempts[m.Name]
|
|
r.mu.Unlock()
|
|
|
|
rep := &Replica{
|
|
ID: id, Model: m.Name, Profile: prof.Name,
|
|
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
|
|
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
|
|
}
|
|
|
|
// The leader's address and port are fixed before any shard starts: every
|
|
// rank needs the same rendezvous point, and a worker that starts first must
|
|
// already know where to dial.
|
|
leader := pl.Nodes[0]
|
|
leaderNode, ok := r.inv.Get(leader.NodeID)
|
|
if !ok {
|
|
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
|
|
}
|
|
basePort := r.allocatePorts(pl)
|
|
|
|
for _, a := range pl.Nodes {
|
|
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
|
|
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
|
|
HTTPPort: basePort, State: supervisor.Pending,
|
|
})
|
|
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
|
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
|
})
|
|
}
|
|
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
|
|
|
// Everything read from rep has to be read now: once it is in the map an
|
|
// agent status report can mutate it concurrently, and the race detector is
|
|
// right to object.
|
|
//
|
|
// The leader first, then workers ascending. Both engines put the rendezvous
|
|
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
|
|
// worker started first has nothing to connect to and spends its startup
|
|
// budget retrying.
|
|
//
|
|
// An earlier version started workers first, on the theory that the head
|
|
// should wait for them. That is backwards: vLLM's server blocks until the
|
|
// Ray cluster has the GPUs it needs, so the leader can be started
|
|
// immediately and will wait on its own.
|
|
order := slices.Clone(rep.Shards)
|
|
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
|
|
nodeIDs := rep.NodeIDs()
|
|
|
|
r.mu.Lock()
|
|
r.replicas[id] = rep
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range order {
|
|
cmd := ShardCommand{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
NodeID: s.NodeID,
|
|
Spec: engines.ShardSpec{
|
|
ReplicaID: id,
|
|
ShardID: s.ID,
|
|
Rank: s.Rank,
|
|
NNodes: len(pl.Nodes),
|
|
LeaderAddr: addrOf(leaderNode),
|
|
LeaderPort: basePort + 1,
|
|
GPUIndices: s.GPUIndices,
|
|
HCAs: s.HCAs,
|
|
MgmtInterface: r.cfg.MgmtInterface,
|
|
HTTPPort: s.HTTPPort,
|
|
Profile: espec,
|
|
},
|
|
StartupTimeout: prof.StartupTimeout,
|
|
Pool: pl.Pool,
|
|
Fabric: pl.Fabric,
|
|
}
|
|
if err := r.disp.StartShard(ctx, cmd); err != nil {
|
|
r.markShard(id, s.ID, supervisor.Failed, err.Error())
|
|
return fmt.Errorf("starting shard %s: %w", s.ID, err)
|
|
}
|
|
r.markShard(id, s.ID, supervisor.Starting, "")
|
|
}
|
|
|
|
r.log.Info("replica placed",
|
|
"replica", id, "model", m.Name, "profile", prof.Name,
|
|
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
|
|
return nil
|
|
}
|
|
|
|
// allocatePorts picks a port shared by every shard of a replica.
|
|
//
|
|
// The same port on every node keeps the leader address predictable and the
|
|
// port bookkeeping trivial; nodes do not share a network namespace, so there is
|
|
// no conflict in reusing it.
|
|
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
|
|
used := map[int]struct{}{}
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
for _, s := range rep.Shards {
|
|
used[s.HTTPPort] = struct{}{}
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
// Two ports per replica: HTTP and the distributed init rendezvous.
|
|
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
|
|
if _, taken := used[p]; !taken {
|
|
return p
|
|
}
|
|
}
|
|
return r.cfg.PortRange[0]
|
|
}
|
|
|
|
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
|
for _, id := range excess {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
continue
|
|
}
|
|
rep.State = Draining
|
|
rep.DrainingAt = r.now()
|
|
shards := slices.Clone(rep.Shards)
|
|
r.mu.Unlock()
|
|
|
|
r.log.Info("draining replica", "replica", id)
|
|
for _, s := range shards {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
if !ok {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
shards := slices.Clone(rep.Shards)
|
|
rep.State = Stopped
|
|
r.mu.Unlock()
|
|
|
|
for _, s := range shards {
|
|
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
r.inv.ReleaseReplica(id)
|
|
|
|
r.mu.Lock()
|
|
delete(r.replicas, id)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// Adopt rebuilds replica state from shards an agent reports it is already
|
|
// running (plan §6.5).
|
|
//
|
|
// This is what makes a control plane restart survivable. Without it the
|
|
// reconciler sees an empty cluster, places duplicates, and the engines already
|
|
// running keep holding GPUs the inventory believes are free — so the duplicates
|
|
// fail to allocate, with an error naming the new process rather than the old.
|
|
//
|
|
// Shards whose model or profile is no longer in the catalog are returned for
|
|
// the caller to stop: nothing will ever claim them.
|
|
func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
|
byReplica := map[string][]AdoptedShard{}
|
|
for _, s := range shards {
|
|
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
for replicaID, group := range byReplica {
|
|
if _, known := r.replicas[replicaID]; known {
|
|
// Already tracked — an agent reconnecting rather than a control
|
|
// plane restart. Its status reports keep the state current.
|
|
continue
|
|
}
|
|
|
|
first := group[0]
|
|
m, ok := r.cat.Resolve(first.Model)
|
|
if !ok {
|
|
unclaimed = append(unclaimed, group...)
|
|
continue
|
|
}
|
|
prof, ok := m.ProfileByName(first.Profile)
|
|
if !ok {
|
|
unclaimed = append(unclaimed, group...)
|
|
continue
|
|
}
|
|
|
|
rep := &Replica{
|
|
ID: replicaID, Model: m.Name, Profile: prof.Name,
|
|
Fabric: first.Fabric, Pool: first.Pool,
|
|
Weight: prof.Weight, CreatedAt: r.now(),
|
|
Endpoint: first.Endpoint,
|
|
}
|
|
for _, s := range group {
|
|
rep.Shards = append(rep.Shards, Shard{
|
|
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
|
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
|
HTTPPort: s.HTTPPort, State: s.State,
|
|
})
|
|
// Reclaim the GPUs, or the next placement will hand them out twice.
|
|
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
|
ShardID: s.ShardID, ReplicaID: replicaID, GPUIndices: s.GPUIndices,
|
|
})
|
|
}
|
|
// The endpoint comes from the leader; a group missing rank 0 is a
|
|
// partial replica and will be failed by deriveState below.
|
|
for _, s := range group {
|
|
if s.Rank == 0 && s.Endpoint != "" {
|
|
rep.Endpoint = s.Endpoint
|
|
}
|
|
}
|
|
rep.deriveState()
|
|
r.replicas[replicaID] = rep
|
|
|
|
r.log.Info("adopted replica from the fleet",
|
|
"replica", replicaID, "model", m.Name, "shards", len(group), "state", rep.State)
|
|
}
|
|
return unclaimed
|
|
}
|
|
|
|
// AdoptedShard is one running shard an agent reported at registration.
|
|
type AdoptedShard struct {
|
|
NodeID string
|
|
ReplicaID string
|
|
ShardID string
|
|
Rank int
|
|
GPUIndices []int
|
|
HCAs []string
|
|
HTTPPort int
|
|
State supervisor.State
|
|
Model string
|
|
Profile string
|
|
Pool string
|
|
Fabric string
|
|
// Endpoint is the leader's base URL, set on the rank 0 shard.
|
|
Endpoint string
|
|
}
|
|
|
|
// OnShardStatus records a status report from an agent.
|
|
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.markShard(replicaID, shardID, st, errMsg)
|
|
r.Notify()
|
|
}
|
|
|
|
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
rep, ok := r.replicas[replicaID]
|
|
if !ok {
|
|
return
|
|
}
|
|
for i := range rep.Shards {
|
|
if rep.Shards[i].ID != shardID {
|
|
continue
|
|
}
|
|
rep.Shards[i].State = st
|
|
if errMsg != "" {
|
|
rep.Shards[i].Err = errMsg
|
|
}
|
|
break
|
|
}
|
|
|
|
prev := rep.State
|
|
rep.deriveState()
|
|
if rep.State == Ready && prev != Ready {
|
|
// A model that comes up clears its backoff; the next failure starts
|
|
// counting from scratch rather than from an old streak.
|
|
r.attempts[rep.Model] = 0
|
|
delete(r.nextAttempt, rep.Model)
|
|
}
|
|
}
|
|
|
|
// Replicas returns a snapshot of every replica, sorted by id.
|
|
func (r *Reconciler) Replicas() []Replica {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
out := make([]Replica, 0, len(r.replicas))
|
|
for _, rep := range r.replicas {
|
|
clone := *rep
|
|
clone.Shards = slices.Clone(rep.Shards)
|
|
out = append(out, clone)
|
|
}
|
|
slices.SortFunc(out, func(a, b Replica) int {
|
|
if a.ID < b.ID {
|
|
return -1
|
|
}
|
|
if a.ID > b.ID {
|
|
return 1
|
|
}
|
|
return 0
|
|
})
|
|
return out
|
|
}
|
|
|
|
// ReadyReplicas returns the replicas the router may send traffic to.
|
|
func (r *Reconciler) ReadyReplicas() []Replica {
|
|
var out []Replica
|
|
for _, rep := range r.Replicas() {
|
|
if rep.State == Ready {
|
|
out = append(out, rep)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Explain returns the most recent placement result for a model, which is what
|
|
// `ctl explain-placement` prints.
|
|
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
res, ok := r.lastExplain[model]
|
|
return res, ok
|
|
}
|
|
|
|
func addrOf(n inventory.Node) string {
|
|
if n.MgmtAddr != "" {
|
|
return n.MgmtAddr
|
|
}
|
|
if n.Hostname != "" {
|
|
return n.Hostname
|
|
}
|
|
return n.ID
|
|
}
|