Files
llmgw/internal/control/reconciler/reconciler.go
T
ilkerandClaude Opus 5 1d1a6aebb0
CI / test (push) Failing after 53s
Phase 4: make a multi-node replica able to start
Placement and environment generation had landed early, which made multi-node
look further along than it was. A replica spanning nodes could not come up at
all, and all three reasons came down to assumptions that only hold on one node.

Not every rank serves HTTP. The agent health-checked every shard at its own
port, but only the leader answers; workers timed out, reported Failed and took
the replica with them. Engine.ServesHTTP now says which ranks serve, and a
worker is judged by staying alive. That is deliberately weaker evidence: the
replica's real readiness is the leader answering, which only happens once every
worker has joined.

A shard is not always one process. Engine.Commands returns a sequence — every
command but the last is a sidecar kept alive for the shard's lifetime and torn
down with it, and the last one's exit is the shard's. vLLM under Ray is why: the
leader runs a Ray head and then the server against it, while a worker runs only
the join and no vLLM process at all. RayCommand had been written and tested but
never called, so launch_mode: ray would have run vllm serve against a cluster
that did not exist. Sidecars are killed with the shard, because a Ray head left
behind holds the port the next replica needs.

Start order was backwards. Workers went first, on the theory that the head
should wait for them. Both engines put the rendezvous point on rank 0, so a
worker started first has nothing to connect to. vLLM's server blocks until the
Ray cluster has the GPUs it needs, so the leader can start immediately and waits
on its own. A command can now also require that rendezvous point to be listening
before the next starts, bounded by a timeout so a head that never appears fails
the shard instead of hanging it.

Stop was also killing only the engine, leaving sidecars behind.

None of this is verified against real engines — it is tested with stub
processes. The Ray sequencing, and which launch mode a given vLLM version wants,
can only be confirmed on hardware.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 06:20:03 +02:00

710 lines
20 KiB
Go

package reconciler
import (
"context"
"fmt"
"log/slog"
"math/rand/v2"
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
// ShardCommand is everything an agent needs to launch one shard.
type ShardCommand struct {
ReplicaID string
ShardID string
NodeID string
Spec engines.ShardSpec
// StartupTimeout comes from the profile; model loading dominates it.
StartupTimeout time.Duration
// Pool and Fabric are control plane concepts the agent cannot derive. They
// travel with the command so the agent can report them back after a
// control plane restart (plan §6.5).
Pool string
Fabric string
}
// Dispatcher sends commands to agents.
//
// It is an interface so the reconciliation loop can be tested without a network:
// the loop's logic is where the interesting failures are, not the transport.
type Dispatcher interface {
StartShard(ctx context.Context, cmd ShardCommand) error
StopShard(ctx context.Context, nodeID, shardID string) error
DrainShard(ctx context.Context, nodeID, shardID string) error
}
// Config tunes the reconciliation loop.
type Config struct {
// Interval is how often desired and actual are compared when nothing else
// happens. Events wake the loop sooner.
Interval time.Duration
// BaseBackoff is how long to wait before rebuilding a failed replica.
BaseBackoff time.Duration
// MaxBackoff caps the exponential growth.
MaxBackoff time.Duration
// UnreachableAfter is how long without a heartbeat before a node is
// unreachable.
UnreachableAfter time.Duration
// LostAfter is how long unreachable before its replicas are declared dead.
// Longer than UnreachableAfter on purpose: a brief network blip must not
// tear down a replica that is still serving perfectly well.
LostAfter time.Duration
// PortRange is the per-node port range for engine HTTP and dist-init.
PortRange [2]int
// MgmtInterface is the ethernet interface used for NCCL bootstrap.
MgmtInterface string
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Interval: 5 * time.Second,
BaseBackoff: 10 * time.Second,
MaxBackoff: 10 * time.Minute,
UnreachableAfter: 15 * time.Second,
LostAfter: 60 * time.Second,
PortRange: [2]int{20000, 20999},
MgmtInterface: "eth0",
}
}
func (c *Config) normalize() {
d := DefaultConfig()
if c.Interval <= 0 {
c.Interval = d.Interval
}
if c.BaseBackoff <= 0 {
c.BaseBackoff = d.BaseBackoff
}
if c.MaxBackoff < c.BaseBackoff {
c.MaxBackoff = max(c.BaseBackoff, d.MaxBackoff)
}
if c.UnreachableAfter <= 0 {
c.UnreachableAfter = d.UnreachableAfter
}
if c.LostAfter < c.UnreachableAfter {
c.LostAfter = max(c.UnreachableAfter*4, d.LostAfter)
}
if c.PortRange[1] <= c.PortRange[0] {
c.PortRange = d.PortRange
}
if c.MgmtInterface == "" {
c.MgmtInterface = d.MgmtInterface
}
}
// Reconciler compares the catalog against what is running and closes the gap.
type Reconciler struct {
cfg Config
cat *catalog.Catalog
inv *inventory.Store
sched *placement.Scheduler
disp Dispatcher
log *slog.Logger
now func() time.Time
mu sync.Mutex
replicas map[string]*Replica
// nextAttempt is the earliest a model may be rebuilt, per model.
nextAttempt map[string]time.Time
// attempts counts consecutive failures per model, for the backoff.
attempts map[string]int
// lastExplain keeps the most recent placement failure per model, so
// `ctl explain-placement` can answer without re-running placement.
lastExplain map[string]placement.Result
seq int
wake chan struct{}
stop chan struct{}
done chan struct{}
once sync.Once
}
// New builds a reconciler.
func New(cfg Config, cat *catalog.Catalog, inv *inventory.Store, disp Dispatcher, log *slog.Logger) *Reconciler {
cfg.normalize()
if log == nil {
log = slog.Default()
}
return &Reconciler{
cfg: cfg, cat: cat, inv: inv, disp: disp, log: log,
sched: placement.New(inv),
now: time.Now,
replicas: map[string]*Replica{},
nextAttempt: map[string]time.Time{},
attempts: map[string]int{},
lastExplain: map[string]placement.Result{},
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
}
}
// SetClock replaces the time source, for tests.
func (r *Reconciler) SetClock(now func() time.Time) { r.now = now }
// Start runs the loop until Close.
func (r *Reconciler) Start(ctx context.Context) {
go func() {
defer close(r.done)
t := time.NewTicker(r.cfg.Interval)
defer t.Stop()
r.Reconcile(ctx)
for {
select {
case <-r.stop:
return
case <-ctx.Done():
return
case <-r.wake:
r.Reconcile(ctx)
case <-t.C:
r.Reconcile(ctx)
}
}
}()
}
// Close stops the loop.
func (r *Reconciler) Close() {
r.once.Do(func() {
close(r.stop)
<-r.done
})
}
// Notify wakes the loop, for when something changed and waiting for the next
// tick would add pointless latency.
func (r *Reconciler) Notify() {
select {
case r.wake <- struct{}{}:
default:
}
}
// Reconcile performs one pass. It is exported so tests can drive it directly
// instead of racing a ticker.
func (r *Reconciler) Reconcile(ctx context.Context) {
r.expireNodes()
r.reapFailed(ctx)
for _, m := range r.cat.Models() {
r.reconcileModel(ctx, m)
}
}
// expireNodes moves silent nodes through Unreachable to Lost, and fails the
// replicas of nodes that are gone.
//
// The two-stage delay is deliberate: a brief network blip must not tear down
// replicas that are still serving traffic perfectly well, and rebuilding a
// multi-node replica costs tens of minutes of model loading.
func (r *Reconciler) expireNodes() {
now := r.now()
for _, n := range r.inv.Nodes() {
silent := now.Sub(n.LastSeen)
switch {
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
r.inv.SetState(n.ID, inventory.Unreachable)
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
}
}
}
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
r.mu.Lock()
defer r.mu.Unlock()
for _, rep := range r.replicas {
if rep.State.Terminal() || !rep.OnNode(nodeID) {
continue
}
rep.State = Failed
rep.Err = reason
}
}
// reapFailed releases the resources of terminal replicas and tells the agents
// to clean up whatever might still be running.
func (r *Reconciler) reapFailed(ctx context.Context) {
r.mu.Lock()
var doomed []*Replica
for _, rep := range r.replicas {
if rep.State.Terminal() {
doomed = append(doomed, rep)
}
}
for _, rep := range doomed {
delete(r.replicas, rep.ID)
if rep.State == Failed {
// Carry the failure count on the model, not the replica: the
// replica is about to disappear, and the backoff has to survive it.
r.attempts[rep.Model]++
r.nextAttempt[rep.Model] = r.now().Add(r.backoff(r.attempts[rep.Model]))
}
}
r.mu.Unlock()
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", retry.Round(time.Second))
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
for _, s := range rep.Shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard of a dead replica failed",
"replica", rep.ID, "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(rep.ID)
}
}
// backoff grows with consecutive failures and is jittered, so a cluster-wide
// problem does not produce synchronized retry storms across every model.
func (r *Reconciler) backoff(attempt int) time.Duration {
if attempt <= 0 {
return 0
}
d := r.cfg.BaseBackoff << min(attempt-1, 16)
if d > r.cfg.MaxBackoff || d <= 0 {
d = r.cfg.MaxBackoff
}
jitter := time.Duration(rand.Int64N(int64(d / 4))) //nolint:gosec // jitter
return d - d/8 + jitter
}
func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// Snapshot under the lock and work from values afterwards. A replica
// pointer stops being safe the moment it is in the map: an agent's status
// report can mutate it from another goroutine at any time.
r.mu.Lock()
var liveIDs, drainingIDs, avoid []string
for _, rep := range r.replicas {
if rep.Model != m.Name {
continue
}
switch {
case rep.State.Counts():
liveIDs = append(liveIDs, rep.ID)
avoid = append(avoid, rep.NodeIDs()...)
case rep.State == Draining:
drainingIDs = append(drainingIDs, rep.ID)
}
}
blockedUntil := r.nextAttempt[m.Name]
r.mu.Unlock()
desired := m.Scaling.Min
switch {
case len(liveIDs) < desired:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
// model that cannot start — a missing weights path, say — and bury
// the real error in noise.
return
}
r.scaleUp(ctx, m, avoid)
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
}
// Drained replicas can finally be stopped.
for _, id := range drainingIDs {
r.stopReplica(ctx, id)
}
}
// scaleUp places and starts one replica per pass.
//
// One at a time on purpose: placement decisions depend on what is already
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) {
res := r.sched.Place(m.PlacementProfiles(), avoid)
if res.Placement == nil && len(avoid) > 0 {
// Spreading across failure domains is a preference, not a rule. If it
// is the only thing standing in the way, take the placement anyway:
// a second replica on one node still survives an engine crash.
res = r.sched.Place(m.PlacementProfiles(), nil)
}
r.mu.Lock()
r.lastExplain[m.Name] = res
r.mu.Unlock()
if res.Placement == nil {
r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain())
return
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
r.log.Error("starting replica failed", "model", m.Name, "err", err)
}
}
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
prof, ok := m.ProfileByName(pl.Profile.Name)
if !ok {
return fmt.Errorf("profile %q vanished from model %q", pl.Profile.Name, m.Name)
}
espec, ok := m.EngineProfile(prof.Name)
if !ok {
return fmt.Errorf("model %q: no engine profile %q", m.Name, prof.Name)
}
r.mu.Lock()
r.seq++
id := fmt.Sprintf("%s-%d", m.Name, r.seq)
attempt := r.attempts[m.Name]
r.mu.Unlock()
rep := &Replica{
ID: id, Model: m.Name, Profile: prof.Name,
Fabric: pl.Fabric, Pool: pl.Pool, State: Placing,
Weight: prof.Weight, CreatedAt: r.now(), Attempt: attempt,
}
// The leader's address and port are fixed before any shard starts: every
// rank needs the same rendezvous point, and a worker that starts first must
// already know where to dial.
leader := pl.Nodes[0]
leaderNode, ok := r.inv.Get(leader.NodeID)
if !ok {
return fmt.Errorf("leader node %s disappeared", leader.NodeID)
}
basePort := r.allocatePorts(pl)
for _, a := range pl.Nodes {
shardID := fmt.Sprintf("%s-r%d", id, a.Rank)
rep.Shards = append(rep.Shards, Shard{
ID: shardID, NodeID: a.NodeID, Rank: a.Rank,
GPUIndices: a.GPUIndices, HCAs: a.HCAs,
HTTPPort: basePort, State: supervisor.Pending,
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
// Everything read from rep has to be read now: once it is in the map an
// agent status report can mutate it concurrently, and the race detector is
// right to object.
//
// The leader first, then workers ascending. Both engines put the rendezvous
// point on rank 0 — SGLang's dist-init address and vLLM's Ray head — so a
// worker started first has nothing to connect to and spends its startup
// budget retrying.
//
// An earlier version started workers first, on the theory that the head
// should wait for them. That is backwards: vLLM's server blocks until the
// Ray cluster has the GPUs it needs, so the leader can be started
// immediately and will wait on its own.
order := slices.Clone(rep.Shards)
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
nodeIDs := rep.NodeIDs()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
for _, s := range order {
cmd := ShardCommand{
ReplicaID: id,
ShardID: s.ID,
NodeID: s.NodeID,
Spec: engines.ShardSpec{
ReplicaID: id,
ShardID: s.ID,
Rank: s.Rank,
NNodes: len(pl.Nodes),
LeaderAddr: addrOf(leaderNode),
LeaderPort: basePort + 1,
GPUIndices: s.GPUIndices,
HCAs: s.HCAs,
MgmtInterface: r.cfg.MgmtInterface,
HTTPPort: s.HTTPPort,
Profile: espec,
},
StartupTimeout: prof.StartupTimeout,
Pool: pl.Pool,
Fabric: pl.Fabric,
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
return fmt.Errorf("starting shard %s: %w", s.ID, err)
}
r.markShard(id, s.ID, supervisor.Starting, "")
}
r.log.Info("replica placed",
"replica", id, "model", m.Name, "profile", prof.Name,
"pool", pl.Pool, "fabric", pl.Fabric, "nodes", nodeIDs)
return nil
}
// allocatePorts picks a port shared by every shard of a replica.
//
// The same port on every node keeps the leader address predictable and the
// port bookkeeping trivial; nodes do not share a network namespace, so there is
// no conflict in reusing it.
func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
used := map[int]struct{}{}
r.mu.Lock()
for _, rep := range r.replicas {
for _, s := range rep.Shards {
used[s.HTTPPort] = struct{}{}
}
}
r.mu.Unlock()
// Two ports per replica: HTTP and the distributed init rendezvous.
for p := r.cfg.PortRange[0]; p < r.cfg.PortRange[1]; p += 2 {
if _, taken := used[p]; !taken {
return p
}
}
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
continue
}
rep.State = Draining
shards := slices.Clone(rep.Shards)
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
}
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
r.mu.Unlock()
return
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
r.mu.Unlock()
for _, s := range shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard failed", "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(id)
r.mu.Lock()
delete(r.replicas, id)
r.mu.Unlock()
}
// Adopt rebuilds replica state from shards an agent reports it is already
// running (plan §6.5).
//
// This is what makes a control plane restart survivable. Without it the
// reconciler sees an empty cluster, places duplicates, and the engines already
// running keep holding GPUs the inventory believes are free — so the duplicates
// fail to allocate, with an error naming the new process rather than the old.
//
// Shards whose model or profile is no longer in the catalog are returned for
// the caller to stop: nothing will ever claim them.
func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
byReplica := map[string][]AdoptedShard{}
for _, s := range shards {
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
}
r.mu.Lock()
defer r.mu.Unlock()
for replicaID, group := range byReplica {
if _, known := r.replicas[replicaID]; known {
// Already tracked — an agent reconnecting rather than a control
// plane restart. Its status reports keep the state current.
continue
}
first := group[0]
m, ok := r.cat.Resolve(first.Model)
if !ok {
unclaimed = append(unclaimed, group...)
continue
}
prof, ok := m.ProfileByName(first.Profile)
if !ok {
unclaimed = append(unclaimed, group...)
continue
}
rep := &Replica{
ID: replicaID, Model: m.Name, Profile: prof.Name,
Fabric: first.Fabric, Pool: first.Pool,
Weight: prof.Weight, CreatedAt: r.now(),
Endpoint: first.Endpoint,
}
for _, s := range group {
rep.Shards = append(rep.Shards, Shard{
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
HTTPPort: s.HTTPPort, State: s.State,
})
// Reclaim the GPUs, or the next placement will hand them out twice.
r.inv.Allocate(s.NodeID, inventory.Allocation{
ShardID: s.ShardID, ReplicaID: replicaID, GPUIndices: s.GPUIndices,
})
}
// The endpoint comes from the leader; a group missing rank 0 is a
// partial replica and will be failed by deriveState below.
for _, s := range group {
if s.Rank == 0 && s.Endpoint != "" {
rep.Endpoint = s.Endpoint
}
}
rep.deriveState()
r.replicas[replicaID] = rep
r.log.Info("adopted replica from the fleet",
"replica", replicaID, "model", m.Name, "shards", len(group), "state", rep.State)
}
return unclaimed
}
// AdoptedShard is one running shard an agent reported at registration.
type AdoptedShard struct {
NodeID string
ReplicaID string
ShardID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Model string
Profile string
Pool string
Fabric string
// Endpoint is the leader's base URL, set on the rank 0 shard.
Endpoint string
}
// OnShardStatus records a status report from an agent.
func (r *Reconciler) OnShardStatus(replicaID, shardID string, st supervisor.State, errMsg string) {
r.markShard(replicaID, shardID, st, errMsg)
r.Notify()
}
func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, errMsg string) {
r.mu.Lock()
defer r.mu.Unlock()
rep, ok := r.replicas[replicaID]
if !ok {
return
}
for i := range rep.Shards {
if rep.Shards[i].ID != shardID {
continue
}
rep.Shards[i].State = st
if errMsg != "" {
rep.Shards[i].Err = errMsg
}
break
}
prev := rep.State
rep.deriveState()
if rep.State == Ready && prev != Ready {
// A model that comes up clears its backoff; the next failure starts
// counting from scratch rather than from an old streak.
r.attempts[rep.Model] = 0
delete(r.nextAttempt, rep.Model)
}
}
// Replicas returns a snapshot of every replica, sorted by id.
func (r *Reconciler) Replicas() []Replica {
r.mu.Lock()
defer r.mu.Unlock()
out := make([]Replica, 0, len(r.replicas))
for _, rep := range r.replicas {
clone := *rep
clone.Shards = slices.Clone(rep.Shards)
out = append(out, clone)
}
slices.SortFunc(out, func(a, b Replica) int {
if a.ID < b.ID {
return -1
}
if a.ID > b.ID {
return 1
}
return 0
})
return out
}
// ReadyReplicas returns the replicas the router may send traffic to.
func (r *Reconciler) ReadyReplicas() []Replica {
var out []Replica
for _, rep := range r.Replicas() {
if rep.State == Ready {
out = append(out, rep)
}
}
return out
}
// Explain returns the most recent placement result for a model, which is what
// `ctl explain-placement` prints.
func (r *Reconciler) Explain(model string) (placement.Result, bool) {
r.mu.Lock()
defer r.mu.Unlock()
res, ok := r.lastExplain[model]
return res, ok
}
func addrOf(n inventory.Node) string {
if n.MgmtAddr != "" {
return n.MgmtAddr
}
if n.Hostname != "" {
return n.Hostname
}
return n.ID
}