Files
llmgw/internal/control/reconciler/replica.go
T
Ilker ManapandClaude Opus 5 6f3bde099e
CI / test (push) Successful in 1m20s
Let a multi-node replica be adopted one agent at a time
A replica spanning two nodes is reported by two agents, registering
separately about a second apart. Adoption assumed one call carried a whole
replica, so the first half was adopted with WantShards=2 and immediately
failed for "replica has 1 of 2 shards" — and when the second node
registered, its half was condemned as unclaimed because the replica was
already written off.

big-model therefore could not survive a control plane restart. Seen after
today's deploy: adopted, failed, both halves stopped, rebuilt from
scratch, backing off 9s, 20s, 38s, and never becoming ready. The first
reconcile tick also runs before any agent has reconnected, which is the
harmless-looking "no node reports a fabric" that precedes it.

Two changes. Adopt merges a group into a replica it is still assembling
instead of treating it as a competing claim, and takes the endpoint from
whichever group carries rank 0 — the leader's agent is not necessarily the
first to register. deriveState leaves a replica that is short of shards
while still assembling in Starting rather than Failed, which puts it under
the startup timeout that already exists: a sibling that never arrives
still fails, one second later than it used to rather than one second
before its sibling reports in. AdoptedAt is cleared once the replica is
whole, so losing a shard after that is a failure again.

The existing multi-node adoption test passed throughout, because it hands
both shards to a single Adopt call — which no caller makes. Same shape as
the agentsvc bug in c97b671: a test that calls the function under test
directly cannot tell you how the caller calls it. The new test registers
one node at a time, and fails without this change.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 17:15:00 +02:00

216 lines
6.4 KiB
Go

// Package reconciler drives the cluster toward the catalog's desired state
// (plan §6.3).
package reconciler
import (
"fmt"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
)
// ReplicaState is where a replica stands.
type ReplicaState int
const (
// Placing means it has been scheduled but no shard has started.
Placing ReplicaState = iota
// Starting means shards are launching or loading.
Starting
// Ready means it serves traffic.
Ready
// Draining means it takes no new requests and will be stopped.
Draining
// Stopped means it was shut down on purpose.
Stopped
// Failed means it died or never became ready.
Failed
)
func (s ReplicaState) String() string {
switch s {
case Placing:
return "placing"
case Starting:
return "starting"
case Ready:
return "ready"
case Draining:
return "draining"
case Stopped:
return "stopped"
case Failed:
return "failed"
default:
return "unknown"
}
}
// Terminal reports whether the replica has finished for good.
func (s ReplicaState) Terminal() bool { return s == Stopped || s == Failed }
// Counts reports whether the replica should count toward the desired total.
// Draining replicas do not: they are on the way out, and counting them would
// stop a replacement from being started.
func (s ReplicaState) Counts() bool { return s == Placing || s == Starting || s == Ready }
// Shard is one node's part of a replica.
type Shard struct {
ID string
NodeID string
Rank int
GPUIndices []int
HCAs []string
HTTPPort int
State supervisor.State
Err string
}
// Leader reports whether this shard serves HTTP for the replica.
func (s Shard) Leader() bool { return s.Rank == 0 }
// Replica is one running copy of a model. It is the unit the router sees.
type Replica struct {
ID string
Model string
Profile string
// Fabric is set only for multi-node replicas.
Fabric string
Pool string
Shards []Shard
State ReplicaState
// Endpoint is rank 0's base URL.
Endpoint string
// Weight is the profile's capacity weight, passed to the router.
Weight float64
// WantShards is how many shards this replica should have, from its profile.
//
// Without it a replica cannot tell "every shard I have is ready" from
// "every shard I have left is ready". A control plane that adopted half of
// a two-node replica reported it Ready with one shard and no endpoint, and
// the router was handed a replica nothing could route to.
WantShards int
// AdoptedAt is set while a replica is being reassembled from what agents
// report, and cleared once every shard has arrived.
//
// A multi-node replica does not arrive at once: each node has its own
// agent, each agent registers separately, and a replica spanning two nodes
// is therefore reported in two halves a second or so apart. Between them it
// is legitimately short of shards, and failing it there condemned it before
// its other half had a chance to register — after which that half was
// stopped as unclaimed and the model rebuilt from nothing on every restart.
AdoptedAt time.Time
CreatedAt time.Time
ReadyAt time.Time
// DrainingAt is when the replica stopped taking new work, so the wait for a
// replacement can be bounded.
DrainingAt time.Time
// Attempt counts how many times this model has failed to come up in a row,
// carried across rebuilds so the backoff does not reset on every attempt.
Attempt int
Err string
}
// NodeIDs lists the nodes this replica occupies.
func (r Replica) NodeIDs() []string {
out := make([]string, 0, len(r.Shards))
for _, s := range r.Shards {
out = append(out, s.NodeID)
}
slices.Sort(out)
return out
}
// OnNode reports whether any shard runs on a node.
func (r Replica) OnNode(id string) bool {
return slices.ContainsFunc(r.Shards, func(s Shard) bool { return s.NodeID == id })
}
// deriveState recomputes the replica's state from its shards.
//
// A replica is only as healthy as its worst shard: one failed shard makes the
// whole thing useless, because the others are waiting on collectives that will
// never complete. That is why there is no path here that keeps a replica
// partially alive (plan §5.4).
// hasLiveShard reports whether any shard has actually got somewhere.
//
// Starting and Pending are not evidence of anything: they are what a replica
// looks like both while an engine loads a model and when nothing was ever
// started at all. Loading means the port is open, which only a real process
// does.
func (r *Replica) hasLiveShard() bool {
for _, s := range r.Shards {
if s.State == supervisor.Ready || s.State == supervisor.Loading {
return true
}
}
return false
}
func (r *Replica) deriveState() {
if r.State == Draining || r.State == Stopped {
return
}
if len(r.Shards) == 0 {
r.State = Placing
return
}
ready := 0
for _, s := range r.Shards {
switch s.State {
case supervisor.Failed:
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("shard %s failed: %s", s.ID, s.Err)
}
return
case supervisor.Stopped:
// A shard stopping on its own, while the replica was not draining,
// is a failure of the replica.
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("shard %s stopped unexpectedly", s.ID)
}
return
case supervisor.Ready:
ready++
}
}
// Both conditions. A replica missing a shard is not ready however healthy
// the shards it still has look — and rank 0's absence in particular means
// there is no endpoint to route to at all.
if r.WantShards > 0 && len(r.Shards) >= r.WantShards {
// Fully assembled. From here it is an ordinary replica: losing a shard
// now is a failure, not a half that has yet to report.
r.AdoptedAt = time.Time{}
}
if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) {
if r.State != Ready {
r.ReadyAt = time.Now()
}
r.State = Ready
return
}
if r.WantShards > 0 && len(r.Shards) < r.WantShards {
// Still assembling from the fleet: the rest of the shards are on nodes
// whose agents have not registered yet. Starting rather than Failed,
// which puts it under the ordinary startup timeout — so a half that
// never arrives is still failed, just by the deadline that already
// exists rather than a second before its sibling reports in.
if !r.AdoptedAt.IsZero() {
r.State = Starting
return
}
r.State = Failed
if r.Err == "" {
r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards)
}
return
}
r.State = Starting
}