Let a multi-node replica be adopted one agent at a time
A replica spanning two nodes is reported by two agents, registering
separately about a second apart. Adoption assumed one call carried a whole
replica, so the first half was adopted with WantShards=2 and immediately
failed for "replica has 1 of 2 shards" — and when the second node
registered, its half was condemned as unclaimed because the replica was
already written off.
big-model therefore could not survive a control plane restart. Seen after
today's deploy: adopted, failed, both halves stopped, rebuilt from
scratch, backing off 9s, 20s, 38s, and never becoming ready. The first
reconcile tick also runs before any agent has reconnected, which is the
harmless-looking "no node reports a fabric" that precedes it.
Two changes. Adopt merges a group into a replica it is still assembling
instead of treating it as a competing claim, and takes the endpoint from
whichever group carries rank 0 — the leader's agent is not necessarily the
first to register. deriveState leaves a replica that is short of shards
while still assembling in Starting rather than Failed, which puts it under
the startup timeout that already exists: a sibling that never arrives
still fails, one second later than it used to rather than one second
before its sibling reports in. AdoptedAt is cleared once the replica is
whole, so losing a shard after that is a failure again.
The existing multi-node adoption test passed throughout, because it hands
both shards to a single Adopt call — which no caller makes. Same shape as
the agentsvc bug in c97b671: a test that calls the function under test
directly cannot tell you how the caller calls it. The new test registers
one node at a time, and fails without this change.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
72b03a1a1c
commit
6f3bde099e
3 files changed
+171
-21
No files matched your search
@@ -714,6 +714,17 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
|
||||
|
||||
for replicaID, group := range byReplica {
|
||||
if existing, known := r.replicas[replicaID]; known {
|
||||
// The other half of a replica we are still assembling. Merge rather
|
||||
// than choose: these shards belong to the same replica as the ones
|
||||
// already adopted, they are simply on a different node whose agent
|
||||
// registered a moment later. Treating them as a competing claim is
|
||||
// what tore multi-node replicas apart on every control plane
|
||||
// restart — one half adopted and failed for being short, the other
|
||||
// stopped as unclaimed when it finally arrived.
|
||||
if !existing.AdoptedAt.IsZero() && len(existing.Shards) < existing.WantShards {
|
||||
r.mergeAdopted(existing, group)
|
||||
continue
|
||||
}
|
||||
// Usually this is an agent reconnecting rather than a control plane
|
||||
// restart, and its status reports keep the state current.
|
||||
//
|
||||
@@ -777,34 +788,55 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad
|
||||
// From the profile, not from what was reported: the point is to
|
||||
// notice when what was reported is short.
|
||||
WantShards: max(prof.Nodes, 1),
|
||||
AdoptedAt: r.now(),
|
||||
}
|
||||
for _, s := range group {
|
||||
rep.Shards = append(rep.Shards, Shard{
|
||||
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
||||
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
||||
HTTPPort: s.HTTPPort, State: s.State,
|
||||
})
|
||||
// Reclaim the GPUs, or the next placement will hand them out twice.
|
||||
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
||||
ShardID: s.ShardID, ReplicaID: replicaID, GPUIndices: s.GPUIndices,
|
||||
})
|
||||
}
|
||||
// The endpoint comes from the leader; a group missing rank 0 is a
|
||||
// partial replica and will be failed by deriveState below.
|
||||
for _, s := range group {
|
||||
if s.Rank == 0 && s.Endpoint != "" {
|
||||
rep.Endpoint = s.Endpoint
|
||||
}
|
||||
}
|
||||
rep.deriveState()
|
||||
r.mergeAdopted(rep, group)
|
||||
r.replicas[replicaID] = rep
|
||||
|
||||
r.log.Info("adopted replica from the fleet",
|
||||
"replica", replicaID, "model", m.Name, "shards", len(group), "state", rep.State)
|
||||
}
|
||||
return unclaimed
|
||||
}
|
||||
|
||||
// mergeAdopted folds shards an agent reports into a replica being reassembled.
|
||||
//
|
||||
// Called both for the first group, which creates the replica, and for every
|
||||
// later group belonging to it, so a replica spanning N nodes is built up over N
|
||||
// registrations. Shard ids already present are skipped: an agent that
|
||||
// reconnects repeats what it is running, and counting those again would carry a
|
||||
// replica past its shard count without a second node ever appearing.
|
||||
func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||
have := make(map[string]struct{}, len(rep.Shards))
|
||||
for _, sh := range rep.Shards {
|
||||
have[sh.ID] = struct{}{}
|
||||
}
|
||||
for _, s := range group {
|
||||
if _, dup := have[s.ShardID]; dup {
|
||||
continue
|
||||
}
|
||||
have[s.ShardID] = struct{}{}
|
||||
rep.Shards = append(rep.Shards, Shard{
|
||||
ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank,
|
||||
GPUIndices: s.GPUIndices, HCAs: s.HCAs,
|
||||
HTTPPort: s.HTTPPort, State: s.State,
|
||||
})
|
||||
// Reclaim the GPUs, or the next placement will hand them out twice.
|
||||
r.inv.Allocate(s.NodeID, inventory.Allocation{
|
||||
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
|
||||
})
|
||||
// The endpoint comes from the leader, which may be in any of the
|
||||
// groups — the rank 0 node is not necessarily the first to register.
|
||||
if s.Rank == 0 && s.Endpoint != "" {
|
||||
rep.Endpoint = s.Endpoint
|
||||
}
|
||||
}
|
||||
slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank })
|
||||
rep.deriveState()
|
||||
|
||||
r.log.Info("adopted replica from the fleet",
|
||||
"replica", rep.ID, "model", rep.Model,
|
||||
"shards", len(rep.Shards), "want", rep.WantShards, "state", rep.State)
|
||||
}
|
||||
|
||||
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
|
||||
// agent, having just told us everything it is running, did not mention.
|
||||
//
|
||||
|
||||
@@ -898,3 +898,97 @@ func TestCordonedNodeKeepsItsWork(t *testing.T) {
|
||||
t.Errorf("the replica did not survive a cordon: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdoptMultiNodeReplicaOneAgentAtATime(t *testing.T) {
|
||||
// How it really arrives. Every other multi-node adoption test hands both
|
||||
// shards to a single Adopt call, which no caller ever does: each node has
|
||||
// its own agent and each agent registers separately, so a two-node replica
|
||||
// is reported in two halves a second or so apart.
|
||||
//
|
||||
// Observed on the cluster after a control plane restart: the first half was
|
||||
// adopted and immediately failed for "replica has 1 of 2 shards", and when
|
||||
// the second node registered its half was stopped as unclaimed. big-model
|
||||
// never came back — it rebuilt from scratch on every restart, with
|
||||
// exponential backoff, forever.
|
||||
h := newHarness(t, testModel(1, catalog.Profile{
|
||||
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
||||
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
||||
|
||||
half := func(shardID, nodeID string, rank int) []AdoptedShard {
|
||||
s := adopted("llama-7", shardID, nodeID, rank, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
||||
s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g"
|
||||
return []AdoptedShard{s}
|
||||
}
|
||||
|
||||
// Rank 1 first, so the endpoint arrives in the second group: whichever node
|
||||
// registers first is not necessarily the leader.
|
||||
if unclaimed := h.r.Adopt("2", half("llama-7-r1", "2", 1)); len(unclaimed) != 0 {
|
||||
t.Fatalf("the first half was not claimed: %+v", unclaimed)
|
||||
}
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("expected the half to be adopted, got %d replicas", len(reps))
|
||||
}
|
||||
if reps[0].State == Failed {
|
||||
t.Fatalf("a replica still assembling was failed: %q", reps[0].Err)
|
||||
}
|
||||
|
||||
if unclaimed := h.r.Adopt("1", half("llama-7-r0", "1", 0)); len(unclaimed) != 0 {
|
||||
t.Fatalf("the second half was stopped as unclaimed: %+v", unclaimed)
|
||||
}
|
||||
|
||||
reps = h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("the halves became %d replicas", len(reps))
|
||||
}
|
||||
rep := reps[0]
|
||||
if len(rep.Shards) != 2 {
|
||||
t.Fatalf("the halves were not merged: %d shards", len(rep.Shards))
|
||||
}
|
||||
if rep.State != Ready {
|
||||
t.Errorf("state=%s err=%q; both shards reported Ready", rep.State, rep.Err)
|
||||
}
|
||||
if rep.Endpoint == "" {
|
||||
t.Error("no endpoint; rank 0 arrived in the second group and was dropped")
|
||||
}
|
||||
for _, id := range []string{"1", "2"} {
|
||||
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
||||
t.Errorf("node %s has %d free GPUs; its half's allocation was lost", id, free)
|
||||
}
|
||||
}
|
||||
|
||||
// And the reconciler must not then place a duplicate alongside it.
|
||||
h.disp.reset()
|
||||
h.r.Reconcile(t.Context())
|
||||
if got := len(h.disp.startedShards()); got != 0 {
|
||||
t.Errorf("a duplicate was placed despite the replica being whole (%d shards started)", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHalfThatNeverArrivesStillFails(t *testing.T) {
|
||||
// The grace period must be a deadline, not an amnesty: a replica whose
|
||||
// other node is simply gone has to fail so it can be replaced.
|
||||
h := newHarness(t, testModel(1, catalog.Profile{
|
||||
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
||||
StartupTimeout: time.Minute,
|
||||
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
||||
|
||||
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
||||
s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g"
|
||||
h.r.Adopt("1", []AdoptedShard{s})
|
||||
|
||||
if reps := h.r.Replicas(); reps[0].State != Starting {
|
||||
t.Fatalf("state=%s; the half should be waiting for its sibling", reps[0].State)
|
||||
}
|
||||
|
||||
h.advance(2 * time.Minute)
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) == 1 && reps[0].State != Failed {
|
||||
t.Errorf("state=%s; a half whose sibling never registered must fail", reps[0].State)
|
||||
}
|
||||
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
||||
t.Errorf("node has %d free GPUs; the abandoned half still holds them", free)
|
||||
}
|
||||
}
|
||||
@@ -91,6 +91,16 @@ type Replica struct {
|
||||
// a two-node replica reported it Ready with one shard and no endpoint, and
|
||||
// the router was handed a replica nothing could route to.
|
||||
WantShards int
|
||||
// AdoptedAt is set while a replica is being reassembled from what agents
|
||||
// report, and cleared once every shard has arrived.
|
||||
//
|
||||
// A multi-node replica does not arrive at once: each node has its own
|
||||
// agent, each agent registers separately, and a replica spanning two nodes
|
||||
// is therefore reported in two halves a second or so apart. Between them it
|
||||
// is legitimately short of shards, and failing it there condemned it before
|
||||
// its other half had a chance to register — after which that half was
|
||||
// stopped as unclaimed and the model rebuilt from nothing on every restart.
|
||||
AdoptedAt time.Time
|
||||
|
||||
CreatedAt time.Time
|
||||
ReadyAt time.Time
|
||||
@@ -173,6 +183,11 @@ func (r *Replica) deriveState() {
|
||||
// Both conditions. A replica missing a shard is not ready however healthy
|
||||
// the shards it still has look — and rank 0's absence in particular means
|
||||
// there is no endpoint to route to at all.
|
||||
if r.WantShards > 0 && len(r.Shards) >= r.WantShards {
|
||||
// Fully assembled. From here it is an ordinary replica: losing a shard
|
||||
// now is a failure, not a half that has yet to report.
|
||||
r.AdoptedAt = time.Time{}
|
||||
}
|
||||
if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) {
|
||||
if r.State != Ready {
|
||||
r.ReadyAt = time.Now()
|
||||
@@ -181,6 +196,15 @@ func (r *Replica) deriveState() {
|
||||
return
|
||||
}
|
||||
if r.WantShards > 0 && len(r.Shards) < r.WantShards {
|
||||
// Still assembling from the fleet: the rest of the shards are on nodes
|
||||
// whose agents have not registered yet. Starting rather than Failed,
|
||||
// which puts it under the ordinary startup timeout — so a half that
|
||||
// never arrives is still failed, just by the deadline that already
|
||||
// exists rather than a second before its sibling reports in.
|
||||
if !r.AdoptedAt.IsZero() {
|
||||
r.State = Starting
|
||||
return
|
||||
}
|
||||
r.State = Failed
|
||||
if r.Err == "" {
|
||||
r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards)
|
||||
|
||||
Reference in new issue
Block a user