diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 8d6d0bc..c556668 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -714,6 +714,17 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad for replicaID, group := range byReplica { if existing, known := r.replicas[replicaID]; known { + // The other half of a replica we are still assembling. Merge rather + // than choose: these shards belong to the same replica as the ones + // already adopted, they are simply on a different node whose agent + // registered a moment later. Treating them as a competing claim is + // what tore multi-node replicas apart on every control plane + // restart — one half adopted and failed for being short, the other + // stopped as unclaimed when it finally arrived. + if !existing.AdoptedAt.IsZero() && len(existing.Shards) < existing.WantShards { + r.mergeAdopted(existing, group) + continue + } // Usually this is an agent reconnecting rather than a control plane // restart, and its status reports keep the state current. // @@ -777,34 +788,55 @@ func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []Ad // From the profile, not from what was reported: the point is to // notice when what was reported is short. WantShards: max(prof.Nodes, 1), + AdoptedAt: r.now(), } - for _, s := range group { - rep.Shards = append(rep.Shards, Shard{ - ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank, - GPUIndices: s.GPUIndices, HCAs: s.HCAs, - HTTPPort: s.HTTPPort, State: s.State, - }) - // Reclaim the GPUs, or the next placement will hand them out twice. - r.inv.Allocate(s.NodeID, inventory.Allocation{ - ShardID: s.ShardID, ReplicaID: replicaID, GPUIndices: s.GPUIndices, - }) - } - // The endpoint comes from the leader; a group missing rank 0 is a - // partial replica and will be failed by deriveState below. - for _, s := range group { - if s.Rank == 0 && s.Endpoint != "" { - rep.Endpoint = s.Endpoint - } - } - rep.deriveState() + r.mergeAdopted(rep, group) r.replicas[replicaID] = rep - r.log.Info("adopted replica from the fleet", - "replica", replicaID, "model", m.Name, "shards", len(group), "state", rep.State) } return unclaimed } +// mergeAdopted folds shards an agent reports into a replica being reassembled. +// +// Called both for the first group, which creates the replica, and for every +// later group belonging to it, so a replica spanning N nodes is built up over N +// registrations. Shard ids already present are skipped: an agent that +// reconnects repeats what it is running, and counting those again would carry a +// replica past its shard count without a second node ever appearing. +func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) { + have := make(map[string]struct{}, len(rep.Shards)) + for _, sh := range rep.Shards { + have[sh.ID] = struct{}{} + } + for _, s := range group { + if _, dup := have[s.ShardID]; dup { + continue + } + have[s.ShardID] = struct{}{} + rep.Shards = append(rep.Shards, Shard{ + ID: s.ShardID, NodeID: s.NodeID, Rank: s.Rank, + GPUIndices: s.GPUIndices, HCAs: s.HCAs, + HTTPPort: s.HTTPPort, State: s.State, + }) + // Reclaim the GPUs, or the next placement will hand them out twice. + r.inv.Allocate(s.NodeID, inventory.Allocation{ + ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices, + }) + // The endpoint comes from the leader, which may be in any of the + // groups — the rank 0 node is not necessarily the first to register. + if s.Rank == 0 && s.Endpoint != "" { + rep.Endpoint = s.Endpoint + } + } + slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank }) + rep.deriveState() + + r.log.Info("adopted replica from the fleet", + "replica", rep.ID, "model", rep.Model, + "shards", len(rep.Shards), "want", rep.WantShards, "state", rep.State) +} + // forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the // agent, having just told us everything it is running, did not mention. // diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 1d087d8..b3cbec1 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -898,3 +898,97 @@ func TestCordonedNodeKeepsItsWork(t *testing.T) { t.Errorf("the replica did not survive a cordon: %+v", got) } } + +func TestAdoptMultiNodeReplicaOneAgentAtATime(t *testing.T) { + // How it really arrives. Every other multi-node adoption test hands both + // shards to a single Adopt call, which no caller ever does: each node has + // its own agent and each agent registers separately, so a two-node replica + // is reported in two halves a second or so apart. + // + // Observed on the cluster after a control plane restart: the first half was + // adopted and immediately failed for "replica has 1 of 2 shards", and when + // the second node registered its half was stopped as unclaimed. big-model + // never came back — it rebuilt from scratch on every restart, with + // exponential backoff, forever. + h := newHarness(t, testModel(1, catalog.Profile{ + Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, + }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) + + half := func(shardID, nodeID string, rank int) []AdoptedShard { + s := adopted("llama-7", shardID, nodeID, rank, []int{0, 1, 2, 3, 4, 5, 6, 7}) + s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g" + return []AdoptedShard{s} + } + + // Rank 1 first, so the endpoint arrives in the second group: whichever node + // registers first is not necessarily the leader. + if unclaimed := h.r.Adopt("2", half("llama-7-r1", "2", 1)); len(unclaimed) != 0 { + t.Fatalf("the first half was not claimed: %+v", unclaimed) + } + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("expected the half to be adopted, got %d replicas", len(reps)) + } + if reps[0].State == Failed { + t.Fatalf("a replica still assembling was failed: %q", reps[0].Err) + } + + if unclaimed := h.r.Adopt("1", half("llama-7-r0", "1", 0)); len(unclaimed) != 0 { + t.Fatalf("the second half was stopped as unclaimed: %+v", unclaimed) + } + + reps = h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("the halves became %d replicas", len(reps)) + } + rep := reps[0] + if len(rep.Shards) != 2 { + t.Fatalf("the halves were not merged: %d shards", len(rep.Shards)) + } + if rep.State != Ready { + t.Errorf("state=%s err=%q; both shards reported Ready", rep.State, rep.Err) + } + if rep.Endpoint == "" { + t.Error("no endpoint; rank 0 arrived in the second group and was dropped") + } + for _, id := range []string{"1", "2"} { + if free := len(h.inv.FreeGPUs(id)); free != 0 { + t.Errorf("node %s has %d free GPUs; its half's allocation was lost", id, free) + } + } + + // And the reconciler must not then place a duplicate alongside it. + h.disp.reset() + h.r.Reconcile(t.Context()) + if got := len(h.disp.startedShards()); got != 0 { + t.Errorf("a duplicate was placed despite the replica being whole (%d shards started)", got) + } +} + +func TestAHalfThatNeverArrivesStillFails(t *testing.T) { + // The grace period must be a deadline, not an amnesty: a replica whose + // other node is simply gone has to fail so it can be replaced. + h := newHarness(t, testModel(1, catalog.Profile{ + Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, + StartupTimeout: time.Minute, + }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) + + s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}) + s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g" + h.r.Adopt("1", []AdoptedShard{s}) + + if reps := h.r.Replicas(); reps[0].State != Starting { + t.Fatalf("state=%s; the half should be waiting for its sibling", reps[0].State) + } + + h.advance(2 * time.Minute) + h.r.Reconcile(t.Context()) + + reps := h.r.Replicas() + if len(reps) == 1 && reps[0].State != Failed { + t.Errorf("state=%s; a half whose sibling never registered must fail", reps[0].State) + } + if free := len(h.inv.FreeGPUs("1")); free != 8 { + t.Errorf("node has %d free GPUs; the abandoned half still holds them", free) + } +} diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index 1031312..528940b 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -91,6 +91,16 @@ type Replica struct { // a two-node replica reported it Ready with one shard and no endpoint, and // the router was handed a replica nothing could route to. WantShards int + // AdoptedAt is set while a replica is being reassembled from what agents + // report, and cleared once every shard has arrived. + // + // A multi-node replica does not arrive at once: each node has its own + // agent, each agent registers separately, and a replica spanning two nodes + // is therefore reported in two halves a second or so apart. Between them it + // is legitimately short of shards, and failing it there condemned it before + // its other half had a chance to register — after which that half was + // stopped as unclaimed and the model rebuilt from nothing on every restart. + AdoptedAt time.Time CreatedAt time.Time ReadyAt time.Time @@ -173,6 +183,11 @@ func (r *Replica) deriveState() { // Both conditions. A replica missing a shard is not ready however healthy // the shards it still has look — and rank 0's absence in particular means // there is no endpoint to route to at all. + if r.WantShards > 0 && len(r.Shards) >= r.WantShards { + // Fully assembled. From here it is an ordinary replica: losing a shard + // now is a failure, not a half that has yet to report. + r.AdoptedAt = time.Time{} + } if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) { if r.State != Ready { r.ReadyAt = time.Now() @@ -181,6 +196,15 @@ func (r *Replica) deriveState() { return } if r.WantShards > 0 && len(r.Shards) < r.WantShards { + // Still assembling from the fleet: the rest of the shards are on nodes + // whose agents have not registered yet. Starting rather than Failed, + // which puts it under the ordinary startup timeout — so a half that + // never arrives is still failed, just by the deadline that already + // exists rather than a second before its sibling reports in. + if !r.AdoptedAt.IsZero() { + r.State = Starting + return + } r.State = Failed if r.Err == "" { r.Err = fmt.Sprintf("replica has %d of %d shards", len(r.Shards), r.WantShards)