From aef1c143c576e2554aa88bea1b868107fbe1f1fa Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 10:30:49 +0200 Subject: [PATCH] Let go of shards an agent says it is not running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adoption handled "the agent has something we do not know about". The mirror image was missing: "we think the agent has something it does not". Both follow from the same rule — the fleet is authoritative about what is running (plan §6.5) — and without the second half an engine that dies while its agent survives leaves the control plane believing a replica is Ready forever. Measured on the cluster: the engine was killed, the agent restarted and registered with nothing running, and the replica stayed Ready pointing at a dead port for two and a half minutes until I stopped watching. Traffic survived, because the breaker took the endpoint out and retries covered it — 24 requests, none failed — so the only symptom was a cluster quietly running a third short. A registration follows a fresh connection, so anything dispatched over a previous one is not running by definition; there is no window where a shard is legitimately started but unreported. The previous commit's disown rule was too broad, which this turned up: when a known replica had a live shard it disowned everything the agent reported for it, including the shards we already knew. A reconnecting agent would have been told to stop the replica that was serving. It now disowns only what it does not recognise. Co-Authored-By: Claude Opus 5 (1M context) --- internal/agent/agent_test.go | 2 +- internal/control/agentsvc/service.go | 4 +- internal/control/reconciler/reconciler.go | 81 +++++++++++- .../control/reconciler/reconciler_test.go | 16 +-- internal/control/reconciler/stuck_test.go | 116 +++++++++++++++++- 5 files changed, 202 insertions(+), 17 deletions(-) diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 7ab4e4f..e8d85d1 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -53,7 +53,7 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor. } } -func (r *statusRecorder) Adopt(shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { +func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard { r.mu.Lock() defer r.mu.Unlock() r.adopted = append(r.adopted, shards...) diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 0761d2d..25bb703 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -23,7 +23,7 @@ import ( // agents report they are already running. The reconciler implements it. type StatusSink interface { OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string) - Adopt(shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard) + Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard) } // Service is the control plane's AgentService implementation, and at the same @@ -171,7 +171,7 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru if len(shards) == 0 { return } - unclaimed := s.status.Adopt(AdoptedFromRegister(nodeID, shards)) + unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards)) for _, u := range unclaimed { // The model or profile left the catalog while this was running. Nobody // will ever route to it, and it is holding GPUs. diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index 9a43c91..eaf3507 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -700,7 +700,9 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) { // // Shards whose model or profile is no longer in the catalog are returned for // the caller to stop: nothing will ever claim them. -func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) { +func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) { + defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards) + byReplica := map[string][]AdoptedShard{} for _, s := range shards { byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s) @@ -727,8 +729,21 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) { // things really are running and the one we did not ask for is the // stray. if existing.hasLiveShard() { - unclaimed = append(unclaimed, group...) - r.reservePorts(group) + // Only what we do not recognise. A reconnecting agent reports + // the shards we already know, and disowning those would tell it + // to stop the very replica that is serving. + known := map[string]struct{}{} + for _, sh := range existing.Shards { + known[sh.ID] = struct{}{} + } + var strays []AdoptedShard + for _, s := range group { + if _, ok := known[s.ShardID]; !ok { + strays = append(strays, s) + } + } + unclaimed = append(unclaimed, strays...) + r.reservePorts(strays) continue } r.log.Warn("a running replica reported by the fleet replaces one we placed", @@ -786,6 +801,66 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) { return unclaimed } +// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the +// agent, having just told us everything it is running, did not mention. +// +// The mirror image of adoption, and the half that was missing. Adoption handles +// "the agent has something we do not know about"; this handles "we think the +// agent has something it does not". Both follow from the same rule: the fleet is +// authoritative about what is running (plan §6.5). +// +// Without it, an engine that dies while its agent survives leaves the control +// plane believing a replica is Ready forever. Observed on a real cluster: the +// engine was killed, the agent restarted and registered with nothing running, +// and the replica stayed Ready pointing at a dead port for as long as anyone +// watched. Traffic survived — the breaker took the endpoint out — but the +// cluster ran a third short with no sign of it. +// +// Safe against the obvious race. A registration follows a fresh connection, so +// anything we dispatched over a previous one is not running by definition; +// there is no window where a shard is legitimately started but unreported. +func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) { + if nodeID == "" { + return + } + reported := make(map[string]struct{}, len(running)) + for _, s := range running { + reported[s.ShardID] = struct{}{} + } + + var lost []string + r.mu.Lock() + for _, rep := range r.replicas { + if rep.State == Stopped { + continue + } + changed := false + for i := range rep.Shards { + sh := &rep.Shards[i] + if sh.NodeID != nodeID || sh.State.Terminal() { + continue + } + if _, ok := reported[sh.ID]; ok { + continue + } + sh.State = supervisor.Failed + sh.Err = "the agent reports it is not running" + r.inv.Release(sh.ID) + changed = true + lost = append(lost, sh.ID) + } + if changed { + rep.deriveState() + } + } + r.mu.Unlock() + + for _, id := range lost { + r.log.Warn("a shard we believed was running is not; the agent does not have it", + "node", nodeID, "shard", id) + } +} + // reservePorts records ports held by shards nothing owns. // // They are being stopped, but stopping is not instant and the engine keeps its diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index 2bf3bd0..1d087d8 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -624,7 +624,7 @@ func TestAdoptRebuildsReplica(t *testing.T) { // original keeps holding GPUs. h := newHarness(t, testModel(1), gpuNode("1", 8)) - unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) + unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) if len(unclaimed) != 0 { t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed) } @@ -658,7 +658,7 @@ func TestAdoptPreventsDuplicatePlacement(t *testing.T) { // desired count already met. h := newHarness(t, testModel(1), gpuNode("1", 8)) - h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) + h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) h.disp.reset() h.r.Reconcile(t.Context()) @@ -685,7 +685,7 @@ func TestAdoptMultiNodeReplica(t *testing.T) { shards[i].Pool = "ib-a/h100-80g" } - if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 { + if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 { t.Fatalf("unclaimed: %+v", unclaimed) } reps := h.r.Replicas() @@ -713,7 +713,7 @@ func TestAdoptPartialReplicaFails(t *testing.T) { s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}) s.Profile = "2node" s.State = supervisor.Failed - h.r.Adopt([]AdoptedShard{s}) + h.r.Adopt("1", []AdoptedShard{s}) reps := h.r.Replicas() if len(reps) != 1 || reps[0].State != Failed { @@ -735,7 +735,7 @@ func TestAdoptUnknownModelIsUnclaimed(t *testing.T) { s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0}) s.Model = "a-model-that-was-removed" - unclaimed := h.r.Adopt([]AdoptedShard{s}) + unclaimed := h.r.Adopt("1", []AdoptedShard{s}) if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" { t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed) } @@ -750,7 +750,7 @@ func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) { s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0}) s.Profile = "a-profile-that-was-removed" - if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 { + if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 { t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed) } } @@ -761,8 +761,8 @@ func TestAdoptIsIdempotent(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})} - h.r.Adopt(s) - h.r.Adopt(s) + h.r.Adopt("1", s) + h.r.Adopt("1", s) if got := len(h.r.Replicas()); got != 1 { t.Errorf("re-reporting created %d replicas", got) diff --git a/internal/control/reconciler/stuck_test.go b/internal/control/reconciler/stuck_test.go index 07eb64e..0e71f8d 100644 --- a/internal/control/reconciler/stuck_test.go +++ b/internal/control/reconciler/stuck_test.go @@ -134,7 +134,7 @@ func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) { // An agent reports the same replica id, running on the port it was actually // started with. - unclaimed := h.r.Adopt([]AdoptedShard{{ + unclaimed := h.r.Adopt("1", []AdoptedShard{{ NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0", Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000, State: supervisor.Ready, Model: "llama", Profile: "tp4", @@ -168,7 +168,7 @@ func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) { placed := h.r.Replicas()[0] h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "") - unclaimed := h.r.Adopt([]AdoptedShard{{ + unclaimed := h.r.Adopt("1", []AdoptedShard{{ NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard", Rank: 0, HTTPPort: 29999, State: supervisor.Ready, Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999", @@ -215,7 +215,7 @@ func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) { // An agent reports a stray on the same replica id, holding a different // port. It is disowned — and its port must not become available. strayPort := placed.Shards[0].HTTPPort + 2 - unclaimed := h.r.Adopt([]AdoptedShard{{ + unclaimed := h.r.Adopt("1", []AdoptedShard{{ NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0", Rank: 0, HTTPPort: strayPort, State: supervisor.Ready, Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999", @@ -230,3 +230,113 @@ func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) { } } } + +// TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed is the mirror image of +// adoption, and the half that was missing. +// +// Observed on a real cluster: the engine was killed, the agent restarted and +// registered with nothing running, and the replica stayed Ready pointing at a +// dead port for as long as anyone watched. Traffic survived — the breaker took +// the endpoint out — but the cluster ran a third short with no sign of it. +func TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed(t *testing.T) { + h := newHarness(t, testModel(1), gpuNode("1", 8)) + ctx := context.Background() + h.r.Reconcile(ctx) + h.markReady() + + rep := h.r.Replicas()[0] + if rep.State != Ready { + t.Fatalf("state = %s", rep.State) + } + shardID := rep.Shards[0].ID + + // The agent reconnects and says it is running nothing at all. + h.r.Adopt("1", nil) + + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("replicas: %+v", reps) + } + if reps[0].State == Ready { + t.Fatal("a replica whose shard the agent does not have is still Ready") + } + if !strings.Contains(reps[0].Err, "not running") { + t.Errorf("the error does not say what happened: %q", reps[0].Err) + } + + // The GPUs come back, or the replacement has nowhere to go. + if free := len(h.inv.FreeGPUs("1")); free != 8 { + t.Errorf("%d of 8 GPUs are free; the dead shard still holds some", free) + } + _ = shardID + + // And a replacement is placed once the backoff elapses. Several ticks, + // because failing, reaping and placing are separate steps by design. + for range 6 { + h.advance(10 * time.Minute) + h.r.Reconcile(ctx) + h.markReady() + } + h.r.Reconcile(ctx) + + var ready int + for _, r := range h.r.Replicas() { + if r.State == Ready { + ready++ + } + } + if ready != 1 { + t.Errorf("%d ready replicas after the rebuild; want 1", ready) + } +} + +func TestAdoptionAndForgettingDoNotFightEachOther(t *testing.T) { + // The shard the agent does report must survive the same registration that + // forgets the ones it does not. + h := newHarness(t, testModel(1), gpuNode("1", 8)) + ctx := context.Background() + h.r.Reconcile(ctx) + h.markReady() + + rep := h.r.Replicas()[0] + unclaimed := h.r.Adopt("1", []AdoptedShard{{ + NodeID: "1", ReplicaID: rep.ID, ShardID: rep.Shards[0].ID, + Rank: 0, GPUIndices: rep.Shards[0].GPUIndices, HTTPPort: rep.Shards[0].HTTPPort, + State: supervisor.Ready, Model: "llama", Profile: "tp4", + Endpoint: rep.Endpoint, + }}) + if len(unclaimed) != 0 { + t.Fatalf("a shard the agent reported was disowned: %+v", unclaimed) + } + if got := h.r.Replicas()[0].State; got != Ready { + t.Errorf("state = %s; the reported shard was forgotten anyway", got) + } +} + +func TestAnotherNodesShardsAreNotForgotten(t *testing.T) { + // One agent's registration says nothing about any other node. + h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) + ctx := context.Background() + for range 4 { + h.r.Reconcile(ctx) + h.markReady() + h.advance(time.Second) + } + if n := len(h.r.Replicas()); n != 2 { + t.Fatalf("%d replicas", n) + } + + h.r.Adopt("1", nil) + + var onTwo int + for _, rep := range h.r.Replicas() { + for _, s := range rep.Shards { + if s.NodeID == "2" && !s.State.Terminal() { + onTwo++ + } + } + } + if onTwo == 0 { + t.Error("node 1's registration forgot node 2's shards") + } +}