Let go of shards an agent says it is not running

Adoption handled "the agent has something we do not know about". The mirror
image was missing: "we think the agent has something it does not".

Both follow from the same rule — the fleet is authoritative about what is
running (plan §6.5) — and without the second half an engine that dies while
its agent survives leaves the control plane believing a replica is Ready
forever. Measured on the cluster: the engine was killed, the agent restarted
and registered with nothing running, and the replica stayed Ready pointing at
a dead port for two and a half minutes until I stopped watching. Traffic
survived, because the breaker took the endpoint out and retries covered it —
24 requests, none failed — so the only symptom was a cluster quietly running a
third short.

A registration follows a fresh connection, so anything dispatched over a
previous one is not running by definition; there is no window where a shard is
legitimately started but unreported.

The previous commit's disown rule was too broad, which this turned up: when a
known replica had a live shard it disowned everything the agent reported for
it, including the shards we already knew. A reconnecting agent would have been
told to stop the replica that was serving. It now disowns only what it does
not recognise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 10:30:49 +02:00
1 parent 555e4678cb
commit aef1c143c5
5 files changed
+202 -17

No files matched your search

+1 -1
View File
@@ -53,7 +53,7 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
}
}
func (r *statusRecorder) Adopt(shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
r.mu.Lock()
defer r.mu.Unlock()
r.adopted = append(r.adopted, shards...)
+2 -2
View File
@@ -23,7 +23,7 @@ import (
// agents report they are already running. The reconciler implements it.
type StatusSink interface {
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
Adopt(shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
}
// Service is the control plane's AgentService implementation, and at the same
@@ -171,7 +171,7 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
if len(shards) == 0 {
return
}
unclaimed := s.status.Adopt(AdoptedFromRegister(nodeID, shards))
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
+78 -3
View File
@@ -700,7 +700,9 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
//
// Shards whose model or profile is no longer in the catalog are returned for
// the caller to stop: nothing will ever claim them.
func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) {
defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards)
byReplica := map[string][]AdoptedShard{}
for _, s := range shards {
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
@@ -727,8 +729,21 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
// things really are running and the one we did not ask for is the
// stray.
if existing.hasLiveShard() {
unclaimed = append(unclaimed, group...)
r.reservePorts(group)
// Only what we do not recognise. A reconnecting agent reports
// the shards we already know, and disowning those would tell it
// to stop the very replica that is serving.
known := map[string]struct{}{}
for _, sh := range existing.Shards {
known[sh.ID] = struct{}{}
}
var strays []AdoptedShard
for _, s := range group {
if _, ok := known[s.ShardID]; !ok {
strays = append(strays, s)
}
}
unclaimed = append(unclaimed, strays...)
r.reservePorts(strays)
continue
}
r.log.Warn("a running replica reported by the fleet replaces one we placed",
@@ -786,6 +801,66 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
return unclaimed
}
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
// agent, having just told us everything it is running, did not mention.
//
// The mirror image of adoption, and the half that was missing. Adoption handles
// "the agent has something we do not know about"; this handles "we think the
// agent has something it does not". Both follow from the same rule: the fleet is
// authoritative about what is running (plan §6.5).
//
// Without it, an engine that dies while its agent survives leaves the control
// plane believing a replica is Ready forever. Observed on a real cluster: the
// engine was killed, the agent restarted and registered with nothing running,
// and the replica stayed Ready pointing at a dead port for as long as anyone
// watched. Traffic survived — the breaker took the endpoint out — but the
// cluster ran a third short with no sign of it.
//
// Safe against the obvious race. A registration follows a fresh connection, so
// anything we dispatched over a previous one is not running by definition;
// there is no window where a shard is legitimately started but unreported.
func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) {
if nodeID == "" {
return
}
reported := make(map[string]struct{}, len(running))
for _, s := range running {
reported[s.ShardID] = struct{}{}
}
var lost []string
r.mu.Lock()
for _, rep := range r.replicas {
if rep.State == Stopped {
continue
}
changed := false
for i := range rep.Shards {
sh := &rep.Shards[i]
if sh.NodeID != nodeID || sh.State.Terminal() {
continue
}
if _, ok := reported[sh.ID]; ok {
continue
}
sh.State = supervisor.Failed
sh.Err = "the agent reports it is not running"
r.inv.Release(sh.ID)
changed = true
lost = append(lost, sh.ID)
}
if changed {
rep.deriveState()
}
}
r.mu.Unlock()
for _, id := range lost {
r.log.Warn("a shard we believed was running is not; the agent does not have it",
"node", nodeID, "shard", id)
}
}
// reservePorts records ports held by shards nothing owns.
//
// They are being stopped, but stopping is not instant and the engine keeps its
@@ -624,7 +624,7 @@ func TestAdoptRebuildsReplica(t *testing.T) {
// original keeps holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
if len(unclaimed) != 0 {
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
}
@@ -658,7 +658,7 @@ func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
// desired count already met.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.disp.reset()
h.r.Reconcile(t.Context())
@@ -685,7 +685,7 @@ func TestAdoptMultiNodeReplica(t *testing.T) {
shards[i].Pool = "ib-a/h100-80g"
}
if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 {
if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 {
t.Fatalf("unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
@@ -713,7 +713,7 @@ func TestAdoptPartialReplicaFails(t *testing.T) {
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
s.Profile = "2node"
s.State = supervisor.Failed
h.r.Adopt([]AdoptedShard{s})
h.r.Adopt("1", []AdoptedShard{s})
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
@@ -735,7 +735,7 @@ func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
s.Model = "a-model-that-was-removed"
unclaimed := h.r.Adopt([]AdoptedShard{s})
unclaimed := h.r.Adopt("1", []AdoptedShard{s})
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
}
@@ -750,7 +750,7 @@ func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
s.Profile = "a-profile-that-was-removed"
if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 {
if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 {
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
}
}
@@ -761,8 +761,8 @@ func TestAdoptIsIdempotent(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
h.r.Adopt(s)
h.r.Adopt(s)
h.r.Adopt("1", s)
h.r.Adopt("1", s)
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("re-reporting created %d replicas", got)
+113 -3
View File
@@ -134,7 +134,7 @@ func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) {
// An agent reports the same replica id, running on the port it was actually
// started with.
unclaimed := h.r.Adopt([]AdoptedShard{{
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0",
Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000,
State: supervisor.Ready, Model: "llama", Profile: "tp4",
@@ -168,7 +168,7 @@ func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
unclaimed := h.r.Adopt([]AdoptedShard{{
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard",
Rank: 0, HTTPPort: 29999, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
@@ -215,7 +215,7 @@ func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
// An agent reports a stray on the same replica id, holding a different
// port. It is disowned — and its port must not become available.
strayPort := placed.Shards[0].HTTPPort + 2
unclaimed := h.r.Adopt([]AdoptedShard{{
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0",
Rank: 0, HTTPPort: strayPort, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
@@ -230,3 +230,113 @@ func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
}
}
}
// TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed is the mirror image of
// adoption, and the half that was missing.
//
// Observed on a real cluster: the engine was killed, the agent restarted and
// registered with nothing running, and the replica stayed Ready pointing at a
// dead port for as long as anyone watched. Traffic survived — the breaker took
// the endpoint out — but the cluster ran a third short with no sign of it.
func TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.markReady()
rep := h.r.Replicas()[0]
if rep.State != Ready {
t.Fatalf("state = %s", rep.State)
}
shardID := rep.Shards[0].ID
// The agent reconnects and says it is running nothing at all.
h.r.Adopt("1", nil)
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("replicas: %+v", reps)
}
if reps[0].State == Ready {
t.Fatal("a replica whose shard the agent does not have is still Ready")
}
if !strings.Contains(reps[0].Err, "not running") {
t.Errorf("the error does not say what happened: %q", reps[0].Err)
}
// The GPUs come back, or the replacement has nowhere to go.
if free := len(h.inv.FreeGPUs("1")); free != 8 {
t.Errorf("%d of 8 GPUs are free; the dead shard still holds some", free)
}
_ = shardID
// And a replacement is placed once the backoff elapses. Several ticks,
// because failing, reaping and placing are separate steps by design.
for range 6 {
h.advance(10 * time.Minute)
h.r.Reconcile(ctx)
h.markReady()
}
h.r.Reconcile(ctx)
var ready int
for _, r := range h.r.Replicas() {
if r.State == Ready {
ready++
}
}
if ready != 1 {
t.Errorf("%d ready replicas after the rebuild; want 1", ready)
}
}
func TestAdoptionAndForgettingDoNotFightEachOther(t *testing.T) {
// The shard the agent does report must survive the same registration that
// forgets the ones it does not.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.markReady()
rep := h.r.Replicas()[0]
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: rep.ID, ShardID: rep.Shards[0].ID,
Rank: 0, GPUIndices: rep.Shards[0].GPUIndices, HTTPPort: rep.Shards[0].HTTPPort,
State: supervisor.Ready, Model: "llama", Profile: "tp4",
Endpoint: rep.Endpoint,
}})
if len(unclaimed) != 0 {
t.Fatalf("a shard the agent reported was disowned: %+v", unclaimed)
}
if got := h.r.Replicas()[0].State; got != Ready {
t.Errorf("state = %s; the reported shard was forgotten anyway", got)
}
}
func TestAnotherNodesShardsAreNotForgotten(t *testing.T) {
// One agent's registration says nothing about any other node.
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
ctx := context.Background()
for range 4 {
h.r.Reconcile(ctx)
h.markReady()
h.advance(time.Second)
}
if n := len(h.r.Replicas()); n != 2 {
t.Fatalf("%d replicas", n)
}
h.r.Adopt("1", nil)
var onTwo int
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
if s.NodeID == "2" && !s.State.Terminal() {
onTwo++
}
}
}
if onTwo == 0 {
t.Error("node 1's registration forgot node 2's shards")
}
}