Adoption handled "the agent has something we do not know about". The mirror image was missing: "we think the agent has something it does not". Both follow from the same rule — the fleet is authoritative about what is running (plan §6.5) — and without the second half an engine that dies while its agent survives leaves the control plane believing a replica is Ready forever. Measured on the cluster: the engine was killed, the agent restarted and registered with nothing running, and the replica stayed Ready pointing at a dead port for two and a half minutes until I stopped watching. Traffic survived, because the breaker took the endpoint out and retries covered it — 24 requests, none failed — so the only symptom was a cluster quietly running a third short. A registration follows a fresh connection, so anything dispatched over a previous one is not running by definition; there is no window where a shard is legitimately started but unreported. The previous commit's disown rule was too broad, which this turned up: when a known replica had a live shard it disowned everything the agent reported for it, including the shards we already knew. A reconnecting agent would have been told to stop the replica that was serving. It now disowns only what it does not recognise. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
555e4678cb
commit
aef1c143c5
5 files changed
+202
-17
No files matched your search
@@ -53,7 +53,7 @@ func (r *statusRecorder) OnShardStatus(replicaID, shardID string, st supervisor.
|
||||
}
|
||||
}
|
||||
|
||||
func (r *statusRecorder) Adopt(shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
||||
func (r *statusRecorder) Adopt(_ string, shards []reconciler.AdoptedShard) []reconciler.AdoptedShard {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.adopted = append(r.adopted, shards...)
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
// agents report they are already running. The reconciler implements it.
|
||||
type StatusSink interface {
|
||||
OnShardStatus(replicaID, shardID string, state supervisor.State, errMsg string)
|
||||
Adopt(shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
||||
Adopt(nodeID string, shards []reconciler.AdoptedShard) (unclaimed []reconciler.AdoptedShard)
|
||||
}
|
||||
|
||||
// Service is the control plane's AgentService implementation, and at the same
|
||||
@@ -171,7 +171,7 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
|
||||
if len(shards) == 0 {
|
||||
return
|
||||
}
|
||||
unclaimed := s.status.Adopt(AdoptedFromRegister(nodeID, shards))
|
||||
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
|
||||
for _, u := range unclaimed {
|
||||
// The model or profile left the catalog while this was running. Nobody
|
||||
// will ever route to it, and it is holding GPUs.
|
||||
|
||||
@@ -700,7 +700,9 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
||||
//
|
||||
// Shards whose model or profile is no longer in the catalog are returned for
|
||||
// the caller to stop: nothing will ever claim them.
|
||||
func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
||||
func (r *Reconciler) Adopt(nodeID string, shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
||||
defer r.forgetShardsTheAgentDoesNotHave(nodeID, shards)
|
||||
|
||||
byReplica := map[string][]AdoptedShard{}
|
||||
for _, s := range shards {
|
||||
byReplica[s.ReplicaID] = append(byReplica[s.ReplicaID], s)
|
||||
@@ -727,8 +729,21 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
||||
// things really are running and the one we did not ask for is the
|
||||
// stray.
|
||||
if existing.hasLiveShard() {
|
||||
unclaimed = append(unclaimed, group...)
|
||||
r.reservePorts(group)
|
||||
// Only what we do not recognise. A reconnecting agent reports
|
||||
// the shards we already know, and disowning those would tell it
|
||||
// to stop the very replica that is serving.
|
||||
known := map[string]struct{}{}
|
||||
for _, sh := range existing.Shards {
|
||||
known[sh.ID] = struct{}{}
|
||||
}
|
||||
var strays []AdoptedShard
|
||||
for _, s := range group {
|
||||
if _, ok := known[s.ShardID]; !ok {
|
||||
strays = append(strays, s)
|
||||
}
|
||||
}
|
||||
unclaimed = append(unclaimed, strays...)
|
||||
r.reservePorts(strays)
|
||||
continue
|
||||
}
|
||||
r.log.Warn("a running replica reported by the fleet replaces one we placed",
|
||||
@@ -786,6 +801,66 @@ func (r *Reconciler) Adopt(shards []AdoptedShard) (unclaimed []AdoptedShard) {
|
||||
return unclaimed
|
||||
}
|
||||
|
||||
// forgetShardsTheAgentDoesNotHave fails shards we believe are on a node that the
|
||||
// agent, having just told us everything it is running, did not mention.
|
||||
//
|
||||
// The mirror image of adoption, and the half that was missing. Adoption handles
|
||||
// "the agent has something we do not know about"; this handles "we think the
|
||||
// agent has something it does not". Both follow from the same rule: the fleet is
|
||||
// authoritative about what is running (plan §6.5).
|
||||
//
|
||||
// Without it, an engine that dies while its agent survives leaves the control
|
||||
// plane believing a replica is Ready forever. Observed on a real cluster: the
|
||||
// engine was killed, the agent restarted and registered with nothing running,
|
||||
// and the replica stayed Ready pointing at a dead port for as long as anyone
|
||||
// watched. Traffic survived — the breaker took the endpoint out — but the
|
||||
// cluster ran a third short with no sign of it.
|
||||
//
|
||||
// Safe against the obvious race. A registration follows a fresh connection, so
|
||||
// anything we dispatched over a previous one is not running by definition;
|
||||
// there is no window where a shard is legitimately started but unreported.
|
||||
func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []AdoptedShard) {
|
||||
if nodeID == "" {
|
||||
return
|
||||
}
|
||||
reported := make(map[string]struct{}, len(running))
|
||||
for _, s := range running {
|
||||
reported[s.ShardID] = struct{}{}
|
||||
}
|
||||
|
||||
var lost []string
|
||||
r.mu.Lock()
|
||||
for _, rep := range r.replicas {
|
||||
if rep.State == Stopped {
|
||||
continue
|
||||
}
|
||||
changed := false
|
||||
for i := range rep.Shards {
|
||||
sh := &rep.Shards[i]
|
||||
if sh.NodeID != nodeID || sh.State.Terminal() {
|
||||
continue
|
||||
}
|
||||
if _, ok := reported[sh.ID]; ok {
|
||||
continue
|
||||
}
|
||||
sh.State = supervisor.Failed
|
||||
sh.Err = "the agent reports it is not running"
|
||||
r.inv.Release(sh.ID)
|
||||
changed = true
|
||||
lost = append(lost, sh.ID)
|
||||
}
|
||||
if changed {
|
||||
rep.deriveState()
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, id := range lost {
|
||||
r.log.Warn("a shard we believed was running is not; the agent does not have it",
|
||||
"node", nodeID, "shard", id)
|
||||
}
|
||||
}
|
||||
|
||||
// reservePorts records ports held by shards nothing owns.
|
||||
//
|
||||
// They are being stopped, but stopping is not instant and the engine keeps its
|
||||
|
||||
@@ -624,7 +624,7 @@ func TestAdoptRebuildsReplica(t *testing.T) {
|
||||
// original keeps holding GPUs.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
|
||||
unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
||||
unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
||||
if len(unclaimed) != 0 {
|
||||
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
|
||||
}
|
||||
@@ -658,7 +658,7 @@ func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
|
||||
// desired count already met.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
|
||||
h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
||||
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
||||
h.disp.reset()
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
@@ -685,7 +685,7 @@ func TestAdoptMultiNodeReplica(t *testing.T) {
|
||||
shards[i].Pool = "ib-a/h100-80g"
|
||||
}
|
||||
|
||||
if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 {
|
||||
if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 {
|
||||
t.Fatalf("unclaimed: %+v", unclaimed)
|
||||
}
|
||||
reps := h.r.Replicas()
|
||||
@@ -713,7 +713,7 @@ func TestAdoptPartialReplicaFails(t *testing.T) {
|
||||
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
||||
s.Profile = "2node"
|
||||
s.State = supervisor.Failed
|
||||
h.r.Adopt([]AdoptedShard{s})
|
||||
h.r.Adopt("1", []AdoptedShard{s})
|
||||
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 || reps[0].State != Failed {
|
||||
@@ -735,7 +735,7 @@ func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
|
||||
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
|
||||
s.Model = "a-model-that-was-removed"
|
||||
|
||||
unclaimed := h.r.Adopt([]AdoptedShard{s})
|
||||
unclaimed := h.r.Adopt("1", []AdoptedShard{s})
|
||||
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
|
||||
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
|
||||
}
|
||||
@@ -750,7 +750,7 @@ func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
|
||||
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
|
||||
s.Profile = "a-profile-that-was-removed"
|
||||
|
||||
if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 {
|
||||
if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 {
|
||||
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
|
||||
}
|
||||
}
|
||||
@@ -761,8 +761,8 @@ func TestAdoptIsIdempotent(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
|
||||
|
||||
h.r.Adopt(s)
|
||||
h.r.Adopt(s)
|
||||
h.r.Adopt("1", s)
|
||||
h.r.Adopt("1", s)
|
||||
|
||||
if got := len(h.r.Replicas()); got != 1 {
|
||||
t.Errorf("re-reporting created %d replicas", got)
|
||||
|
||||
@@ -134,7 +134,7 @@ func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) {
|
||||
|
||||
// An agent reports the same replica id, running on the port it was actually
|
||||
// started with.
|
||||
unclaimed := h.r.Adopt([]AdoptedShard{{
|
||||
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
||||
NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0",
|
||||
Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000,
|
||||
State: supervisor.Ready, Model: "llama", Profile: "tp4",
|
||||
@@ -168,7 +168,7 @@ func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
|
||||
placed := h.r.Replicas()[0]
|
||||
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
|
||||
|
||||
unclaimed := h.r.Adopt([]AdoptedShard{{
|
||||
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
||||
NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard",
|
||||
Rank: 0, HTTPPort: 29999, State: supervisor.Ready,
|
||||
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
|
||||
@@ -215,7 +215,7 @@ func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
|
||||
// An agent reports a stray on the same replica id, holding a different
|
||||
// port. It is disowned — and its port must not become available.
|
||||
strayPort := placed.Shards[0].HTTPPort + 2
|
||||
unclaimed := h.r.Adopt([]AdoptedShard{{
|
||||
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
||||
NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0",
|
||||
Rank: 0, HTTPPort: strayPort, State: supervisor.Ready,
|
||||
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
|
||||
@@ -230,3 +230,113 @@ func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed is the mirror image of
|
||||
// adoption, and the half that was missing.
|
||||
//
|
||||
// Observed on a real cluster: the engine was killed, the agent restarted and
|
||||
// registered with nothing running, and the replica stayed Ready pointing at a
|
||||
// dead port for as long as anyone watched. Traffic survived — the breaker took
|
||||
// the endpoint out — but the cluster ran a third short with no sign of it.
|
||||
func TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
ctx := context.Background()
|
||||
h.r.Reconcile(ctx)
|
||||
h.markReady()
|
||||
|
||||
rep := h.r.Replicas()[0]
|
||||
if rep.State != Ready {
|
||||
t.Fatalf("state = %s", rep.State)
|
||||
}
|
||||
shardID := rep.Shards[0].ID
|
||||
|
||||
// The agent reconnects and says it is running nothing at all.
|
||||
h.r.Adopt("1", nil)
|
||||
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("replicas: %+v", reps)
|
||||
}
|
||||
if reps[0].State == Ready {
|
||||
t.Fatal("a replica whose shard the agent does not have is still Ready")
|
||||
}
|
||||
if !strings.Contains(reps[0].Err, "not running") {
|
||||
t.Errorf("the error does not say what happened: %q", reps[0].Err)
|
||||
}
|
||||
|
||||
// The GPUs come back, or the replacement has nowhere to go.
|
||||
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
||||
t.Errorf("%d of 8 GPUs are free; the dead shard still holds some", free)
|
||||
}
|
||||
_ = shardID
|
||||
|
||||
// And a replacement is placed once the backoff elapses. Several ticks,
|
||||
// because failing, reaping and placing are separate steps by design.
|
||||
for range 6 {
|
||||
h.advance(10 * time.Minute)
|
||||
h.r.Reconcile(ctx)
|
||||
h.markReady()
|
||||
}
|
||||
h.r.Reconcile(ctx)
|
||||
|
||||
var ready int
|
||||
for _, r := range h.r.Replicas() {
|
||||
if r.State == Ready {
|
||||
ready++
|
||||
}
|
||||
}
|
||||
if ready != 1 {
|
||||
t.Errorf("%d ready replicas after the rebuild; want 1", ready)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdoptionAndForgettingDoNotFightEachOther(t *testing.T) {
|
||||
// The shard the agent does report must survive the same registration that
|
||||
// forgets the ones it does not.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
ctx := context.Background()
|
||||
h.r.Reconcile(ctx)
|
||||
h.markReady()
|
||||
|
||||
rep := h.r.Replicas()[0]
|
||||
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
||||
NodeID: "1", ReplicaID: rep.ID, ShardID: rep.Shards[0].ID,
|
||||
Rank: 0, GPUIndices: rep.Shards[0].GPUIndices, HTTPPort: rep.Shards[0].HTTPPort,
|
||||
State: supervisor.Ready, Model: "llama", Profile: "tp4",
|
||||
Endpoint: rep.Endpoint,
|
||||
}})
|
||||
if len(unclaimed) != 0 {
|
||||
t.Fatalf("a shard the agent reported was disowned: %+v", unclaimed)
|
||||
}
|
||||
if got := h.r.Replicas()[0].State; got != Ready {
|
||||
t.Errorf("state = %s; the reported shard was forgotten anyway", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnotherNodesShardsAreNotForgotten(t *testing.T) {
|
||||
// One agent's registration says nothing about any other node.
|
||||
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
||||
ctx := context.Background()
|
||||
for range 4 {
|
||||
h.r.Reconcile(ctx)
|
||||
h.markReady()
|
||||
h.advance(time.Second)
|
||||
}
|
||||
if n := len(h.r.Replicas()); n != 2 {
|
||||
t.Fatalf("%d replicas", n)
|
||||
}
|
||||
|
||||
h.r.Adopt("1", nil)
|
||||
|
||||
var onTwo int
|
||||
for _, rep := range h.r.Replicas() {
|
||||
for _, s := range rep.Shards {
|
||||
if s.NodeID == "2" && !s.State.Terminal() {
|
||||
onTwo++
|
||||
}
|
||||
}
|
||||
}
|
||||
if onTwo == 0 {
|
||||
t.Error("node 1's registration forgot node 2's shards")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user