Stop reusing replica ids, and stop handing out ports strays still hold

The previous commit's adoption fix made a deeper problem visible instead of
fixing it. Replica ids were the model name and a counter, and the counter
restarted with the process — so a control plane that came back placed a
replica called llama-1 while an engine started before the restart, still
running on a different node, also called itself llama-1.

Two nodes then claimed the same shard id. Adoption handed it back and forth
between them, every placement landed on a port an orphan was still bound to,
and the engine failed to bind, which failed the replica, which placed another
one on the same port. On the real cluster that loop ran for minutes:

    couldn't bind HTTP server socket, hostname: 0.0.0.0, port: 20002

Ids now carry four random bytes, which costs nothing and makes the collision
impossible. The "already known" branch in adoption goes back to meaning what
it says: an agent reconnecting, not a name clash across restarts.

Ports a stray still holds are also reserved now. Stopping is not instant and
an engine keeps its socket until it exits, so the allocator has to know about
sockets no replica claims — otherwise the same loop reappears in the window
between telling an agent to stop something and it actually stopping.

Two of the tests written for the previous commit had quietly become vacuous
under the new ids, asserting things about a literal llama-1 that no longer
exists. They capture the id now.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 09:54:57 +02:00
1 parent 05bdc7d6d4
commit 8a062fec35
2 files changed
+117 -12

No files matched your search

+54 -2
View File
@@ -29,6 +29,9 @@ func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
if len(reps) != 1 || reps[0].State != Starting {
t.Fatalf("expected one starting replica, got %+v", reps)
}
// Captured rather than assumed: replica ids carry a random suffix so a
// restarted control plane cannot collide with engines from its last life.
stuckID := reps[0].ID
// The agent says nothing at all, which is exactly the case.
h.advance(90 * time.Second)
@@ -45,7 +48,7 @@ func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
// point of failing it.
reps = h.r.Replicas()
for _, rep := range reps {
if rep.ID == "llama-1" && rep.State == Starting {
if rep.ID == stuckID && rep.State == Starting {
t.Fatal("the replica is still stuck in starting past its deadline")
}
}
@@ -57,7 +60,7 @@ func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
reps = h.r.Replicas()
var replaced bool
for _, rep := range reps {
if rep.ID != "llama-1" {
if rep.ID != stuckID {
replaced = true
}
}
@@ -178,3 +181,52 @@ func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
t.Error("a stray replaced the replica that was actually serving")
}
}
func TestReplicaIDsDoNotRepeatAcrossRestarts(t *testing.T) {
// The collision that caused the bind-conflict loop on a real cluster. A
// control plane that restarts begins counting again, so the replica it
// places next was called llama-1 — which is also what an engine started
// before the restart, still running on another node, called itself. Two
// nodes claimed the same shard id and every placement landed on a port an
// orphan was already holding.
seen := map[string]bool{}
for i := range 1000 {
// The same counter value every time, which is exactly what a restart
// reproduces.
id := replicaID("llama", 1)
if seen[id] {
t.Fatalf("id %q repeated after %d placements", id, i)
}
seen[id] = true
}
}
func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
// Stopping is not instant, and an engine keeps its socket until it exits.
// Giving its port to a new shard makes that shard fail to bind, which fails
// its replica, which places another one on the same port.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
// An agent reports a stray on the same replica id, holding a different
// port. It is disowned — and its port must not become available.
strayPort := placed.Shards[0].HTTPPort + 2
unclaimed := h.r.Adopt([]AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0",
Rank: 0, HTTPPort: strayPort, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
}})
if len(unclaimed) != 1 {
t.Fatalf("the stray was not disowned: %+v", unclaimed)
}
for range 5 {
if got := h.r.allocatePorts(nil); got == strayPort {
t.Fatalf("port %d was handed out while a stray still holds it", got)
}
}
}