Files
llmgw/internal/control/reconciler/stuck_test.go
T
Ilker ManapandClaude Opus 5 8a062fec35 Stop reusing replica ids, and stop handing out ports strays still hold
The previous commit's adoption fix made a deeper problem visible instead of
fixing it. Replica ids were the model name and a counter, and the counter
restarted with the process — so a control plane that came back placed a
replica called llama-1 while an engine started before the restart, still
running on a different node, also called itself llama-1.

Two nodes then claimed the same shard id. Adoption handed it back and forth
between them, every placement landed on a port an orphan was still bound to,
and the engine failed to bind, which failed the replica, which placed another
one on the same port. On the real cluster that loop ran for minutes:

    couldn't bind HTTP server socket, hostname: 0.0.0.0, port: 20002

Ids now carry four random bytes, which costs nothing and makes the collision
impossible. The "already known" branch in adoption goes back to meaning what
it says: an agent reconnecting, not a name clash across restarts.

Ports a stray still holds are also reserved now. Stopping is not instant and
an engine keeps its socket until it exits, so the allocator has to know about
sockets no replica claims — otherwise the same loop reappears in the window
between telling an agent to stop something and it actually stopping.

Two of the tests written for the previous commit had quietly become vacuous
under the new ids, asserting things about a literal llama-1 that no longer
exists. They capture the id now.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 09:54:57 +02:00

233 lines
7.9 KiB
Go

package reconciler
import (
"context"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// TestAReplicaThatNeverComesUpIsEventuallyFailed pins down the hole found on a
// real cluster: two replicas sat in Starting for eight minutes past a two-minute
// startup timeout, holding GPUs and counting towards the model's replica count
// while serving nothing.
//
// Every other way out of Starting needs the agent to report something. When it
// never does, this is the only thing that moves.
func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 2 * time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Starting {
t.Fatalf("expected one starting replica, got %+v", reps)
}
// Captured rather than assumed: replica ids carry a random suffix so a
// restarted control plane cannot collide with engines from its last life.
stuckID := reps[0].ID
// The agent says nothing at all, which is exactly the case.
h.advance(90 * time.Second)
h.r.Reconcile(ctx)
if got := h.r.Replicas()[0].State; got != Starting {
t.Fatalf("failed early, at %s: %s", 90*time.Second, got)
}
h.advance(60 * time.Second)
h.r.Reconcile(ctx)
// Failing it is not the outcome that matters; being unstuck is. The same
// reconcile reaps the failure and places a replacement, which is the whole
// point of failing it.
reps = h.r.Replicas()
for _, rep := range reps {
if rep.ID == stuckID && rep.State == Starting {
t.Fatal("the replica is still stuck in starting past its deadline")
}
}
// The replacement waits out the model's backoff, which is the point of the
// backoff: a model that cannot start should not be restarted in a loop.
h.advance(10 * time.Minute)
h.r.Reconcile(ctx)
reps = h.r.Replicas()
var replaced bool
for _, rep := range reps {
if rep.ID != stuckID {
replaced = true
}
}
if !replaced {
t.Errorf("no replacement was ever placed; replicas are %+v", reps)
}
// And the shards of the stuck one were told to stop, or its GPUs stay
// allocated to a replica nobody is waiting for any more.
h.disp.mu.Lock()
stopped := len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the stuck replica's shards were never stopped")
}
}
func TestTheStuckReplicaSaysWhyItFailed(t *testing.T) {
// Reaping removes the evidence, so the reason has to be recorded when the
// deadline passes rather than found afterwards.
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
h.r.Reconcile(context.Background())
h.advance(2 * time.Minute)
h.r.expireStuckReplicas()
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("replicas: %+v", reps)
}
if !strings.Contains(reps[0].Err, "did not come up") {
t.Errorf("the error does not say what happened: %q", reps[0].Err)
}
}
func TestAReplicaThatIsLoadingIsNotFailedEarly(t *testing.T) {
// Loading a large model legitimately takes a long time. The deadline is
// what the profile says, and nothing shorter.
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Hour,
})
h := newHarness(t, m, gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.advance(30 * time.Minute)
h.r.Reconcile(ctx)
if got := h.r.Replicas()[0].State; got == Failed {
t.Error("a replica still inside its startup timeout was failed")
}
}
// TestTheFleetWinsOverAReplicaWeOnlyPlaced is the other half of the same
// incident. Replica ids come from the model name and a counter, so a control
// plane that restarts and places before its agents reconnect builds a different
// replica under the same id, on a freshly allocated port. Dropping the agent's
// report then leaves the cluster pointing at a port nothing listens on while the
// engine that is actually serving is treated as a stray.
func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
if placed.State != Starting {
t.Fatalf("state = %s", placed.State)
}
// An agent reports the same replica id, running on the port it was actually
// started with.
unclaimed := h.r.Adopt([]AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0",
Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000,
State: supervisor.Ready, Model: "llama", Profile: "tp4",
Endpoint: "http://node1:20000",
}})
if len(unclaimed) != 0 {
t.Fatalf("the running engine was disowned: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("replicas: %+v", reps)
}
if reps[0].Endpoint != "http://node1:20000" {
t.Errorf("endpoint = %q; the cluster is still pointing at what we placed",
reps[0].Endpoint)
}
if reps[0].State != Ready {
t.Errorf("state = %s, want ready", reps[0].State)
}
}
func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
// The other direction. If our copy has a shard that has come up, two things
// really are running and the one we did not ask for is the stray — the
// agent is told to stop it rather than the live one being replaced.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
unclaimed := h.r.Adopt([]AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard",
Rank: 0, HTTPPort: 29999, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
}})
if len(unclaimed) != 1 {
t.Fatalf("the stray was not disowned: %+v", unclaimed)
}
if got := h.r.Replicas()[0].Endpoint; got == "http://node1:29999" {
t.Error("a stray replaced the replica that was actually serving")
}
}
func TestReplicaIDsDoNotRepeatAcrossRestarts(t *testing.T) {
// The collision that caused the bind-conflict loop on a real cluster. A
// control plane that restarts begins counting again, so the replica it
// places next was called llama-1 — which is also what an engine started
// before the restart, still running on another node, called itself. Two
// nodes claimed the same shard id and every placement landed on a port an
// orphan was already holding.
seen := map[string]bool{}
for i := range 1000 {
// The same counter value every time, which is exactly what a restart
// reproduces.
id := replicaID("llama", 1)
if seen[id] {
t.Fatalf("id %q repeated after %d placements", id, i)
}
seen[id] = true
}
}
func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
// Stopping is not instant, and an engine keeps its socket until it exits.
// Giving its port to a new shard makes that shard fail to bind, which fails
// its replica, which places another one on the same port.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
// An agent reports a stray on the same replica id, holding a different
// port. It is disowned — and its port must not become available.
strayPort := placed.Shards[0].HTTPPort + 2
unclaimed := h.r.Adopt([]AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0",
Rank: 0, HTTPPort: strayPort, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
}})
if len(unclaimed) != 1 {
t.Fatalf("the stray was not disowned: %+v", unclaimed)
}
for range 5 {
if got := h.r.allocatePorts(nil); got == strayPort {
t.Fatalf("port %d was handed out while a stray still holds it", got)
}
}
}