Files
llmgw/internal/control/reconciler/stuck_test.go
T
Ilker ManapandClaude Opus 5 aef1c143c5
CI / test (push) Successful in 1m19s
Let go of shards an agent says it is not running
Adoption handled "the agent has something we do not know about". The mirror
image was missing: "we think the agent has something it does not".

Both follow from the same rule — the fleet is authoritative about what is
running (plan §6.5) — and without the second half an engine that dies while
its agent survives leaves the control plane believing a replica is Ready
forever. Measured on the cluster: the engine was killed, the agent restarted
and registered with nothing running, and the replica stayed Ready pointing at
a dead port for two and a half minutes until I stopped watching. Traffic
survived, because the breaker took the endpoint out and retries covered it —
24 requests, none failed — so the only symptom was a cluster quietly running a
third short.

A registration follows a fresh connection, so anything dispatched over a
previous one is not running by definition; there is no window where a shard is
legitimately started but unreported.

The previous commit's disown rule was too broad, which this turned up: when a
known replica had a live shard it disowned everything the agent reported for
it, including the shards we already knew. A reconnecting agent would have been
told to stop the replica that was serving. It now disowns only what it does
not recognise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 10:30:49 +02:00

343 lines
11 KiB
Go

package reconciler
import (
"context"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// TestAReplicaThatNeverComesUpIsEventuallyFailed pins down the hole found on a
// real cluster: two replicas sat in Starting for eight minutes past a two-minute
// startup timeout, holding GPUs and counting towards the model's replica count
// while serving nothing.
//
// Every other way out of Starting needs the agent to report something. When it
// never does, this is the only thing that moves.
func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 2 * time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Starting {
t.Fatalf("expected one starting replica, got %+v", reps)
}
// Captured rather than assumed: replica ids carry a random suffix so a
// restarted control plane cannot collide with engines from its last life.
stuckID := reps[0].ID
// The agent says nothing at all, which is exactly the case.
h.advance(90 * time.Second)
h.r.Reconcile(ctx)
if got := h.r.Replicas()[0].State; got != Starting {
t.Fatalf("failed early, at %s: %s", 90*time.Second, got)
}
h.advance(60 * time.Second)
h.r.Reconcile(ctx)
// Failing it is not the outcome that matters; being unstuck is. The same
// reconcile reaps the failure and places a replacement, which is the whole
// point of failing it.
reps = h.r.Replicas()
for _, rep := range reps {
if rep.ID == stuckID && rep.State == Starting {
t.Fatal("the replica is still stuck in starting past its deadline")
}
}
// The replacement waits out the model's backoff, which is the point of the
// backoff: a model that cannot start should not be restarted in a loop.
h.advance(10 * time.Minute)
h.r.Reconcile(ctx)
reps = h.r.Replicas()
var replaced bool
for _, rep := range reps {
if rep.ID != stuckID {
replaced = true
}
}
if !replaced {
t.Errorf("no replacement was ever placed; replicas are %+v", reps)
}
// And the shards of the stuck one were told to stop, or its GPUs stay
// allocated to a replica nobody is waiting for any more.
h.disp.mu.Lock()
stopped := len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the stuck replica's shards were never stopped")
}
}
func TestTheStuckReplicaSaysWhyItFailed(t *testing.T) {
// Reaping removes the evidence, so the reason has to be recorded when the
// deadline passes rather than found afterwards.
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
h.r.Reconcile(context.Background())
h.advance(2 * time.Minute)
h.r.expireStuckReplicas()
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("replicas: %+v", reps)
}
if !strings.Contains(reps[0].Err, "did not come up") {
t.Errorf("the error does not say what happened: %q", reps[0].Err)
}
}
func TestAReplicaThatIsLoadingIsNotFailedEarly(t *testing.T) {
// Loading a large model legitimately takes a long time. The deadline is
// what the profile says, and nothing shorter.
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Hour,
})
h := newHarness(t, m, gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.advance(30 * time.Minute)
h.r.Reconcile(ctx)
if got := h.r.Replicas()[0].State; got == Failed {
t.Error("a replica still inside its startup timeout was failed")
}
}
// TestTheFleetWinsOverAReplicaWeOnlyPlaced is the other half of the same
// incident. Replica ids come from the model name and a counter, so a control
// plane that restarts and places before its agents reconnect builds a different
// replica under the same id, on a freshly allocated port. Dropping the agent's
// report then leaves the cluster pointing at a port nothing listens on while the
// engine that is actually serving is treated as a stray.
func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
if placed.State != Starting {
t.Fatalf("state = %s", placed.State)
}
// An agent reports the same replica id, running on the port it was actually
// started with.
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0",
Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000,
State: supervisor.Ready, Model: "llama", Profile: "tp4",
Endpoint: "http://node1:20000",
}})
if len(unclaimed) != 0 {
t.Fatalf("the running engine was disowned: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("replicas: %+v", reps)
}
if reps[0].Endpoint != "http://node1:20000" {
t.Errorf("endpoint = %q; the cluster is still pointing at what we placed",
reps[0].Endpoint)
}
if reps[0].State != Ready {
t.Errorf("state = %s, want ready", reps[0].State)
}
}
func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
// The other direction. If our copy has a shard that has come up, two things
// really are running and the one we did not ask for is the stray — the
// agent is told to stop it rather than the live one being replaced.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard",
Rank: 0, HTTPPort: 29999, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
}})
if len(unclaimed) != 1 {
t.Fatalf("the stray was not disowned: %+v", unclaimed)
}
if got := h.r.Replicas()[0].Endpoint; got == "http://node1:29999" {
t.Error("a stray replaced the replica that was actually serving")
}
}
func TestReplicaIDsDoNotRepeatAcrossRestarts(t *testing.T) {
// The collision that caused the bind-conflict loop on a real cluster. A
// control plane that restarts begins counting again, so the replica it
// places next was called llama-1 — which is also what an engine started
// before the restart, still running on another node, called itself. Two
// nodes claimed the same shard id and every placement landed on a port an
// orphan was already holding.
seen := map[string]bool{}
for i := range 1000 {
// The same counter value every time, which is exactly what a restart
// reproduces.
id := replicaID("llama", 1)
if seen[id] {
t.Fatalf("id %q repeated after %d placements", id, i)
}
seen[id] = true
}
}
func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
// Stopping is not instant, and an engine keeps its socket until it exits.
// Giving its port to a new shard makes that shard fail to bind, which fails
// its replica, which places another one on the same port.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
placed := h.r.Replicas()[0]
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
// An agent reports a stray on the same replica id, holding a different
// port. It is disowned — and its port must not become available.
strayPort := placed.Shards[0].HTTPPort + 2
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0",
Rank: 0, HTTPPort: strayPort, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
}})
if len(unclaimed) != 1 {
t.Fatalf("the stray was not disowned: %+v", unclaimed)
}
for range 5 {
if got := h.r.allocatePorts(nil); got == strayPort {
t.Fatalf("port %d was handed out while a stray still holds it", got)
}
}
}
// TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed is the mirror image of
// adoption, and the half that was missing.
//
// Observed on a real cluster: the engine was killed, the agent restarted and
// registered with nothing running, and the replica stayed Ready pointing at a
// dead port for as long as anyone watched. Traffic survived — the breaker took
// the endpoint out — but the cluster ran a third short with no sign of it.
func TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.markReady()
rep := h.r.Replicas()[0]
if rep.State != Ready {
t.Fatalf("state = %s", rep.State)
}
shardID := rep.Shards[0].ID
// The agent reconnects and says it is running nothing at all.
h.r.Adopt("1", nil)
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("replicas: %+v", reps)
}
if reps[0].State == Ready {
t.Fatal("a replica whose shard the agent does not have is still Ready")
}
if !strings.Contains(reps[0].Err, "not running") {
t.Errorf("the error does not say what happened: %q", reps[0].Err)
}
// The GPUs come back, or the replacement has nowhere to go.
if free := len(h.inv.FreeGPUs("1")); free != 8 {
t.Errorf("%d of 8 GPUs are free; the dead shard still holds some", free)
}
_ = shardID
// And a replacement is placed once the backoff elapses. Several ticks,
// because failing, reaping and placing are separate steps by design.
for range 6 {
h.advance(10 * time.Minute)
h.r.Reconcile(ctx)
h.markReady()
}
h.r.Reconcile(ctx)
var ready int
for _, r := range h.r.Replicas() {
if r.State == Ready {
ready++
}
}
if ready != 1 {
t.Errorf("%d ready replicas after the rebuild; want 1", ready)
}
}
func TestAdoptionAndForgettingDoNotFightEachOther(t *testing.T) {
// The shard the agent does report must survive the same registration that
// forgets the ones it does not.
h := newHarness(t, testModel(1), gpuNode("1", 8))
ctx := context.Background()
h.r.Reconcile(ctx)
h.markReady()
rep := h.r.Replicas()[0]
unclaimed := h.r.Adopt("1", []AdoptedShard{{
NodeID: "1", ReplicaID: rep.ID, ShardID: rep.Shards[0].ID,
Rank: 0, GPUIndices: rep.Shards[0].GPUIndices, HTTPPort: rep.Shards[0].HTTPPort,
State: supervisor.Ready, Model: "llama", Profile: "tp4",
Endpoint: rep.Endpoint,
}})
if len(unclaimed) != 0 {
t.Fatalf("a shard the agent reported was disowned: %+v", unclaimed)
}
if got := h.r.Replicas()[0].State; got != Ready {
t.Errorf("state = %s; the reported shard was forgotten anyway", got)
}
}
func TestAnotherNodesShardsAreNotForgotten(t *testing.T) {
// One agent's registration says nothing about any other node.
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
ctx := context.Background()
for range 4 {
h.r.Reconcile(ctx)
h.markReady()
h.advance(time.Second)
}
if n := len(h.r.Replicas()); n != 2 {
t.Fatalf("%d replicas", n)
}
h.r.Adopt("1", nil)
var onTwo int
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
if s.NodeID == "2" && !s.State.Terminal() {
onTwo++
}
}
}
if onTwo == 0 {
t.Error("node 1's registration forgot node 2's shards")
}
}