package reconciler import ( "context" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // TestAReplicaThatNeverComesUpIsEventuallyFailed pins down the hole found on a // real cluster: two replicas sat in Starting for eight minutes past a two-minute // startup timeout, holding GPUs and counting towards the model's replica count // while serving nothing. // // Every other way out of Starting needs the agent to report something. When it // never does, this is the only thing that moves. func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) { m := testModel(1, catalog.Profile{ Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 2 * time.Minute, }) h := newHarness(t, m, gpuNode("1", 8)) ctx := context.Background() h.r.Reconcile(ctx) reps := h.r.Replicas() if len(reps) != 1 || reps[0].State != Starting { t.Fatalf("expected one starting replica, got %+v", reps) } // Captured rather than assumed: replica ids carry a random suffix so a // restarted control plane cannot collide with engines from its last life. stuckID := reps[0].ID // The agent says nothing at all, which is exactly the case. h.advance(90 * time.Second) h.r.Reconcile(ctx) if got := h.r.Replicas()[0].State; got != Starting { t.Fatalf("failed early, at %s: %s", 90*time.Second, got) } h.advance(60 * time.Second) h.r.Reconcile(ctx) // Failing it is not the outcome that matters; being unstuck is. The same // reconcile reaps the failure and places a replacement, which is the whole // point of failing it. reps = h.r.Replicas() for _, rep := range reps { if rep.ID == stuckID && rep.State == Starting { t.Fatal("the replica is still stuck in starting past its deadline") } } // The replacement waits out the model's backoff, which is the point of the // backoff: a model that cannot start should not be restarted in a loop. h.advance(10 * time.Minute) h.r.Reconcile(ctx) reps = h.r.Replicas() var replaced bool for _, rep := range reps { if rep.ID != stuckID { replaced = true } } if !replaced { t.Errorf("no replacement was ever placed; replicas are %+v", reps) } // And the shards of the stuck one were told to stop, or its GPUs stay // allocated to a replica nobody is waiting for any more. h.disp.mu.Lock() stopped := len(h.disp.stopped) h.disp.mu.Unlock() if stopped == 0 { t.Error("the stuck replica's shards were never stopped") } } func TestTheStuckReplicaSaysWhyItFailed(t *testing.T) { // Reaping removes the evidence, so the reason has to be recorded when the // deadline passes rather than found afterwards. m := testModel(1, catalog.Profile{ Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Minute, }) h := newHarness(t, m, gpuNode("1", 8)) h.r.Reconcile(context.Background()) h.advance(2 * time.Minute) h.r.expireStuckReplicas() reps := h.r.Replicas() if len(reps) != 1 || reps[0].State != Failed { t.Fatalf("replicas: %+v", reps) } if !strings.Contains(reps[0].Err, "did not come up") { t.Errorf("the error does not say what happened: %q", reps[0].Err) } } func TestAReplicaThatIsLoadingIsNotFailedEarly(t *testing.T) { // Loading a large model legitimately takes a long time. The deadline is // what the profile says, and nothing shorter. m := testModel(1, catalog.Profile{ Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Hour, }) h := newHarness(t, m, gpuNode("1", 8)) ctx := context.Background() h.r.Reconcile(ctx) h.advance(30 * time.Minute) h.r.Reconcile(ctx) if got := h.r.Replicas()[0].State; got == Failed { t.Error("a replica still inside its startup timeout was failed") } } // TestTheFleetWinsOverAReplicaWeOnlyPlaced is the other half of the same // incident. Replica ids come from the model name and a counter, so a control // plane that restarts and places before its agents reconnect builds a different // replica under the same id, on a freshly allocated port. Dropping the agent's // report then leaves the cluster pointing at a port nothing listens on while the // engine that is actually serving is treated as a stray. func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) ctx := context.Background() h.r.Reconcile(ctx) placed := h.r.Replicas()[0] if placed.State != Starting { t.Fatalf("state = %s", placed.State) } // An agent reports the same replica id, running on the port it was actually // started with. unclaimed := h.r.Adopt([]AdoptedShard{{ NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0", Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000, State: supervisor.Ready, Model: "llama", Profile: "tp4", Endpoint: "http://node1:20000", }}) if len(unclaimed) != 0 { t.Fatalf("the running engine was disowned: %+v", unclaimed) } reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("replicas: %+v", reps) } if reps[0].Endpoint != "http://node1:20000" { t.Errorf("endpoint = %q; the cluster is still pointing at what we placed", reps[0].Endpoint) } if reps[0].State != Ready { t.Errorf("state = %s, want ready", reps[0].State) } } func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) { // The other direction. If our copy has a shard that has come up, two things // really are running and the one we did not ask for is the stray — the // agent is told to stop it rather than the live one being replaced. h := newHarness(t, testModel(1), gpuNode("1", 8)) ctx := context.Background() h.r.Reconcile(ctx) placed := h.r.Replicas()[0] h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "") unclaimed := h.r.Adopt([]AdoptedShard{{ NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard", Rank: 0, HTTPPort: 29999, State: supervisor.Ready, Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999", }}) if len(unclaimed) != 1 { t.Fatalf("the stray was not disowned: %+v", unclaimed) } if got := h.r.Replicas()[0].Endpoint; got == "http://node1:29999" { t.Error("a stray replaced the replica that was actually serving") } } func TestReplicaIDsDoNotRepeatAcrossRestarts(t *testing.T) { // The collision that caused the bind-conflict loop on a real cluster. A // control plane that restarts begins counting again, so the replica it // places next was called llama-1 — which is also what an engine started // before the restart, still running on another node, called itself. Two // nodes claimed the same shard id and every placement landed on a port an // orphan was already holding. seen := map[string]bool{} for i := range 1000 { // The same counter value every time, which is exactly what a restart // reproduces. id := replicaID("llama", 1) if seen[id] { t.Fatalf("id %q repeated after %d placements", id, i) } seen[id] = true } } func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) { // Stopping is not instant, and an engine keeps its socket until it exits. // Giving its port to a new shard makes that shard fail to bind, which fails // its replica, which places another one on the same port. h := newHarness(t, testModel(1), gpuNode("1", 8)) ctx := context.Background() h.r.Reconcile(ctx) placed := h.r.Replicas()[0] h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "") // An agent reports a stray on the same replica id, holding a different // port. It is disowned — and its port must not become available. strayPort := placed.Shards[0].HTTPPort + 2 unclaimed := h.r.Adopt([]AdoptedShard{{ NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0", Rank: 0, HTTPPort: strayPort, State: supervisor.Ready, Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999", }}) if len(unclaimed) != 1 { t.Fatalf("the stray was not disowned: %+v", unclaimed) } for range 5 { if got := h.r.allocatePorts(nil); got == strayPort { t.Fatalf("port %d was handed out while a stray still holds it", got) } } }