CI / test (push) Successful in 1m23s
Restarting the control plane over a running two-node replica adopted half of it. Every shard it still had was healthy, so it was reported Ready — with one shard of two and no endpoint at all, because the missing one was rank 0. The router was handed a replica nothing could route to. The cause is that a replica had no idea how many shards it should have. It could tell "every shard I have is ready" and nothing else, which is the same sentence whether or not half of it is gone. It now carries the count from its profile, set at placement and again at adoption, and a replica short of shards fails rather than pretending. Onboarding also wrote no engine paths, so a node onboarded by the script could not find llama-server and failed every shard placed on it with "executable file not found". The script now takes them, and says so when none are given — a node with no engine paths can run the sim engine and nothing else. The cluster now runs both at once: two-node simulated replicas on each fabric, and real llama.cpp inference on the CPU, which is a better test bed than either alone. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
409 lines
13 KiB
Go
409 lines
13 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// TestAReplicaThatNeverComesUpIsEventuallyFailed pins down the hole found on a
|
|
// real cluster: two replicas sat in Starting for eight minutes past a two-minute
|
|
// startup timeout, holding GPUs and counting towards the model's replica count
|
|
// while serving nothing.
|
|
//
|
|
// Every other way out of Starting needs the agent to report something. When it
|
|
// never does, this is the only thing that moves.
|
|
func TestAReplicaThatNeverComesUpIsEventuallyFailed(t *testing.T) {
|
|
m := testModel(1, catalog.Profile{
|
|
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 2 * time.Minute,
|
|
})
|
|
h := newHarness(t, m, gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
|
|
h.r.Reconcile(ctx)
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Starting {
|
|
t.Fatalf("expected one starting replica, got %+v", reps)
|
|
}
|
|
// Captured rather than assumed: replica ids carry a random suffix so a
|
|
// restarted control plane cannot collide with engines from its last life.
|
|
stuckID := reps[0].ID
|
|
|
|
// The agent says nothing at all, which is exactly the case.
|
|
h.advance(90 * time.Second)
|
|
h.r.Reconcile(ctx)
|
|
if got := h.r.Replicas()[0].State; got != Starting {
|
|
t.Fatalf("failed early, at %s: %s", 90*time.Second, got)
|
|
}
|
|
|
|
h.advance(60 * time.Second)
|
|
h.r.Reconcile(ctx)
|
|
|
|
// Failing it is not the outcome that matters; being unstuck is. The same
|
|
// reconcile reaps the failure and places a replacement, which is the whole
|
|
// point of failing it.
|
|
reps = h.r.Replicas()
|
|
for _, rep := range reps {
|
|
if rep.ID == stuckID && rep.State == Starting {
|
|
t.Fatal("the replica is still stuck in starting past its deadline")
|
|
}
|
|
}
|
|
// The replacement waits out the model's backoff, which is the point of the
|
|
// backoff: a model that cannot start should not be restarted in a loop.
|
|
h.advance(10 * time.Minute)
|
|
h.r.Reconcile(ctx)
|
|
|
|
reps = h.r.Replicas()
|
|
var replaced bool
|
|
for _, rep := range reps {
|
|
if rep.ID != stuckID {
|
|
replaced = true
|
|
}
|
|
}
|
|
if !replaced {
|
|
t.Errorf("no replacement was ever placed; replicas are %+v", reps)
|
|
}
|
|
|
|
// And the shards of the stuck one were told to stop, or its GPUs stay
|
|
// allocated to a replica nobody is waiting for any more.
|
|
h.disp.mu.Lock()
|
|
stopped := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the stuck replica's shards were never stopped")
|
|
}
|
|
}
|
|
|
|
func TestTheStuckReplicaSaysWhyItFailed(t *testing.T) {
|
|
// Reaping removes the evidence, so the reason has to be recorded when the
|
|
// deadline passes rather than found afterwards.
|
|
m := testModel(1, catalog.Profile{
|
|
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Minute,
|
|
})
|
|
h := newHarness(t, m, gpuNode("1", 8))
|
|
h.r.Reconcile(context.Background())
|
|
|
|
h.advance(2 * time.Minute)
|
|
h.r.expireStuckReplicas()
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("replicas: %+v", reps)
|
|
}
|
|
if !strings.Contains(reps[0].Err, "did not come up") {
|
|
t.Errorf("the error does not say what happened: %q", reps[0].Err)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaThatIsLoadingIsNotFailedEarly(t *testing.T) {
|
|
// Loading a large model legitimately takes a long time. The deadline is
|
|
// what the profile says, and nothing shorter.
|
|
m := testModel(1, catalog.Profile{
|
|
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: time.Hour,
|
|
})
|
|
h := newHarness(t, m, gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
h.r.Reconcile(ctx)
|
|
|
|
h.advance(30 * time.Minute)
|
|
h.r.Reconcile(ctx)
|
|
if got := h.r.Replicas()[0].State; got == Failed {
|
|
t.Error("a replica still inside its startup timeout was failed")
|
|
}
|
|
}
|
|
|
|
// TestTheFleetWinsOverAReplicaWeOnlyPlaced is the other half of the same
|
|
// incident. Replica ids come from the model name and a counter, so a control
|
|
// plane that restarts and places before its agents reconnect builds a different
|
|
// replica under the same id, on a freshly allocated port. Dropping the agent's
|
|
// report then leaves the cluster pointing at a port nothing listens on while the
|
|
// engine that is actually serving is treated as a stray.
|
|
func TestTheFleetWinsOverAReplicaWeOnlyPlaced(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
|
|
h.r.Reconcile(ctx)
|
|
placed := h.r.Replicas()[0]
|
|
if placed.State != Starting {
|
|
t.Fatalf("state = %s", placed.State)
|
|
}
|
|
|
|
// An agent reports the same replica id, running on the port it was actually
|
|
// started with.
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
|
NodeID: "1", ReplicaID: placed.ID, ShardID: placed.ID + "-r0",
|
|
Rank: 0, GPUIndices: []int{0, 1, 2, 3}, HTTPPort: 20000,
|
|
State: supervisor.Ready, Model: "llama", Profile: "tp4",
|
|
Endpoint: "http://node1:20000",
|
|
}})
|
|
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("the running engine was disowned: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("replicas: %+v", reps)
|
|
}
|
|
if reps[0].Endpoint != "http://node1:20000" {
|
|
t.Errorf("endpoint = %q; the cluster is still pointing at what we placed",
|
|
reps[0].Endpoint)
|
|
}
|
|
if reps[0].State != Ready {
|
|
t.Errorf("state = %s, want ready", reps[0].State)
|
|
}
|
|
}
|
|
|
|
func TestAnAdoptedShardDoesNotDisplaceOneThatIsActuallyRunning(t *testing.T) {
|
|
// The other direction. If our copy has a shard that has come up, two things
|
|
// really are running and the one we did not ask for is the stray — the
|
|
// agent is told to stop it rather than the live one being replaced.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
h.r.Reconcile(ctx)
|
|
|
|
placed := h.r.Replicas()[0]
|
|
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
|
|
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
|
NodeID: "1", ReplicaID: placed.ID, ShardID: "some-other-shard",
|
|
Rank: 0, HTTPPort: 29999, State: supervisor.Ready,
|
|
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
|
|
}})
|
|
|
|
if len(unclaimed) != 1 {
|
|
t.Fatalf("the stray was not disowned: %+v", unclaimed)
|
|
}
|
|
if got := h.r.Replicas()[0].Endpoint; got == "http://node1:29999" {
|
|
t.Error("a stray replaced the replica that was actually serving")
|
|
}
|
|
}
|
|
|
|
func TestReplicaIDsDoNotRepeatAcrossRestarts(t *testing.T) {
|
|
// The collision that caused the bind-conflict loop on a real cluster. A
|
|
// control plane that restarts begins counting again, so the replica it
|
|
// places next was called llama-1 — which is also what an engine started
|
|
// before the restart, still running on another node, called itself. Two
|
|
// nodes claimed the same shard id and every placement landed on a port an
|
|
// orphan was already holding.
|
|
seen := map[string]bool{}
|
|
for i := range 1000 {
|
|
// The same counter value every time, which is exactly what a restart
|
|
// reproduces.
|
|
id := replicaID("llama", 1)
|
|
if seen[id] {
|
|
t.Fatalf("id %q repeated after %d placements", id, i)
|
|
}
|
|
seen[id] = true
|
|
}
|
|
}
|
|
|
|
func TestAPortHeldByAStrayIsNotHandedOut(t *testing.T) {
|
|
// Stopping is not instant, and an engine keeps its socket until it exits.
|
|
// Giving its port to a new shard makes that shard fail to bind, which fails
|
|
// its replica, which places another one on the same port.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
h.r.Reconcile(ctx)
|
|
|
|
placed := h.r.Replicas()[0]
|
|
h.r.OnShardStatus(placed.ID, placed.Shards[0].ID, supervisor.Ready, "")
|
|
|
|
// An agent reports a stray on the same replica id, holding a different
|
|
// port. It is disowned — and its port must not become available.
|
|
strayPort := placed.Shards[0].HTTPPort + 2
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
|
NodeID: "1", ReplicaID: placed.ID, ShardID: "stray-r0",
|
|
Rank: 0, HTTPPort: strayPort, State: supervisor.Ready,
|
|
Model: "llama", Profile: "tp4", Endpoint: "http://node1:29999",
|
|
}})
|
|
if len(unclaimed) != 1 {
|
|
t.Fatalf("the stray was not disowned: %+v", unclaimed)
|
|
}
|
|
|
|
for range 5 {
|
|
if got := h.r.allocatePorts(nil); got == strayPort {
|
|
t.Fatalf("port %d was handed out while a stray still holds it", got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed is the mirror image of
|
|
// adoption, and the half that was missing.
|
|
//
|
|
// Observed on a real cluster: the engine was killed, the agent restarted and
|
|
// registered with nothing running, and the replica stayed Ready pointing at a
|
|
// dead port for as long as anyone watched. Traffic survived — the breaker took
|
|
// the endpoint out — but the cluster ran a third short with no sign of it.
|
|
func TestAnEngineThatDiesWhileItsAgentSurvivesIsNoticed(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
h.r.Reconcile(ctx)
|
|
h.markReady()
|
|
|
|
rep := h.r.Replicas()[0]
|
|
if rep.State != Ready {
|
|
t.Fatalf("state = %s", rep.State)
|
|
}
|
|
shardID := rep.Shards[0].ID
|
|
|
|
// The agent reconnects and says it is running nothing at all.
|
|
h.r.Adopt("1", nil)
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("replicas: %+v", reps)
|
|
}
|
|
if reps[0].State == Ready {
|
|
t.Fatal("a replica whose shard the agent does not have is still Ready")
|
|
}
|
|
if !strings.Contains(reps[0].Err, "not running") {
|
|
t.Errorf("the error does not say what happened: %q", reps[0].Err)
|
|
}
|
|
|
|
// The GPUs come back, or the replacement has nowhere to go.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("%d of 8 GPUs are free; the dead shard still holds some", free)
|
|
}
|
|
_ = shardID
|
|
|
|
// And a replacement is placed once the backoff elapses. Several ticks,
|
|
// because failing, reaping and placing are separate steps by design.
|
|
for range 6 {
|
|
h.advance(10 * time.Minute)
|
|
h.r.Reconcile(ctx)
|
|
h.markReady()
|
|
}
|
|
h.r.Reconcile(ctx)
|
|
|
|
var ready int
|
|
for _, r := range h.r.Replicas() {
|
|
if r.State == Ready {
|
|
ready++
|
|
}
|
|
}
|
|
if ready != 1 {
|
|
t.Errorf("%d ready replicas after the rebuild; want 1", ready)
|
|
}
|
|
}
|
|
|
|
func TestAdoptionAndForgettingDoNotFightEachOther(t *testing.T) {
|
|
// The shard the agent does report must survive the same registration that
|
|
// forgets the ones it does not.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
ctx := context.Background()
|
|
h.r.Reconcile(ctx)
|
|
h.markReady()
|
|
|
|
rep := h.r.Replicas()[0]
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{{
|
|
NodeID: "1", ReplicaID: rep.ID, ShardID: rep.Shards[0].ID,
|
|
Rank: 0, GPUIndices: rep.Shards[0].GPUIndices, HTTPPort: rep.Shards[0].HTTPPort,
|
|
State: supervisor.Ready, Model: "llama", Profile: "tp4",
|
|
Endpoint: rep.Endpoint,
|
|
}})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("a shard the agent reported was disowned: %+v", unclaimed)
|
|
}
|
|
if got := h.r.Replicas()[0].State; got != Ready {
|
|
t.Errorf("state = %s; the reported shard was forgotten anyway", got)
|
|
}
|
|
}
|
|
|
|
func TestAnotherNodesShardsAreNotForgotten(t *testing.T) {
|
|
// One agent's registration says nothing about any other node.
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
ctx := context.Background()
|
|
for range 4 {
|
|
h.r.Reconcile(ctx)
|
|
h.markReady()
|
|
h.advance(time.Second)
|
|
}
|
|
if n := len(h.r.Replicas()); n != 2 {
|
|
t.Fatalf("%d replicas", n)
|
|
}
|
|
|
|
h.r.Adopt("1", nil)
|
|
|
|
var onTwo int
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
if s.NodeID == "2" && !s.State.Terminal() {
|
|
onTwo++
|
|
}
|
|
}
|
|
}
|
|
if onTwo == 0 {
|
|
t.Error("node 1's registration forgot node 2's shards")
|
|
}
|
|
}
|
|
|
|
// TestAReplicaMissingAShardIsNotReady is what a control plane restart exposed:
|
|
// half of a two-node replica was adopted, every shard it still had was healthy,
|
|
// and it was reported Ready with no endpoint — so the router was handed a
|
|
// replica nothing could route to.
|
|
func TestAReplicaMissingAShardIsNotReady(t *testing.T) {
|
|
rep := &Replica{
|
|
ID: "r1", WantShards: 2,
|
|
Shards: []Shard{{ID: "r1-r1", Rank: 1, State: supervisor.Ready}},
|
|
}
|
|
rep.deriveState()
|
|
|
|
if rep.State == Ready {
|
|
t.Fatal("a replica with one of two shards is Ready")
|
|
}
|
|
if !strings.Contains(rep.Err, "1 of 2") {
|
|
t.Errorf("the error does not say what is missing: %q", rep.Err)
|
|
}
|
|
}
|
|
|
|
func TestACompleteReplicaIsReady(t *testing.T) {
|
|
rep := &Replica{
|
|
ID: "r1", WantShards: 2,
|
|
Shards: []Shard{
|
|
{ID: "r1-r0", Rank: 0, State: supervisor.Ready},
|
|
{ID: "r1-r1", Rank: 1, State: supervisor.Ready},
|
|
},
|
|
}
|
|
rep.deriveState()
|
|
if rep.State != Ready {
|
|
t.Errorf("state = %s, want ready", rep.State)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaStillComingUpIsNotFailed(t *testing.T) {
|
|
// Every shard present, not all ready yet: that is Starting, not a shortage.
|
|
rep := &Replica{
|
|
ID: "r1", WantShards: 2,
|
|
Shards: []Shard{
|
|
{ID: "r1-r0", Rank: 0, State: supervisor.Ready},
|
|
{ID: "r1-r1", Rank: 1, State: supervisor.Loading},
|
|
},
|
|
}
|
|
rep.deriveState()
|
|
if rep.State != Starting {
|
|
t.Errorf("state = %s, want starting", rep.State)
|
|
}
|
|
}
|
|
|
|
func TestAMultiNodeReplicaKnowsHowManyShardsItWants(t *testing.T) {
|
|
// End to end through placement, because the count has to reach the replica
|
|
// from the profile rather than from whatever turned up.
|
|
m := testModel(1, catalog.Profile{
|
|
Name: "tp8", GPUClass: "h100-80g", TP: 8, Nodes: 2, GPUsPerNode: 4,
|
|
})
|
|
h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
h.r.Reconcile(context.Background())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("replicas: %+v", reps)
|
|
}
|
|
if reps[0].WantShards != 2 {
|
|
t.Errorf("WantShards = %d, want 2", reps[0].WantShards)
|
|
}
|
|
}
|