package reconciler import ( "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" ) // failAndRetry fails the last replica started, and returns where the next // one is started once the backoff is over. func failAndRetry(t *testing.T, h *harness) ShardCommand { t.Helper() started := h.disp.startedShards() if len(started) == 0 { t.Fatal("nothing was started") } s := started[len(started)-1] h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "the engine exited") h.r.Reconcile(t.Context()) // reaps it h.disp.reset() h.advance(DefaultConfig().MaxBackoff) h.r.Reconcile(t.Context()) again := h.disp.startedShards() if len(again) == 0 { t.Fatal("it was not tried again") } return again[0] } func oneCard() catalog.Model { return testModel(1, catalog.Profile{Name: "one", GPUClass: "h100-80g", GPUsPerNode: 1}) } func TestAReplicaThatFailedIsTriedOnAnotherNode(t *testing.T) { // The same inputs give the same node every time: an on-demand 3D model // that failed to load was started on that node again, and again, until // it was parked, with another node free all along. h := newHarness(t, oneCard(), gpuNode("1", 1), gpuNode("2", 1)) h.r.Reconcile(t.Context()) first := h.disp.startedShards()[0].NodeID if next := failAndRetry(t, h); next.NodeID == first { t.Errorf("tried on node %s again, where it had just failed", first) } } func TestAReplicaThatFailedIsTriedOnAnotherCardOfTheNode(t *testing.T) { h := newHarness(t, oneCard(), gpuNode("1", 2)) h.r.Reconcile(t.Context()) first := h.disp.startedShards()[0].Spec.GPUIndices next := failAndRetry(t, h) if len(next.Spec.GPUIndices) != 1 || len(first) != 1 || next.Spec.GPUIndices[0] == first[0] { t.Errorf("tried on card %v again, where it had just failed", next.Spec.GPUIndices) } } func TestWithNowhereElseItIsTriedWhereItFailed(t *testing.T) { // A preference, not a rule: a model that has one place is not left // unplaced because it failed there once. h := newHarness(t, oneCard(), gpuNode("1", 1)) h.r.Reconcile(t.Context()) if next := failAndRetry(t, h); next.NodeID != "1" { t.Errorf("started on %q", next.NodeID) } } func TestComingUpForgetsWhereItFailed(t *testing.T) { h := newHarness(t, oneCard(), gpuNode("1", 1), gpuNode("2", 1)) h.r.Reconcile(t.Context()) next := failAndRetry(t, h) h.r.OnShardStatus(next.ReplicaID, next.ShardID, supervisor.Ready, "") h.r.mu.Lock() left := len(h.r.failedAt["llama"]) h.r.mu.Unlock() if left != 0 { t.Errorf("a model that came up still remembers %d failures' places", left) } }