Ilker ManapandClaude Opus 5.5
d864438272
CI / test (push) Successful in 9m31s
The same inputs place a replica on the same node and card, so an on-demand model that failed to load there was started there again after every backoff until it was parked. The reconciler remembers where a model's replicas failed and places the next away from them when anything else fits (Scheduler.Away); with nowhere else, there again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
81 lines
2.6 KiB
Go
81 lines
2.6 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// failAndRetry fails the last replica started, and returns where the next
|
|
// one is started once the backoff is over.
|
|
func failAndRetry(t *testing.T, h *harness) ShardCommand {
|
|
t.Helper()
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
t.Fatal("nothing was started")
|
|
}
|
|
s := started[len(started)-1]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "the engine exited")
|
|
h.r.Reconcile(t.Context()) // reaps it
|
|
h.disp.reset()
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.r.Reconcile(t.Context())
|
|
again := h.disp.startedShards()
|
|
if len(again) == 0 {
|
|
t.Fatal("it was not tried again")
|
|
}
|
|
return again[0]
|
|
}
|
|
|
|
func oneCard() catalog.Model {
|
|
return testModel(1, catalog.Profile{Name: "one", GPUClass: "h100-80g", GPUsPerNode: 1})
|
|
}
|
|
|
|
func TestAReplicaThatFailedIsTriedOnAnotherNode(t *testing.T) {
|
|
// The same inputs give the same node every time: an on-demand 3D model
|
|
// that failed to load was started on that node again, and again, until
|
|
// it was parked, with another node free all along.
|
|
h := newHarness(t, oneCard(), gpuNode("1", 1), gpuNode("2", 1))
|
|
h.r.Reconcile(t.Context())
|
|
first := h.disp.startedShards()[0].NodeID
|
|
|
|
if next := failAndRetry(t, h); next.NodeID == first {
|
|
t.Errorf("tried on node %s again, where it had just failed", first)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaThatFailedIsTriedOnAnotherCardOfTheNode(t *testing.T) {
|
|
h := newHarness(t, oneCard(), gpuNode("1", 2))
|
|
h.r.Reconcile(t.Context())
|
|
first := h.disp.startedShards()[0].Spec.GPUIndices
|
|
|
|
next := failAndRetry(t, h)
|
|
if len(next.Spec.GPUIndices) != 1 || len(first) != 1 || next.Spec.GPUIndices[0] == first[0] {
|
|
t.Errorf("tried on card %v again, where it had just failed", next.Spec.GPUIndices)
|
|
}
|
|
}
|
|
|
|
func TestWithNowhereElseItIsTriedWhereItFailed(t *testing.T) {
|
|
// A preference, not a rule: a model that has one place is not left
|
|
// unplaced because it failed there once.
|
|
h := newHarness(t, oneCard(), gpuNode("1", 1))
|
|
h.r.Reconcile(t.Context())
|
|
if next := failAndRetry(t, h); next.NodeID != "1" {
|
|
t.Errorf("started on %q", next.NodeID)
|
|
}
|
|
}
|
|
|
|
func TestComingUpForgetsWhereItFailed(t *testing.T) {
|
|
h := newHarness(t, oneCard(), gpuNode("1", 1), gpuNode("2", 1))
|
|
h.r.Reconcile(t.Context())
|
|
next := failAndRetry(t, h)
|
|
h.r.OnShardStatus(next.ReplicaID, next.ShardID, supervisor.Ready, "")
|
|
h.r.mu.Lock()
|
|
left := len(h.r.failedAt["llama"])
|
|
h.r.mu.Unlock()
|
|
if left != 0 {
|
|
t.Errorf("a model that came up still remembers %d failures' places", left)
|
|
}
|
|
}
|