The settling window ran from process start, so a standby that took over after a minute placed at once, before the agents reconnected and reported what they run: it started second copies of replicas it did not know of and drained the healthy ones when they were adopted. Seen on the PoC: a leader restart replaced llama-guard-3-1b and a qwen2.5-1.5b replica. Reconcile now reopens the window on the first pass after standing by. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
82 lines
2.4 KiB
Go
82 lines
2.4 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
type fixedLeader bool
|
|
|
|
func (f fixedLeader) IsLeader() bool { return bool(f) }
|
|
|
|
func TestAStandbyPlacesNothing(t *testing.T) {
|
|
// The failure this prevents: two control planes against one store both see
|
|
// a model wanting replicas and none running, and both place. The model
|
|
// ends up with twice what it asked for, each replica holding GPUs the
|
|
// other's inventory believes are free, and nothing crashes to say so.
|
|
h := newHarness(t, testModel(1), gpuNode("n1", 8))
|
|
h.r.SetLeadership(fixedLeader(false))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a standby placed %d replicas", got)
|
|
}
|
|
}
|
|
|
|
func TestALeaderPlacesAsBefore(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("n1", 8))
|
|
h.r.SetLeadership(fixedLeader(true))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.r.Replicas()); got == 0 {
|
|
t.Error("a leader placed nothing")
|
|
}
|
|
}
|
|
|
|
func TestWithoutLeadershipConfiguredNothingChanges(t *testing.T) {
|
|
// Every single-control-plane deployment, and every test written before
|
|
// leadership existed. Unset must mean "act", not "wait for permission
|
|
// from a thing that was never wired up".
|
|
h := newHarness(t, testModel(1), gpuNode("n1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.r.Replicas()); got == 0 {
|
|
t.Error("a reconciler with no leadership configured placed nothing")
|
|
}
|
|
}
|
|
|
|
// switchLeader is leadership that can change hands.
|
|
type switchLeader struct{ on bool }
|
|
|
|
func (s *switchLeader) IsLeader() bool { return s.on }
|
|
|
|
func TestAStandbyThatTakesOverWaitsForTheFleetBeforePlacing(t *testing.T) {
|
|
r, inv, disp, c := settlingHarness(t, testModel(1))
|
|
lead := &switchLeader{}
|
|
r.SetLeadership(lead)
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
|
|
// A standby for longer than the window: its own start is long past.
|
|
r.Reconcile(t.Context())
|
|
c.advance(2 * DefaultConfig().SettleFor)
|
|
touchAt(inv, "1", c.now())
|
|
r.Reconcile(t.Context())
|
|
|
|
// It takes over. The agents have not told it what runs yet; placing now
|
|
// would start a second copy of what they are about to report.
|
|
lead.on = true
|
|
r.Reconcile(t.Context())
|
|
if n := len(disp.startedShards()); n != 0 {
|
|
t.Fatalf("placed %d shard(s) the moment it took over", n)
|
|
}
|
|
// Once the fleet has had its window, it places what is still missing.
|
|
c.advance(DefaultConfig().SettleFor)
|
|
touchAt(inv, "1", c.now())
|
|
r.Reconcile(t.Context())
|
|
if len(disp.startedShards()) == 0 {
|
|
t.Fatal("nothing placed after the window")
|
|
}
|
|
}
|