Files
llmgw/internal/control/reconciler/leadership_test.go
T
Ilker ManapandClaude Opus 5.5 0add189e02 A standby that takes over waits for the fleet before placing
The settling window ran from process start, so a standby that took over
after a minute placed at once, before the agents reconnected and reported
what they run: it started second copies of replicas it did not know of and
drained the healthy ones when they were adopted. Seen on the PoC: a leader
restart replaced llama-guard-3-1b and a qwen2.5-1.5b replica. Reconcile now
reopens the window on the first pass after standing by.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-07 13:40:28 +02:00

82 lines
2.4 KiB
Go

package reconciler
import (
"testing"
)
type fixedLeader bool
func (f fixedLeader) IsLeader() bool { return bool(f) }
func TestAStandbyPlacesNothing(t *testing.T) {
// The failure this prevents: two control planes against one store both see
// a model wanting replicas and none running, and both place. The model
// ends up with twice what it asked for, each replica holding GPUs the
// other's inventory believes are free, and nothing crashes to say so.
h := newHarness(t, testModel(1), gpuNode("n1", 8))
h.r.SetLeadership(fixedLeader(false))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 0 {
t.Errorf("a standby placed %d replicas", got)
}
}
func TestALeaderPlacesAsBefore(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("n1", 8))
h.r.SetLeadership(fixedLeader(true))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got == 0 {
t.Error("a leader placed nothing")
}
}
func TestWithoutLeadershipConfiguredNothingChanges(t *testing.T) {
// Every single-control-plane deployment, and every test written before
// leadership existed. Unset must mean "act", not "wait for permission
// from a thing that was never wired up".
h := newHarness(t, testModel(1), gpuNode("n1", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got == 0 {
t.Error("a reconciler with no leadership configured placed nothing")
}
}
// switchLeader is leadership that can change hands.
type switchLeader struct{ on bool }
func (s *switchLeader) IsLeader() bool { return s.on }
func TestAStandbyThatTakesOverWaitsForTheFleetBeforePlacing(t *testing.T) {
r, inv, disp, c := settlingHarness(t, testModel(1))
lead := &switchLeader{}
r.SetLeadership(lead)
addNode(inv, c, gpuNode("1", 8))
// A standby for longer than the window: its own start is long past.
r.Reconcile(t.Context())
c.advance(2 * DefaultConfig().SettleFor)
touchAt(inv, "1", c.now())
r.Reconcile(t.Context())
// It takes over. The agents have not told it what runs yet; placing now
// would start a second copy of what they are about to report.
lead.on = true
r.Reconcile(t.Context())
if n := len(disp.startedShards()); n != 0 {
t.Fatalf("placed %d shard(s) the moment it took over", n)
}
// Once the fleet has had its window, it places what is still missing.
c.advance(DefaultConfig().SettleFor)
touchAt(inv, "1", c.now())
r.Reconcile(t.Context())
if len(disp.startedShards()) == 0 {
t.Fatal("nothing placed after the window")
}
}