Do not place on a fleet that has not reported in yet
CI / test (push) Has been cancelled

A control plane that has just started knows nothing, and its first
reconcile tick runs before any agent has registered. "needs 2 nodes on one
fabric, but no node reports a fabric" in the log is that, and it looked
harmless because adoption catches up.

What is not harmless is placing on the fraction of the fleet that happened
to report first. After one restart both replicas of qwen2.5-0.5b landed on
node04 — the only node registered at that instant — when four were coming,
and nothing ever moved them, because the desired count was met.

Placement now waits, and leaves by either of two doors, because either
alone is wrong. Waiting for the window to expire would cost that much
downtime on every restart even when the whole fleet reconnected in a
second; waiting only for the node count to stop changing would hang forever
on a cluster whose nodes never all come back. So: the count holds steady for
a quarter of the window, or the window expires.

Only placement waits. Node expiry, stuck-replica expiry, draining and
reaping all stay correct on a partial fleet, and adoption in particular must
not wait — it is how a restart recovers what is already running, and
delaying it would leave live engines unaccounted for exactly while the
reconciler decides what to build.

The window is paid once per process. A node joining a running cluster is an
ordinary event, not a reason to stop placing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 23:18:29 +02:00
1 parent db448a9e87
commit e05d5d48ee
2 files changed
+195 -1

No files matched your search

@@ -141,6 +141,14 @@ func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness
for _, n := range nodes {
touchAt(inv, n.ID, c.now())
}
// Past the startup settling window. These nodes are all here — the harness
// built the whole fleet before anything ran — which is the state settling
// waits for, not one it should be made to simulate in every test. The
// window's own behaviour is tested separately.
c.advance(cfg.SettleFor)
for _, n := range nodes {
touchAt(inv, n.ID, c.now())
}
return &harness{r: r, inv: inv, disp: disp, clock: c}
}
@@ -992,3 +1000,117 @@ func TestAHalfThatNeverArrivesStillFails(t *testing.T) {
t.Errorf("node has %d free GPUs; the abandoned half still holds them", free)
}
}
// --- the startup settling window ---
func settlingHarness(t *testing.T, m catalog.Model) (*Reconciler, *inventory.Store, *fakeDispatcher, *clock) {
t.Helper()
cat, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatal(err)
}
inv := inventory.NewStore()
disp := &fakeDispatcher{}
c := newClock()
r := New(DefaultConfig(), cat, inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
return r, inv, disp, c
}
func addNode(inv *inventory.Store, c *clock, n inventory.Node) {
inv.Upsert(n)
touchAt(inv, n.ID, c.now())
}
func TestPlacementWaitsForTheFleetToArrive(t *testing.T) {
// A control plane that has just started knows nothing, and the first tick
// runs before any agent has registered. Placing then puts everything on
// whichever fraction reported first: after one restart both replicas of a
// model landed on the single node that had registered, when four were
// coming.
r, inv, disp, c := settlingHarness(t, testModel(2))
// Nothing registered yet.
r.Reconcile(t.Context())
if got := len(disp.startedShards()); got != 0 {
t.Fatalf("placed %d shards on an empty fleet", got)
}
// One node of four.
addNode(inv, c, gpuNode("1", 8))
r.Reconcile(t.Context())
if got := len(disp.startedShards()); got != 0 {
t.Fatalf("placed %d shards while the fleet was still arriving", got)
}
// The rest arrive.
for _, id := range []string{"2", "3", "4"} {
addNode(inv, c, gpuNode(id, 8))
}
r.Reconcile(t.Context())
if got := len(disp.startedShards()); got != 0 {
t.Fatalf("placed %d shards the instant the last node appeared", got)
}
// Steady for a moment: now it may act, and on the whole fleet.
c.advance(DefaultConfig().SettleFor / 4)
for _, id := range []string{"1", "2", "3", "4"} {
touchAt(inv, id, c.now())
}
r.Reconcile(t.Context())
if got := len(disp.startedShards()); got == 0 {
t.Fatal("placement never resumed after the fleet settled")
}
}
func TestPlacementDoesNotWaitForeverForNodesThatNeverCome(t *testing.T) {
// Waiting only for the count to stop changing would hang forever on a
// cluster whose nodes never all come back.
r, inv, disp, c := settlingHarness(t, testModel(1))
addNode(inv, c, gpuNode("1", 8))
c.advance(DefaultConfig().SettleFor)
touchAt(inv, "1", c.now())
r.Reconcile(t.Context())
if got := len(disp.startedShards()); got == 0 {
t.Error("placement never resumed once the window expired")
}
}
func TestSettlingIsOnlyEverPaidOnce(t *testing.T) {
// A node joining later in the day is an ordinary event, not a reason to
// stop placing while the cluster is running.
r, inv, disp, c := settlingHarness(t, testModel(1))
addNode(inv, c, gpuNode("1", 8))
c.advance(DefaultConfig().SettleFor)
touchAt(inv, "1", c.now())
r.Reconcile(t.Context())
if len(disp.startedShards()) == 0 {
t.Fatal("nothing placed after the window")
}
disp.reset()
addNode(inv, c, gpuNode("2", 8))
if !r.fleetHasSettled() {
t.Error("a node joining a running cluster restarted the settling window")
}
}
func TestAdoptionIsNotBlockedByTheSettlingWindow(t *testing.T) {
// Only placement waits. Adoption is how a restart recovers what is already
// running, and delaying it would leave live engines unaccounted for exactly
// when the reconciler is deciding what to build.
r, inv, _, c := settlingHarness(t, testModel(1))
addNode(inv, c, gpuNode("1", 8))
unclaimed := r.Adopt("1", []AdoptedShard{
adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}),
})
if len(unclaimed) != 0 {
t.Fatalf("adoption was refused during settling: %+v", unclaimed)
}
if got := len(r.Replicas()); got != 1 {
t.Fatalf("expected the replica to be adopted, got %d", got)
}
}