package reconciler import ( "bytes" "context" "fmt" "log/slog" "slices" "strings" "sync" "testing" "time" "go.opentelemetry.io/otel/trace" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) // fakeDispatcher records commands instead of sending them. type fakeDispatcher struct { mu sync.Mutex started []ShardCommand stopped []string drained []string failNext error // traces are the span each start was sent under, which is what the // real dispatcher puts on the command for the agent. traces []trace.SpanContext } func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error { d.mu.Lock() defer d.mu.Unlock() if d.failNext != nil { err := d.failNext d.failNext = nil return err } d.started = append(d.started, cmd) d.traces = append(d.traces, trace.SpanContextFromContext(ctx)) return nil } func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error { d.mu.Lock() defer d.mu.Unlock() d.stopped = append(d.stopped, shardID) return nil } func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error { d.mu.Lock() defer d.mu.Unlock() d.drained = append(d.drained, shardID) return nil } func (d *fakeDispatcher) startedShards() []ShardCommand { d.mu.Lock() defer d.mu.Unlock() return slices.Clone(d.started) } func (d *fakeDispatcher) reset() { d.mu.Lock() defer d.mu.Unlock() d.started, d.stopped, d.drained = nil, nil, nil } // clock lets tests move time without sleeping. type clock struct { mu sync.Mutex t time.Time } func newClock() *clock { return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)} } func (c *clock) now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.t } func (c *clock) advance(d time.Duration) { c.mu.Lock() c.t = c.t.Add(d) c.mu.Unlock() } func gpuNode(id string, n int, fabrics ...string) inventory.Node { nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready} for i := range n { nd.GPUs = append(nd.GPUs, discovery.GPU{ Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0", }) } for _, f := range fabrics { nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{ Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"}, }) } return nd } func testModel(min int, profiles ...catalog.Profile) catalog.Model { if len(profiles) == 0 { profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}} } return catalog.Model{ Name: "llama", Source: "/models/llama", Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles, } } type harness struct { r *Reconciler inv *inventory.Store disp *fakeDispatcher clock *clock live *catalog.Live } // setModels replaces the catalog the reconciler works from, as an edit does. func (h *harness) setModels(t *testing.T, models ...catalog.Model) { t.Helper() cat, err := catalog.New(models) if err != nil { t.Fatal(err) } h.live.Store(cat) } func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness { t.Helper() cat, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatal(err) } inv := inventory.NewStore() c := newClock() for _, n := range nodes { inv.Upsert(n) inv.Touch(n.ID) } disp := &fakeDispatcher{} cfg := DefaultConfig() cfg.UnreachableAfter = 15 * time.Second cfg.LostAfter = 60 * time.Second live := catalog.NewLive(cat) r := New(cfg, live, inv, disp, slog.New(slog.DiscardHandler)) r.SetClock(c.now) // Nodes were just registered, so their LastSeen is wall-clock now, which is // far from the fake clock's epoch. Re-stamp them onto the fake timeline. for _, n := range nodes { touchAt(inv, n.ID, c.now()) } // Past the startup settling window. These nodes are all here — the harness // built the whole fleet before anything ran — which is the state settling // waits for, not one it should be made to simulate in every test. The // window's own behaviour is tested separately. c.advance(cfg.SettleFor) for _, n := range nodes { touchAt(inv, n.ID, c.now()) } return &harness{r: r, inv: inv, disp: disp, clock: c, live: live} } // touchAt rewrites a node's LastSeen onto the fake clock's timeline. func touchAt(inv *inventory.Store, id string, at time.Time) { n, ok := inv.Get(id) if !ok { return } n.LastSeen = at n.State = inventory.Ready inv.Upsert(n) // Upsert stamps LastSeen with the wall clock, so set it again directly. inv.SetLastSeen(id, at) } // advance moves the clock and keeps every node heartbeating. // // Without this, advancing time past UnreachableAfter makes every node look // silent and placement stops — which is correct behaviour, but not what the // backoff tests are trying to exercise. func (h *harness) advance(d time.Duration) { h.clock.advance(d) for _, n := range h.inv.Nodes() { touchAt(h.inv, n.ID, h.clock.now()) } } // markReady walks every started shard to Ready. func (h *harness) markReady() { for _, cmd := range h.disp.startedShards() { h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "") } } func TestPlacesMinimumReplicas(t *testing.T) { h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) h.r.Reconcile(t.Context()) h.markReady() h.r.Reconcile(t.Context()) h.markReady() reps := h.r.Replicas() if len(reps) != 2 { t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps) } for _, rep := range reps { if rep.State != Ready { t.Errorf("replica %s is %s", rep.ID, rep.State) } if rep.Endpoint == "" { t.Errorf("replica %s has no endpoint", rep.ID) } } } func TestOneReplicaPerPass(t *testing.T) { // Placement depends on what is already allocated, so starting several from // one inventory snapshot would hand them the same GPUs. h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) h.r.Reconcile(t.Context()) if got := len(h.r.Replicas()); got != 1 { t.Fatalf("one pass created %d replicas", got) } seen := map[int]string{} for range 3 { h.markReady() h.r.Reconcile(t.Context()) } for _, rep := range h.r.Replicas() { for _, s := range rep.Shards { for _, g := range s.GPUIndices { key := g if prev, dup := seen[key]; dup && prev != s.NodeID { continue // same index on a different node is fine } seen[key] = s.NodeID } } } if got := len(h.r.Replicas()); got != 3 { t.Errorf("expected 3 replicas, got %d", got) } } func TestSpreadsAcrossNodes(t *testing.T) { h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) for range 2 { h.r.Reconcile(t.Context()) h.markReady() } nodes := map[string]struct{}{} for _, rep := range h.r.Replicas() { for _, n := range rep.NodeIDs() { nodes[n] = struct{}{} } } if len(nodes) != 2 { t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes)) } } func TestPacksWhenSpreadingIsImpossible(t *testing.T) { // Spreading is a preference, not a rule: a second replica on one node still // survives an engine crash, which is the common failure. h := newHarness(t, testModel(2), gpuNode("only", 8)) for range 2 { h.r.Reconcile(t.Context()) h.markReady() } if got := len(h.r.Replicas()); got != 2 { t.Fatalf("expected 2 replicas on the single node, got %d", got) } } func TestShardOrderLeaderFirst(t *testing.T) { // Both engines put the rendezvous point on rank 0 — SGLang's dist-init // address and vLLM's Ray head — so a worker started first has nothing to // connect to and spends its startup budget retrying. // // This test previously asserted the opposite, on the theory that the head // should wait for its workers. That is backwards: vLLM's server blocks // until the Ray cluster has the GPUs it needs, so the leader can start // immediately and waits on its own. h := newHarness(t, testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) h.r.Reconcile(t.Context()) started := h.disp.startedShards() if len(started) != 2 { t.Fatalf("expected 2 shards, got %d", len(started)) } if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 { t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+ "point and must start first", started[0].Spec.Rank, started[1].Spec.Rank) } // Every rank must be given the same rendezvous point. if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr { t.Errorf("ranks got different leader addresses: %q and %q", started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr) } if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort { t.Error("ranks got different leader ports") } for _, cmd := range started { if len(cmd.Spec.HCAs) == 0 { t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank) } } } func TestFailedShardFailsWholeReplica(t *testing.T) { // One dead shard makes the others useless: they wait on collectives that // will never complete while holding GPUs (plan §5.4). h := newHarness(t, testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) h.r.Reconcile(t.Context()) started := h.disp.startedShards() // One shard ready, the other dead. h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "") h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory") h.r.Reconcile(t.Context()) // Both shards must have been stopped, not just the failed one. h.disp.mu.Lock() stopped := slices.Clone(h.disp.stopped) h.disp.mu.Unlock() if len(stopped) != 2 { t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped) } // And the GPUs must be free again. for _, id := range []string{"1", "2"} { if got := len(h.inv.FreeGPUs(id)); got != 8 { t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got) } } } func TestBackoffAfterFailure(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) started := h.disp.startedShards() h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom") h.r.Reconcile(t.Context()) // reaps it h.disp.reset() h.r.Reconcile(t.Context()) if got := len(h.disp.startedShards()); got != 0 { t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied") } // Past the backoff, it tries again. h.advance(DefaultConfig().MaxBackoff) h.r.Reconcile(t.Context()) if got := len(h.disp.startedShards()); got == 0 { t.Error("the replica was never rebuilt after the backoff expired") } } func TestSuccessClearsBackoff(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) // Fail once. h.r.Reconcile(t.Context()) s := h.disp.startedShards()[0] h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom") h.r.Reconcile(t.Context()) // Then succeed. h.advance(DefaultConfig().MaxBackoff) h.disp.reset() h.r.Reconcile(t.Context()) if len(h.disp.startedShards()) == 0 { t.Fatal("no replica was started after the backoff expired") } s2 := h.disp.startedShards()[0] h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "") // A later failure should start from a short backoff, not a long streak. h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again") h.r.Reconcile(t.Context()) h.advance(2 * DefaultConfig().BaseBackoff) h.disp.reset() h.r.Reconcile(t.Context()) if got := len(h.disp.startedShards()); got == 0 { t.Error("backoff was not reset by the successful run in between") } } // TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away // must get its replicas rebuilt elsewhere. func TestLostNodeMovesReplicas(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8)) h.r.Reconcile(t.Context()) h.markReady() h.r.Reconcile(t.Context()) reps := h.r.Replicas() if len(reps) != 1 || reps[0].State != Ready { t.Fatalf("setup failed: %+v", reps) } original := reps[0].NodeIDs()[0] survivor := "2" if original == "2" { survivor = "1" } // The surviving node keeps reporting; the other goes silent. h.clock.advance(20 * time.Second) touchAt(h.inv, survivor, h.clock.now()) h.r.Reconcile(t.Context()) if n, _ := h.inv.Get(original); n.State != inventory.Unreachable { t.Errorf("silent node is %s, expected unreachable", n.State) } // Unreachable is not lost: a blip must not tear down a working replica. if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready { t.Fatalf("an unreachable node already destroyed the replica: %+v", reps) } h.clock.advance(90 * time.Second) touchAt(h.inv, survivor, h.clock.now()) h.disp.reset() h.r.Reconcile(t.Context()) // declares it lost and reaps the replica if n, _ := h.inv.Get(original); n.State != inventory.Lost { t.Fatalf("node is %s, expected lost", n.State) } // Past the backoff, the replica is rebuilt on the survivor. Only the // survivor heartbeats; the lost node stays gone. h.clock.advance(DefaultConfig().MaxBackoff) touchAt(h.inv, survivor, h.clock.now()) h.r.Reconcile(t.Context()) h.markReady() h.r.Reconcile(t.Context()) reps = h.r.Replicas() if len(reps) != 1 { t.Fatalf("expected the replica to be rebuilt, got %d", len(reps)) } if got := reps[0].NodeIDs()[0]; got != survivor { t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor) } if reps[0].State != Ready { t.Errorf("rebuilt replica is %s", reps[0].State) } } func TestHeartbeatRevivesNode(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) h.clock.advance(20 * time.Second) // deliberately without a heartbeat h.r.Reconcile(t.Context()) if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable { t.Fatalf("node is %s", n.State) } h.inv.Touch("1") if n, _ := h.inv.Get("1"); n.State != inventory.Ready { t.Errorf("a heartbeat did not revive the node: %s", n.State) } } func TestScaleDownDrainsFirst(t *testing.T) { // Draining before stopping lets in-flight requests finish instead of being // cut off mid-stream. h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) for range 2 { h.r.Reconcile(t.Context()) h.markReady() } if got := len(h.r.Replicas()); got != 2 { t.Fatalf("setup: %d replicas", got) } // Shrink the catalog to one replica. cat, err := catalog.New([]catalog.Model{testModel(1)}) if err != nil { t.Fatal(err) } h.r.cat.Store(cat) h.disp.reset() h.r.Reconcile(t.Context()) h.disp.mu.Lock() drained, stopped := len(h.disp.drained), len(h.disp.stopped) h.disp.mu.Unlock() if drained == 0 { t.Error("the excess replica was not drained") } if stopped != 0 { t.Error("a replica was stopped before it had drained") } // The next pass stops what has drained. h.r.Reconcile(t.Context()) h.disp.mu.Lock() stopped = len(h.disp.stopped) h.disp.mu.Unlock() if stopped == 0 { t.Error("the drained replica was never stopped") } if got := len(h.r.Replicas()); got != 1 { t.Errorf("expected 1 replica after scale-down, got %d", got) } } func TestUnplaceableModelExplains(t *testing.T) { // A model that cannot be placed must leave a usable explanation rather than // a silent absence. h := newHarness(t, testModel(1, catalog.Profile{ Name: "tp8", GPUClass: "h100-80g", TP: 8, }), gpuNode("1", 4)) h.r.Reconcile(t.Context()) if got := len(h.r.Replicas()); got != 0 { t.Fatalf("something was placed: %d", got) } res, ok := h.r.Explain("llama") if !ok { t.Fatal("no explanation was recorded") } exp := res.Explain() if !strings.Contains(exp, "needs 8") { t.Errorf("explanation does not say what was missing:\n%s", exp) } } func TestDispatcherFailureFailsReplica(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) h.disp.failNext = fmt.Errorf("agent unreachable") h.r.Reconcile(t.Context()) reps := h.r.Replicas() if len(reps) != 1 || reps[0].State != Failed { t.Fatalf("a failed dispatch did not fail the replica: %+v", reps) } // And the next pass must release its GPUs. h.r.Reconcile(t.Context()) if got := len(h.inv.FreeGPUs("1")); got != 8 { t.Errorf("node has %d free GPUs; the failed replica still holds some", got) } } func TestReadyReplicasFeedTheRouter(t *testing.T) { h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8)) h.r.Reconcile(t.Context()) if got := len(h.r.ReadyReplicas()); got != 0 { t.Errorf("a starting replica was offered to the router") } h.markReady() h.r.Reconcile(t.Context()) h.markReady() ready := h.r.ReadyReplicas() if len(ready) != 2 { t.Fatalf("expected 2 ready replicas, got %d", len(ready)) } for _, rep := range ready { if !strings.HasPrefix(rep.Endpoint, "http://") { t.Errorf("endpoint %q is not a URL", rep.Endpoint) } if rep.Weight <= 0 { t.Errorf("replica %s has no weight; the router needs one", rep.ID) } } } func TestPortsDoNotCollide(t *testing.T) { h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) for range 3 { h.r.Reconcile(t.Context()) h.markReady() } ports := map[int]string{} for _, rep := range h.r.Replicas() { for _, s := range rep.Shards { if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID { t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort) } ports[s.HTTPPort] = rep.ID } } } func TestReplicaStateStrings(t *testing.T) { for st, want := range map[ReplicaState]string{ Placing: "placing", Starting: "starting", Ready: "ready", Draining: "draining", Stopped: "stopped", Failed: "failed", } { if got := st.String(); got != want { t.Errorf("%d -> %q", int(st), got) } } if !Ready.Counts() || Draining.Counts() || Failed.Counts() { // Draining must not count, or a replacement is never started for it. t.Error("Counts is wrong") } } // --- adoption after a control plane restart (plan §6.5) --- func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard { a := AdoptedShard{ ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank, GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready, Model: "llama", Profile: "tp4", Pool: "none/h100-80g", } if rank == 0 { a.Endpoint = "http://10.0.0." + nodeID + ":20000" } return a } func TestAdoptRebuildsReplica(t *testing.T) { // The control plane restarted. Nothing is known, but a node reports a live // replica. Without this the reconciler would place a duplicate while the // original keeps holding GPUs. h := newHarness(t, testModel(1), gpuNode("1", 8)) unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) if len(unclaimed) != 0 { t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed) } reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("expected 1 adopted replica, got %d", len(reps)) } rep := reps[0] if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" { t.Errorf("identity wrong: %+v", rep) } if rep.State != Ready { t.Errorf("state=%s; every shard reported Ready", rep.State) } if rep.Endpoint == "" { t.Error("no endpoint; the router would have nowhere to send") } if rep.Weight <= 0 { t.Error("weight not taken from the catalog") } // The GPUs must be reclaimed, or the next placement hands them out twice. if free := len(h.inv.FreeGPUs("1")); free != 4 { t.Errorf("node has %d free GPUs, expected 4 after adoption", free) } } func TestAdoptPreventsDuplicatePlacement(t *testing.T) { // The whole point: after adopting, a reconciliation pass must see the // desired count already met. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) h.disp.reset() h.r.Reconcile(t.Context()) if got := len(h.disp.startedShards()); got != 0 { t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got) } if got := len(h.r.Replicas()); got != 1 { t.Errorf("expected 1 replica, got %d", got) } } func TestAdoptMultiNodeReplica(t *testing.T) { h := newHarness(t, testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) shards := []AdoptedShard{ adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}), adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}), } for i := range shards { shards[i].Profile = "2node" shards[i].Fabric = "ib-a" shards[i].Pool = "ib-a/h100-80g" } if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 { t.Fatalf("unclaimed: %+v", unclaimed) } reps := h.r.Replicas() if len(reps) != 1 || len(reps[0].Shards) != 2 { t.Fatalf("the two shards were not grouped into one replica: %+v", reps) } if reps[0].Fabric != "ib-a" { t.Errorf("fabric lost: %q", reps[0].Fabric) } // Both nodes' GPUs must be reclaimed. for _, id := range []string{"1", "2"} { if free := len(h.inv.FreeGPUs(id)); free != 0 { t.Errorf("node %s has %d free GPUs after adoption", id, free) } } } func TestAdoptPartialReplicaFails(t *testing.T) { // A multi-node replica missing a shard is useless: the survivors wait on // collectives that will never complete while holding GPUs. h := newHarness(t, testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}) s.Profile = "2node" s.State = supervisor.Failed h.r.Adopt("1", []AdoptedShard{s}) reps := h.r.Replicas() if len(reps) != 1 || reps[0].State != Failed { t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps) } // And the next pass releases its GPUs. h.r.Reconcile(t.Context()) if free := len(h.inv.FreeGPUs("1")); free != 8 { t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free) } } func TestAdoptUnknownModelIsUnclaimed(t *testing.T) { // The model left the catalog while the engine kept running. Nothing will // ever route to it, and it is holding GPUs. h := newHarness(t, testModel(1), gpuNode("1", 8)) s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0}) s.Model = "a-model-that-was-removed" unclaimed := h.r.Adopt("1", []AdoptedShard{s}) if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" { t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed) } if got := len(h.r.Replicas()); got != 0 { t.Errorf("a replica was created for an unknown model: %d", got) } } func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0}) s.Profile = "a-profile-that-was-removed" if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 { t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed) } } func TestAdoptIsIdempotent(t *testing.T) { // An agent reconnecting re-reports the same shards; that must not double // the allocation. h := newHarness(t, testModel(1), gpuNode("1", 8)) s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})} h.r.Adopt("1", s) h.r.Adopt("1", s) if got := len(h.r.Replicas()); got != 1 { t.Errorf("re-reporting created %d replicas", got) } if free := len(h.inv.FreeGPUs("1")); free != 4 { t.Errorf("node has %d free GPUs; the allocation was applied twice", free) } } // --- draining a node (ctl drain) --- func TestDrainingNodeMovesWorkOff(t *testing.T) { // The operator's view: a node is about to be worked on, its replicas must // end up elsewhere, and nothing may be cut off mid-request on the way. h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8)) h.r.Reconcile(t.Context()) h.markReady() h.r.Reconcile(t.Context()) reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("setup: %d replicas", len(reps)) } victim := reps[0].NodeIDs()[0] survivor := "2" if victim == "2" { survivor = "1" } h.inv.SetState(victim, inventory.Draining) h.disp.reset() h.r.Reconcile(t.Context()) // It is drained, not stopped: the engine is healthy and still serving. h.disp.mu.Lock() drained, stopped := len(h.disp.drained), len(h.disp.stopped) h.disp.mu.Unlock() if drained == 0 { t.Error("the replica on the draining node was not drained") } if stopped != 0 { t.Error("the replica was stopped before a replacement existed; in-flight requests would be cut") } // A draining replica does not count, so a replacement is placed elsewhere. if got := len(h.disp.startedShards()); got == 0 { t.Fatal("no replacement was placed") } for _, cmd := range h.disp.startedShards() { if cmd.NodeID == victim { t.Errorf("the replacement was placed back on the draining node %s", victim) } } h.markReady() h.r.Reconcile(t.Context()) // Now that the replacement serves, the original can go. h.disp.mu.Lock() stopped = len(h.disp.stopped) h.disp.mu.Unlock() if stopped == 0 { t.Error("the drained replica was never stopped once its replacement was ready") } for _, rep := range h.r.Replicas() { if rep.OnNode(victim) { t.Errorf("replica %s is still on the drained node", rep.ID) } if !slices.Contains(rep.NodeIDs(), survivor) { t.Errorf("replica %s did not move to the surviving node", rep.ID) } } } func TestDrainGraceStopsWithoutReplacement(t *testing.T) { // Draining the only node that can host a model would otherwise never // finish, leaving the node undrainable. h := newHarness(t, testModel(1), gpuNode("only", 8)) h.r.Reconcile(t.Context()) h.markReady() h.r.Reconcile(t.Context()) h.inv.SetState("only", inventory.Draining) h.r.Reconcile(t.Context()) // drains it; nowhere to place a replacement if got := h.r.Replicas(); len(got) != 1 || got[0].State != Draining { t.Fatalf("expected a draining replica, got %+v", got) } h.disp.reset() h.r.Reconcile(t.Context()) h.disp.mu.Lock() early := len(h.disp.stopped) h.disp.mu.Unlock() if early != 0 { t.Error("the replica was stopped before the grace period expired") } h.advance(DefaultConfig().DrainGrace + time.Second) h.inv.SetState("only", inventory.Draining) // advance() resets node state h.r.Reconcile(t.Context()) h.disp.mu.Lock() stopped := len(h.disp.stopped) h.disp.mu.Unlock() if stopped == 0 { t.Error("the drained replica was never stopped; the node cannot be drained") } } func TestCordonedNodeKeepsItsWork(t *testing.T) { // Cordon is not drain: it stops new placement and leaves what is running. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) h.markReady() h.r.Reconcile(t.Context()) h.inv.SetState("1", inventory.Cordoned) h.disp.reset() h.r.Reconcile(t.Context()) h.disp.mu.Lock() drained, stopped := len(h.disp.drained), len(h.disp.stopped) h.disp.mu.Unlock() if drained != 0 || stopped != 0 { t.Errorf("cordoning disturbed a running replica (drained=%d stopped=%d)", drained, stopped) } if got := h.r.Replicas(); len(got) != 1 || got[0].State != Ready { t.Errorf("the replica did not survive a cordon: %+v", got) } } func TestAdoptMultiNodeReplicaOneAgentAtATime(t *testing.T) { // How it really arrives. Every other multi-node adoption test hands both // shards to a single Adopt call, which no caller ever does: each node has // its own agent and each agent registers separately, so a two-node replica // is reported in two halves a second or so apart. // // Observed on the cluster after a control plane restart: the first half was // adopted and immediately failed for "replica has 1 of 2 shards", and when // the second node registered its half was stopped as unclaimed. big-model // never came back — it rebuilt from scratch on every restart, with // exponential backoff, forever. h := newHarness(t, testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) half := func(shardID, nodeID string, rank int) []AdoptedShard { s := adopted("llama-7", shardID, nodeID, rank, []int{0, 1, 2, 3, 4, 5, 6, 7}) s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g" return []AdoptedShard{s} } // Rank 1 first, so the endpoint arrives in the second group: whichever node // registers first is not necessarily the leader. if unclaimed := h.r.Adopt("2", half("llama-7-r1", "2", 1)); len(unclaimed) != 0 { t.Fatalf("the first half was not claimed: %+v", unclaimed) } reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("expected the half to be adopted, got %d replicas", len(reps)) } if reps[0].State == Failed { t.Fatalf("a replica still assembling was failed: %q", reps[0].Err) } if unclaimed := h.r.Adopt("1", half("llama-7-r0", "1", 0)); len(unclaimed) != 0 { t.Fatalf("the second half was stopped as unclaimed: %+v", unclaimed) } reps = h.r.Replicas() if len(reps) != 1 { t.Fatalf("the halves became %d replicas", len(reps)) } rep := reps[0] if len(rep.Shards) != 2 { t.Fatalf("the halves were not merged: %d shards", len(rep.Shards)) } if rep.State != Ready { t.Errorf("state=%s err=%q; both shards reported Ready", rep.State, rep.Err) } if rep.Endpoint == "" { t.Error("no endpoint; rank 0 arrived in the second group and was dropped") } for _, id := range []string{"1", "2"} { if free := len(h.inv.FreeGPUs(id)); free != 0 { t.Errorf("node %s has %d free GPUs; its half's allocation was lost", id, free) } } // And the reconciler must not then place a duplicate alongside it. h.disp.reset() h.r.Reconcile(t.Context()) if got := len(h.disp.startedShards()); got != 0 { t.Errorf("a duplicate was placed despite the replica being whole (%d shards started)", got) } } func TestAHalfThatNeverArrivesStillFails(t *testing.T) { // The grace period must be a deadline, not an amnesty: a replica whose // other node is simply gone has to fail so it can be replaced. h := newHarness(t, testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, StartupTimeout: time.Minute, }), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}) s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g" h.r.Adopt("1", []AdoptedShard{s}) if reps := h.r.Replicas(); reps[0].State != Starting { t.Fatalf("state=%s; the half should be waiting for its sibling", reps[0].State) } h.advance(2 * time.Minute) h.r.Reconcile(t.Context()) reps := h.r.Replicas() if len(reps) == 1 && reps[0].State != Failed { t.Errorf("state=%s; a half whose sibling never registered must fail", reps[0].State) } if free := len(h.inv.FreeGPUs("1")); free != 8 { t.Errorf("node has %d free GPUs; the abandoned half still holds them", free) } } // --- the startup settling window --- func settlingHarness(t *testing.T, m catalog.Model) (*Reconciler, *inventory.Store, *fakeDispatcher, *clock) { t.Helper() cat, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatal(err) } inv := inventory.NewStore() disp := &fakeDispatcher{} c := newClock() r := New(DefaultConfig(), catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler)) r.SetClock(c.now) return r, inv, disp, c } func addNode(inv *inventory.Store, c *clock, n inventory.Node) { inv.Upsert(n) touchAt(inv, n.ID, c.now()) } func TestPlacementWaitsForTheFleetToArrive(t *testing.T) { // A control plane that has just started knows nothing, and the first tick // runs before any agent has registered. Placing then puts everything on // whichever fraction reported first: after one restart both replicas of a // model landed on the single node that had registered, when four were // coming. r, inv, disp, c := settlingHarness(t, testModel(2)) // Nothing registered yet. r.Reconcile(t.Context()) if got := len(disp.startedShards()); got != 0 { t.Fatalf("placed %d shards on an empty fleet", got) } // One node of four. addNode(inv, c, gpuNode("1", 8)) r.Reconcile(t.Context()) if got := len(disp.startedShards()); got != 0 { t.Fatalf("placed %d shards while the fleet was still arriving", got) } // The rest arrive. for _, id := range []string{"2", "3", "4"} { addNode(inv, c, gpuNode(id, 8)) } r.Reconcile(t.Context()) if got := len(disp.startedShards()); got != 0 { t.Fatalf("placed %d shards the instant the last node appeared", got) } // Steady for long enough that a reconnect backoff would have finished: now // it may act, and on the whole fleet. c.advance(DefaultConfig().SettleFor / 2) for _, id := range []string{"1", "2", "3", "4"} { touchAt(inv, id, c.now()) } r.Reconcile(t.Context()) if got := len(disp.startedShards()); got == 0 { t.Fatal("placement never resumed after the fleet settled") } } func TestPlacementDoesNotWaitForeverForNodesThatNeverCome(t *testing.T) { // Waiting only for the count to stop changing would hang forever on a // cluster whose nodes never all come back. r, inv, disp, c := settlingHarness(t, testModel(1)) addNode(inv, c, gpuNode("1", 8)) c.advance(DefaultConfig().SettleFor) touchAt(inv, "1", c.now()) r.Reconcile(t.Context()) if got := len(disp.startedShards()); got == 0 { t.Error("placement never resumed once the window expired") } } func TestSettlingIsOnlyEverPaidOnce(t *testing.T) { // A node joining later in the day is an ordinary event, not a reason to // stop placing while the cluster is running. r, inv, disp, c := settlingHarness(t, testModel(1)) addNode(inv, c, gpuNode("1", 8)) c.advance(DefaultConfig().SettleFor) touchAt(inv, "1", c.now()) r.Reconcile(t.Context()) if len(disp.startedShards()) == 0 { t.Fatal("nothing placed after the window") } disp.reset() addNode(inv, c, gpuNode("2", 8)) if !r.fleetHasSettled() { t.Error("a node joining a running cluster restarted the settling window") } } func TestAdoptionIsNotBlockedByTheSettlingWindow(t *testing.T) { // Only placement waits. Adoption is how a restart recovers what is already // running, and delaying it would leave live engines unaccounted for exactly // when the reconciler is deciding what to build. r, inv, _, c := settlingHarness(t, testModel(1)) addNode(inv, c, gpuNode("1", 8)) unclaimed := r.Adopt("1", []AdoptedShard{ adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}), }) if len(unclaimed) != 0 { t.Fatalf("adoption was refused during settling: %+v", unclaimed) } if got := len(r.Replicas()); got != 1 { t.Fatalf("expected the replica to be adopted, got %d", got) } } func TestSettlingOutlastsAnAgentReconnectBackoff(t *testing.T) { // The failure this was measured against. After a restart node05 registered // in one second and the other four arrived seventeen seconds later, because // a disconnected agent reconnects on a backoff reaching thirty seconds. A // short steady period declared a fleet of one settled and resumed placement // while four nodes were still coming — the exact situation the window // exists to avoid, reached by being impatient about it. r, inv, disp, c := settlingHarness(t, testModel(2)) addNode(inv, c, gpuNode("first", 8)) // Fifteen seconds of one node, which a short threshold would call settled. for range 3 { c.advance(5 * time.Second) touchAt(inv, "first", c.now()) r.Reconcile(t.Context()) } if got := len(disp.startedShards()); got != 0 { t.Fatalf("placed %d shards on one node while others were backing off", got) } // The stragglers arrive, as they did at seventeen seconds. The loop runs on // a ticker, so the new count is noticed on the next pass, not the instant // the nodes appear. for _, id := range []string{"2", "3", "4"} { addNode(inv, c, gpuNode(id, 8)) } r.Reconcile(t.Context()) c.advance(DefaultConfig().SettleFor / 2) for _, id := range []string{"first", "2", "3", "4"} { touchAt(inv, id, c.now()) } r.Reconcile(t.Context()) if got := len(disp.startedShards()); got == 0 { t.Fatal("placement never resumed") } } // --- idle tracking --- func TestUsageIsRecordedAgainstReplicas(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("expected a replica, got %d", len(reps)) } id := reps[0].ID if _, known := h.r.IdleFor(id); known { t.Error("a replica nothing has used reported a known idleness") } used := h.clock.now() h.r.RecordUsage(map[string]time.Time{id: used}) h.advance(5 * time.Minute) idle, known := h.r.IdleFor(id) if !known { t.Fatal("usage was recorded but idleness is still unknown") } if idle != 5*time.Minute { t.Errorf("idle = %s, want 5m", idle) } } func TestUsageNeverMovesBackwards(t *testing.T) { // Two routers reporting, or a snapshot arriving late, must not make a busy // replica look abandoned. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) id := h.r.Replicas()[0].ID recent := h.clock.now() h.r.RecordUsage(map[string]time.Time{id: recent}) h.r.RecordUsage(map[string]time.Time{id: recent.Add(-time.Hour)}) h.advance(time.Minute) if idle, _ := h.r.IdleFor(id); idle != time.Minute { t.Errorf("idle = %s; a stale report moved last-use backwards", idle) } } func TestUsageForReplicasWeDoNotHaveIsIgnored(t *testing.T) { // The router's list and the reconciler's drift constantly on a live fleet: // a replica stops between the snapshot and the read. That is ordinary, not // an error, and must not panic or create phantom state. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) before := len(h.r.Replicas()) h.r.RecordUsage(map[string]time.Time{"a-replica-that-left": h.clock.now()}) if got := len(h.r.Replicas()); got != before { t.Errorf("replica count changed from %d to %d", before, got) } } func TestAnEmptyUsageReportLeavesEverythingAlone(t *testing.T) { // A router that has just started knows nothing, and treating that as "never // used" would make every replica look abandoned at once — exactly when a // fleet is least able to afford acting on it. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) id := h.r.Replicas()[0].ID h.r.RecordUsage(map[string]time.Time{id: h.clock.now()}) h.advance(time.Minute) h.r.RecordUsage(nil) if idle, known := h.r.IdleFor(id); !known || idle != time.Minute { t.Errorf("idle=%s known=%v; an empty report erased what was known", idle, known) } } func TestAModelAddedWhileRunningIsPlaced(t *testing.T) { // The reason the catalog stopped being fixed at startup. Adding a model was // an operator procedure with downtime: edit a file on the control plane, // restart it, wait out the settling window, watch adoption churn. None of // that was necessary — the loop already compares desired against actual // every pass, so a new catalog is simply a different desired. h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8)) h.r.Reconcile(t.Context()) if got := len(h.r.Replicas()); got != 1 { t.Fatalf("expected the original model's replica, got %d", got) } second := testModel(1) second.Name = "llama-two" next, err := catalog.New([]catalog.Model{testModel(1), second}) if err != nil { t.Fatal(err) } h.r.cat.Store(next) h.r.Reconcile(t.Context()) byModel := map[string]int{} for _, rep := range h.r.Replicas() { byModel[rep.Model]++ } if byModel["llama-two"] == 0 { t.Errorf("a model added while running was never placed: %v", byModel) } if byModel["llama"] == 0 { t.Errorf("the model that was already running was disturbed: %v", byModel) } } func TestAModelRemovedWhileRunningIsStopped(t *testing.T) { // The other direction, and the one with consequences: a replica whose model // has left the catalog holds GPUs for something nothing can ask for. first := testModel(1) second := testModel(1) second.Name = "llama-two" both, err := catalog.New([]catalog.Model{first, second}) if err != nil { t.Fatal(err) } h := newHarness(t, first, gpuNode("1", 8), gpuNode("2", 8)) h.r.cat.Store(both) h.r.Reconcile(t.Context()) h.r.Reconcile(t.Context()) if len(h.r.Replicas()) < 2 { t.Fatalf("expected both models placed, got %d", len(h.r.Replicas())) } only, err := catalog.New([]catalog.Model{first}) if err != nil { t.Fatal(err) } h.r.cat.Store(only) // Several passes: stopping is deliberate rather than immediate, and a // replica is not torn down in the same breath it is noticed. for range 4 { h.r.Reconcile(t.Context()) } for _, rep := range h.r.Replicas() { if rep.Model == "llama-two" && rep.State.Counts() { t.Errorf("a replica of a removed model is still counted: %+v", rep) } } } // --- on-demand loading and idle eviction --- func onDemandModel(idle time.Duration) catalog.Model { m := testModel(1) m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle} return m } func TestAnOnDemandModelIsNotRunningUntilItIsAskedFor(t *testing.T) { // The point of the whole feature: a cluster with more models than GPUs can // hold every model in its catalog and only the wanted ones in its cards. h := newHarness(t, onDemandModel(time.Hour), gpuNode("1", 8)) h.r.Reconcile(t.Context()) if got := len(h.r.Replicas()); got != 0 { t.Fatalf("an on-demand model was placed before anyone asked: %d replicas", got) } if !h.r.Want("llama") { t.Fatal("Want reported the model is not on-demand") } h.r.Reconcile(t.Context()) if got := len(h.r.Replicas()); got != 1 { t.Errorf("asking for the model produced %d replicas, want 1", got) } } func TestWantSaysWhenAModelDoesNotLoadOnDemand(t *testing.T) { // The caller needs to tell "being loaded" from "there is genuinely nothing // serving this", because those are different answers to give a client. h := newHarness(t, testModel(1), gpuNode("1", 8)) // min 1, not on-demand if h.r.Want("llama") { t.Error("an ordinary model was reported as loading on demand") } if h.r.Want("a-model-that-does-not-exist") { t.Error("an unknown model was reported as loading on demand") } } func TestAnIdleOnDemandModelIsStopped(t *testing.T) { // The other half. Without it, on-demand is just a slower way to fill the // cluster permanently. h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8)) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() if len(h.r.Replicas()) != 1 { t.Fatalf("the model did not start") } h.advance(11 * time.Minute) h.r.Reconcile(t.Context()) for _, rep := range h.r.Replicas() { if rep.State.Counts() { t.Errorf("a replica idle past its timeout is still counted: %+v", rep) } } } func TestUseKeepsAnOnDemandModelAlive(t *testing.T) { // Without this the timer measures time since the request that loaded the // model rather than the last one, so a busy model would be evicted exactly // IdleTimeout after it started however hard it was working. h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8)) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("the model did not start") } id := reps[0].ID // Busy for half an hour, in six-minute steps. for range 5 { h.advance(6 * time.Minute) h.r.RecordUsage(map[string]time.Time{id: h.clock.now()}) h.r.Reconcile(t.Context()) } var live int for _, rep := range h.r.Replicas() { if rep.State.Counts() { live++ } } if live != 1 { t.Errorf("a model in continuous use was evicted: %d live replicas", live) } } func TestAModelWithMaxZeroStaysOff(t *testing.T) { // min 0 max 0 is configured and deliberately not running, which is a // different and equally useful thing from "load it when wanted". m := testModel(1) m.Scaling = catalog.Scaling{Min: 0, Max: 0} h := newHarness(t, m, gpuNode("1", 8)) if h.r.Want("llama") { t.Error("a model with max 0 was treated as on-demand") } h.r.Reconcile(t.Context()) if got := len(h.r.Replicas()); got != 0 { t.Errorf("a model switched off was placed: %d replicas", got) } } func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) { // Every model that exists today. A model with min 1 stays up whether anyone // uses it or not, because that is what min 1 means. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) // Ready, or the startup timeout is what removes it twenty-four hours later // and the test proves nothing about idleness. h.markReady() if len(h.r.Replicas()) != 1 { t.Fatal("the model did not start") } h.advance(24 * time.Hour) h.r.Reconcile(t.Context()) var live int for _, rep := range h.r.Replicas() { if rep.State.Counts() { live++ } } if live != 1 { t.Errorf("an always-on model was evicted for being idle: %d live", live) } } // --- eviction under pressure --- // twoOnDemandModels builds a catalog where each model fills the whole node, so // the second can only run if the first stops. func twoOnDemandModels(t *testing.T) *catalog.Catalog { t.Helper() mk := func(name string) catalog.Model { m := testModel(1) m.Name = name m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: time.Hour} return m } built, err := catalog.New([]catalog.Model{mk("llama"), mk("second")}) if err != nil { t.Fatal(err) } return built } func TestAWantedModelEvictsAnIdleOneWhenTheClusterIsFull(t *testing.T) { // On-demand loading works right up to the moment the cluster is full and // then stops meaning anything: the model somebody just asked for cannot be // placed, and the cards are held by one nobody has asked for in an hour. // That is the only situation in which any of this matters. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.cat.Store(twoOnDemandModels(t)) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() first := h.r.Replicas() if len(first) != 1 { t.Fatalf("the first model did not start: %d replicas", len(first)) } h.r.RecordUsage(map[string]time.Time{first[0].ID: h.clock.now()}) // It goes quiet. Past the eviction grace and the minimum lifetime, so it is // genuinely a candidate rather than merely recent. h.advance(30 * time.Minute) // Asked for directly, rather than through a Reconcile pass, because two // on-demand models both wanting the only free node is a race the scheduler // resolves in whatever order the catalog iterates — and what is being // tested here is that room gets made, not who wins. second := mustResolve(t, h, "second") if !h.r.makeRoomFor(t.Context(), second) { t.Fatal("no room was made for a model that could not be placed") } for _, rep := range h.r.Replicas() { if rep.Model == "llama" && rep.State.Counts() { t.Error("the idle replica was not evicted") } } // And its GPUs are back, which is what the wanted model needed. if free := len(h.inv.FreeGPUs("1")); free != 4 { t.Errorf("node has %d free GPUs after the eviction, want 4", free) } } func TestPlacementFailureTriggersEviction(t *testing.T) { // The wiring: scaleUp is what notices there is nowhere to put a model, and // is the only place that knows to try making room. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.cat.Store(twoOnDemandModels(t)) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("setup: %d replicas", len(reps)) } h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) h.advance(30 * time.Minute) // The full node has no room, so placing this must free some. h.r.scaleUp(t.Context(), mustResolve(t, h, "second"), nil) if free := len(h.inv.FreeGPUs("1")); free == 0 { t.Error("a placement failure did not make room") } } func TestAnAlwaysOnModelIsNeverEvicted(t *testing.T) { // min 1 is a promise that it stays up. Quietly breaking it to serve another // request would make every guarantee in the catalog conditional on load. alwaysOn := testModel(1) // min 1 onDemand := testModel(1) onDemand.Name = "second" onDemand.Scaling = catalog.Scaling{Min: 0, Max: 1} built, err := catalog.New([]catalog.Model{alwaysOn, onDemand}) if err != nil { t.Fatal(err) } h := newHarness(t, alwaysOn, gpuNode("1", 4)) h.r.cat.Store(built) h.r.Reconcile(t.Context()) h.markReady() reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("the always-on model did not start") } h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) h.advance(24 * time.Hour) if got := h.r.evictable("second"); len(got) != 0 { t.Errorf("an always-on model was offered for eviction: %+v", got) } } func TestAReplicaNothingHasReportedIsNeverEvicted(t *testing.T) { // Unknown is not idle. After a router restart that is every replica, and // evicting on the strength of a fact nobody has reported would empty the // cluster at exactly the wrong moment. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.cat.Store(twoOnDemandModels(t)) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() h.advance(24 * time.Hour) // idle by any measure, but nothing has said so if got := h.r.evictable("second"); len(got) != 0 { t.Errorf("a replica with no reported usage was offered for eviction: %+v", got) } } func TestAMultiNodeReplicaIsLeftAlone(t *testing.T) { // Evicting one frees several machines at once and takes tens of minutes to // rebuild — a decision worth making deliberately rather than as a side // effect of one request. m := testModel(1, catalog.Profile{ Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, }) m.Scaling = catalog.Scaling{Min: 0, Max: 1} other := testModel(1) other.Name = "second" other.Scaling = catalog.Scaling{Min: 0, Max: 1} built, err := catalog.New([]catalog.Model{m, other}) if err != nil { t.Fatal(err) } h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) h.r.cat.Store(built) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() reps := h.r.Replicas() if len(reps) != 1 || len(reps[0].Shards) != 2 { t.Fatalf("expected a two-node replica, got %+v", reps) } h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) h.advance(24 * time.Hour) if got := h.r.evictable("second"); len(got) != 0 { t.Errorf("a multi-node replica was offered for eviction: %+v", got) } } func TestOnlyOneReplicaIsEvictedPerPass(t *testing.T) { // Freeing everything that qualifies would empty a cluster to satisfy a // request that may only have needed one node. h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4)) h.r.cat.Store(twoOnDemandModels(t)) h.r.Want("llama") for range 3 { h.r.Reconcile(t.Context()) h.markReady() } reps := h.r.Replicas() used := map[string]time.Time{} for _, rep := range reps { used[rep.ID] = h.clock.now() } h.r.RecordUsage(used) h.advance(time.Hour) before := len(h.r.evictable("second")) if before < 1 { t.Skip("nothing evictable to begin with") } h.r.makeRoomFor(t.Context(), mustResolve(t, h, "second")) var stopped int for _, rep := range h.r.Replicas() { if !rep.State.Counts() { stopped++ } } if stopped > 1 { t.Errorf("%d replicas were stopped in one pass", stopped) } } func mustResolve(t *testing.T, h *harness, name string) catalog.Model { t.Helper() m, ok := h.r.cat.Load().Resolve(name) if !ok { t.Fatalf("model %q is not in the catalog", name) } return m } func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) { // What stops two models on a cluster with room for one from taking turns // evicting each other, paying a model load each time. A load is expensive // enough that waiting is strictly better than winning. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.cat.Store(twoOnDemandModels(t)) h.r.Want("llama") h.r.Reconcile(t.Context()) h.markReady() reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("setup: %d replicas", len(reps)) } // Used once, then idle well past the eviction grace — but only just // started, which is the case this protects. h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) h.advance(DefaultConfig().EvictionGrace + time.Minute) if got := h.r.evictable("second"); len(got) != 0 { t.Errorf("a replica younger than the minimum lifetime was offered: %+v", got) } // Once it has been up long enough, it is fair game. h.advance(DefaultConfig().MinLifetime) if got := h.r.evictable("second"); len(got) == 0 { t.Error("a long-running idle replica was never offered for eviction") } } func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) { // The fingerprint has to leave the control plane, or it cannot come back // after a restart — and a restart is exactly when the configuration is // likely to have been edited. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Reconcile(t.Context()) started := h.disp.startedShards() if len(started) == 0 { t.Fatal("nothing was started") } // From the catalog rather than the literal, because the catalog fills in // defaults and the fingerprint is of the profile as it is actually run. m, ok := h.r.cat.Load().Resolve("llama") if !ok { t.Fatal("the model is not in the catalog") } want := m.Profiles[0].Fingerprint() for _, cmd := range started { if cmd.ProfileFP != want { t.Errorf("shard %s carries %q, expected %q", cmd.ShardID, cmd.ProfileFP, want) } } } func TestAnAdoptedReplicaKeepsTheProfileVersionItWasStartedWith(t *testing.T) { // Adoption used to leave this empty, which made every replica read as // unknown after a control plane restart — so an edited profile could never // be noticed on the machines that were already running. h := newHarness(t, testModel(1), gpuNode("1", 8)) shard := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}) shard.ProfileFP = "1122334455667788" h.r.Adopt("1", []AdoptedShard{shard}) reps := h.r.Replicas() if len(reps) != 1 { t.Fatalf("expected 1 adopted replica, got %d", len(reps)) } if reps[0].ProfileFP != "1122334455667788" { t.Errorf("the agent's answer was dropped: %q", reps[0].ProfileFP) } } func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}) // Empty, not filled in from the catalog. Copying the current profile here // would make the replica claim to match whatever the catalog says next, // which is the opposite of what the field is for. if fp := h.r.Replicas()[0].ProfileFP; fp != "" { t.Errorf("an unreported fingerprint should stay empty, got %q", fp) } } func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) { // Engines listen on every address a node has. Started without a key, any // machine that could reach the port reached the model past the gateway's // authentication, quotas and accounting — checked from a neighbouring node // on the live cluster. h := newHarness(t, testModel(1), gpuNode("1", 8)) h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id } h.r.Reconcile(t.Context()) started := h.disp.startedShards() if len(started) == 0 { t.Fatal("nothing was started") } for _, c := range started { if c.Spec.APIKey != "key-for-"+c.ReplicaID { t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey) } } } func replicaByID(h *harness, id string) (Replica, bool) { for _, rep := range h.r.Replicas() { if rep.ID == id { return rep, true } } return Replica{}, false } func TestARestartedReplicaServesUntilItsReplacementIsReady(t *testing.T) { // Restarting is how a running engine picks up a key, an edited profile or // a new engine version. Stopping first would leave a one-replica model // unserved for as long as its replacement takes to load. h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4)) h.r.Reconcile(t.Context()) h.markReady() old := h.r.ReadyReplicas()[0].ID if err := h.r.Restart(old); err != nil { t.Fatal(err) } h.r.Reconcile(t.Context()) if rep, _ := replicaByID(h, old); rep.State != Ready || h.r.RestartState(old) != "replacing" { t.Fatalf("the old replica stopped serving before its replacement was up: %+v", rep) } if n := len(h.r.Replicas()); n != 2 { t.Fatalf("no replacement was placed beside it: %d replicas", n) } h.markReady() h.r.Reconcile(t.Context()) h.r.Reconcile(t.Context()) if _, still := replicaByID(h, old); still { if rep, _ := replicaByID(h, old); rep.State != Draining && rep.State != Stopped { t.Errorf("the old replica is %s after its replacement became ready", rep.State) } } if got := len(h.r.ReadyReplicas()); got != 1 { t.Errorf("%d ready replicas at the end; the model asks for 1", got) } } func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) { // One node, room for one replica: the replacement can only start once the // old one has gone. It is the only thing in the way, so it goes now — // waiting out a grace period first changed nothing but the wait, which // in the field read as a restart that never happened. h := newHarness(t, testModel(1), gpuNode("1", 4)) h.r.Reconcile(t.Context()) h.markReady() old := h.r.ReadyReplicas()[0].ID if err := h.r.Restart(old); err != nil { t.Fatal(err) } h.r.Reconcile(t.Context()) if rep, ok := replicaByID(h, old); ok && rep.State == Ready { t.Fatal("still serving with no replacement possible beside it") } for i := 0; i < 3; i++ { h.r.Reconcile(t.Context()) h.markReady() } ready := h.r.ReadyReplicas() if len(ready) != 1 || ready[0].ID == old { t.Errorf("expected one new replica in its place, got %+v", ready) } } func TestARestartWhoseReplacementFitsNowhereIsGivenUp(t *testing.T) { // The profile was edited to need more cards than the node has: stopping // the old replica would leave the model with nothing at all. m := testModel(1) h := newHarness(t, m, gpuNode("1", 4)) h.r.Reconcile(t.Context()) h.markReady() old := h.r.ReadyReplicas()[0].ID m.Profiles[0].TP, m.Profiles[0].GPUsPerNode = 8, 8 h.setModels(t, m) if err := h.r.Restart(old); err != nil { t.Fatal(err) } for range 3 { h.r.Reconcile(t.Context()) h.advance(h.r.cfg.DrainGrace + time.Second) } if rep, ok := replicaByID(h, old); !ok || rep.State != Ready { t.Fatalf("the only replica was stopped for a replacement that fits nowhere: %+v", rep) } if st := h.r.RestartState(old); !strings.HasPrefix(st, "refused: ") || !strings.Contains(st, "fits on no node") { t.Errorf("restart state %q should say it was given up, and why", st) } // Asked again, it is tried again. if err := h.r.Restart(old); err != nil || h.r.RestartState(old) != "queued" { t.Errorf("a new restart did not clear the refusal: %q, %v", h.r.RestartState(old), err) } } func TestARestartAskedInPlaceStopsFirst(t *testing.T) { // Room beside it, but in place was asked for: an engine that must not // run twice, or memory somebody knows is short. h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4)) h.r.Reconcile(t.Context()) h.markReady() old := h.r.ReadyReplicas()[0].ID if err := h.r.RestartInPlace(old); err != nil { t.Fatal(err) } h.r.Reconcile(t.Context()) if rep, ok := replicaByID(h, old); ok && rep.State == Ready { t.Fatalf("asked in place, it kept serving beside its replacement: %+v", rep) } } func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) { h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) for i := 0; i < 3; i++ { h.r.Reconcile(t.Context()) h.markReady() } var ids []string for _, rep := range h.r.ReadyReplicas() { ids = append(ids, rep.ID) } if err := h.r.Restart(ids...); err != nil { t.Fatal(err) } h.r.Reconcile(t.Context()) states := []string{h.r.RestartState(ids[0]), h.r.RestartState(ids[1])} if !slices.Contains(states, "replacing") || !slices.Contains(states, "queued") { t.Errorf("expected one replacing and one queued, got %v", states) } if err := h.r.Restart("no-such"); err == nil { t.Error("an unknown replica was accepted") } } // failRepeatedly starts and fails the model's replica n times over. func failRepeatedly(t *testing.T, h *harness, n int, why string) { t.Helper() for i := 0; i < n; i++ { h.disp.reset() h.r.Reconcile(t.Context()) started := h.disp.startedShards() if len(started) == 0 { return // parked, or still in backoff } s := started[0] h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, why) h.r.Reconcile(t.Context()) h.advance(DefaultConfig().MaxBackoff + time.Second) } } func TestAModelThatAlwaysFailsIsParkedNotRetriedForever(t *testing.T) { // kev-0.5b: an architecture the node's llama.cpp did not know, started // 155 times in eight hours, the same error every time. h := newHarness(t, testModel(1), gpuNode("1", 8)) failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'") h.disp.reset() h.advance(DefaultConfig().MaxBackoff + time.Second) h.r.Reconcile(t.Context()) if n := len(h.disp.startedShards()); n != 0 { t.Fatalf("a parked model was started again (%d shards)", n) } var mh ModelHealth for _, x := range h.r.ModelHealth() { if x.Model == "llama" { mh = x } } if !mh.Parked || !strings.Contains(mh.Reason, "ggmlc") || mh.Failures < DefaultConfig().ParkAfter { t.Errorf("health %+v", mh) } if why, no := h.r.Unservable("llama"); !no || !strings.Contains(why, "ggmlc") { t.Errorf("a request would not be told why: %q %v", why, no) } // Retried by the operator once fixed. if !h.r.Retry("llama") { t.Error("Retry did not report the model as parked") } h.r.Reconcile(t.Context()) if len(h.disp.startedShards()) == 0 { t.Error("a retried model was not placed") } } func TestChangingAParkedModelsDefinitionTriesItAgain(t *testing.T) { h := newHarness(t, testModel(1), gpuNode("1", 8)) failRepeatedly(t, h, DefaultConfig().ParkAfter, "boom") fixed := testModel(1) fixed.Source = "/models/llama-fixed" cat, err := catalog.New([]catalog.Model{fixed}) if err != nil { t.Fatal(err) } h.r.cat.Store(cat) h.disp.reset() h.r.Reconcile(t.Context()) if len(h.disp.startedShards()) == 0 { t.Error("a model whose definition changed stayed parked") } } func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) { var buf bytes.Buffer h := newHarness(t, testModel(1), gpuNode("1", 1)) // tp4 needs 4 cards h.r.log = slog.New(slog.NewTextHandler(&buf, nil)) for i := 0; i < 20; i++ { h.r.Reconcile(t.Context()) h.advance(5 * time.Second) } if n := strings.Count(buf.String(), "no placement for model"); n != 1 { t.Errorf("logged %d times over 20 passes", n) } h.advance(placementLogEvery) h.r.Reconcile(t.Context()) if n := strings.Count(buf.String(), "no placement for model"); n != 2 { t.Errorf("not repeated after %s: %d", placementLogEvery, n) } var mh ModelHealth for _, x := range h.r.ModelHealth() { mh = x } if !mh.Unplaceable || mh.Desired != 1 { t.Errorf("health %+v", mh) } } func liveCount(h *harness) int { n := 0 for _, r := range h.r.Replicas() { if r.State.Counts() { n++ } } return n } func TestLoadAddsReplicasUpToMaxAndQuietTakesThemBack(t *testing.T) { m := testModel(1) m.Scaling.Max = 3 h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8)) h.r.Reconcile(t.Context()) h.markReady() busy := map[string]Load{"llama": {Replicas: 1, Running: 4, Waiting: 3}} // A spike shorter than ScaleUpAfter adds nothing. h.r.RecordLoad(busy) h.advance(30 * time.Second) h.r.RecordLoad(busy) h.r.Reconcile(t.Context()) if n := liveCount(h); n != 1 { t.Fatalf("%d replicas after 30s of load", n) } // Held past it, one more. h.advance(31 * time.Second) h.r.RecordLoad(busy) h.r.Reconcile(t.Context()) if n := liveCount(h); n != 2 { t.Fatalf("%d replicas after a minute of load; expected 2", n) } // Not another while that one is still starting. h.advance(2 * time.Minute) h.r.RecordLoad(busy) h.r.Reconcile(t.Context()) if n := liveCount(h); n != 2 { t.Fatalf("stacked a replica on one still starting: %d", n) } // Once it is up and load holds, the third — and never a fourth. h.markReady() for i := 0; i < 5; i++ { h.r.RecordLoad(busy) h.advance(61 * time.Second) h.r.RecordLoad(busy) h.r.Reconcile(t.Context()) h.markReady() } if n := liveCount(h); n != 3 { t.Fatalf("%d replicas under sustained load; max is 3", n) } // Quiet: back down one step per ScaleDownAfter, to min and no further. quiet := map[string]Load{"llama": {Replicas: 3}} for i := 0; i < 6; i++ { h.r.RecordLoad(quiet) h.advance(DefaultConfig().ScaleDownAfter + time.Second) h.r.RecordLoad(quiet) for j := 0; j < 3; j++ { h.r.Reconcile(t.Context()) } } if n := liveCount(h); n != 1 { t.Errorf("%d replicas after a long quiet spell; min is 1", n) } }