Files
llmgw/internal/control/reconciler/reconciler_test.go
T
Ilker ManapandClaude Opus 5 aef1c143c5 Let go of shards an agent says it is not running
Adoption handled "the agent has something we do not know about". The mirror
image was missing: "we think the agent has something it does not".

Both follow from the same rule — the fleet is authoritative about what is
running (plan §6.5) — and without the second half an engine that dies while
its agent survives leaves the control plane believing a replica is Ready
forever. Measured on the cluster: the engine was killed, the agent restarted
and registered with nothing running, and the replica stayed Ready pointing at
a dead port for two and a half minutes until I stopped watching. Traffic
survived, because the breaker took the endpoint out and retries covered it —
24 requests, none failed — so the only symptom was a cluster quietly running a
third short.

A registration follows a fresh connection, so anything dispatched over a
previous one is not running by definition; there is no window where a shard is
legitimately started but unreported.

The previous commit's disown rule was too broad, which this turned up: when a
known replica had a live shard it disowned everything the agent reported for
it, including the shards we already knew. A reconnecting agent would have been
told to stop the replica that was serving. It now disowns only what it does
not recognise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 10:30:49 +02:00

901 lines
26 KiB
Go

package reconciler
import (
"context"
"fmt"
"log/slog"
"slices"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// fakeDispatcher records commands instead of sending them.
type fakeDispatcher struct {
mu sync.Mutex
started []ShardCommand
stopped []string
drained []string
failNext error
}
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.failNext != nil {
err := d.failNext
d.failNext = nil
return err
}
d.started = append(d.started, cmd)
return nil
}
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.stopped = append(d.stopped, shardID)
return nil
}
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.drained = append(d.drained, shardID)
return nil
}
func (d *fakeDispatcher) startedShards() []ShardCommand {
d.mu.Lock()
defer d.mu.Unlock()
return slices.Clone(d.started)
}
func (d *fakeDispatcher) reset() {
d.mu.Lock()
defer d.mu.Unlock()
d.started, d.stopped, d.drained = nil, nil, nil
}
// clock lets tests move time without sleeping.
type clock struct {
mu sync.Mutex
t time.Time
}
func newClock() *clock {
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) advance(d time.Duration) {
c.mu.Lock()
c.t = c.t.Add(d)
c.mu.Unlock()
}
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
for i := range n {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
})
}
for _, f := range fabrics {
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
})
}
return nd
}
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
if len(profiles) == 0 {
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
}
return catalog.Model{
Name: "llama", Source: "/models/llama",
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
}
}
type harness struct {
r *Reconciler
inv *inventory.Store
disp *fakeDispatcher
clock *clock
}
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
t.Helper()
cat, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatal(err)
}
inv := inventory.NewStore()
c := newClock()
for _, n := range nodes {
inv.Upsert(n)
inv.Touch(n.ID)
}
disp := &fakeDispatcher{}
cfg := DefaultConfig()
cfg.UnreachableAfter = 15 * time.Second
cfg.LostAfter = 60 * time.Second
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
// Nodes were just registered, so their LastSeen is wall-clock now, which is
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
for _, n := range nodes {
touchAt(inv, n.ID, c.now())
}
return &harness{r: r, inv: inv, disp: disp, clock: c}
}
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
func touchAt(inv *inventory.Store, id string, at time.Time) {
n, ok := inv.Get(id)
if !ok {
return
}
n.LastSeen = at
n.State = inventory.Ready
inv.Upsert(n)
// Upsert stamps LastSeen with the wall clock, so set it again directly.
inv.SetLastSeen(id, at)
}
// advance moves the clock and keeps every node heartbeating.
//
// Without this, advancing time past UnreachableAfter makes every node look
// silent and placement stops — which is correct behaviour, but not what the
// backoff tests are trying to exercise.
func (h *harness) advance(d time.Duration) {
h.clock.advance(d)
for _, n := range h.inv.Nodes() {
touchAt(h.inv, n.ID, h.clock.now())
}
}
// markReady walks every started shard to Ready.
func (h *harness) markReady() {
for _, cmd := range h.disp.startedShards() {
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
}
}
func TestPlacesMinimumReplicas(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.markReady()
reps := h.r.Replicas()
if len(reps) != 2 {
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
}
for _, rep := range reps {
if rep.State != Ready {
t.Errorf("replica %s is %s", rep.ID, rep.State)
}
if rep.Endpoint == "" {
t.Errorf("replica %s has no endpoint", rep.ID)
}
}
}
func TestOneReplicaPerPass(t *testing.T) {
// Placement depends on what is already allocated, so starting several from
// one inventory snapshot would hand them the same GPUs.
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 1 {
t.Fatalf("one pass created %d replicas", got)
}
seen := map[int]string{}
for range 3 {
h.markReady()
h.r.Reconcile(t.Context())
}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
for _, g := range s.GPUIndices {
key := g
if prev, dup := seen[key]; dup && prev != s.NodeID {
continue // same index on a different node is fine
}
seen[key] = s.NodeID
}
}
}
if got := len(h.r.Replicas()); got != 3 {
t.Errorf("expected 3 replicas, got %d", got)
}
}
func TestSpreadsAcrossNodes(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
nodes := map[string]struct{}{}
for _, rep := range h.r.Replicas() {
for _, n := range rep.NodeIDs() {
nodes[n] = struct{}{}
}
}
if len(nodes) != 2 {
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
}
}
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
// Spreading is a preference, not a rule: a second replica on one node still
// survives an engine crash, which is the common failure.
h := newHarness(t, testModel(2), gpuNode("only", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
if got := len(h.r.Replicas()); got != 2 {
t.Fatalf("expected 2 replicas on the single node, got %d", got)
}
}
func TestShardOrderLeaderFirst(t *testing.T) {
// Both engines put the rendezvous point on rank 0 — SGLang's dist-init
// address and vLLM's Ray head — so a worker started first has nothing to
// connect to and spends its startup budget retrying.
//
// This test previously asserted the opposite, on the theory that the head
// should wait for its workers. That is backwards: vLLM's server blocks
// until the Ray cluster has the GPUs it needs, so the leader can start
// immediately and waits on its own.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
if len(started) != 2 {
t.Fatalf("expected 2 shards, got %d", len(started))
}
if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 {
t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+
"point and must start first",
started[0].Spec.Rank, started[1].Spec.Rank)
}
// Every rank must be given the same rendezvous point.
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
t.Errorf("ranks got different leader addresses: %q and %q",
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
}
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
t.Error("ranks got different leader ports")
}
for _, cmd := range started {
if len(cmd.Spec.HCAs) == 0 {
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
}
}
}
func TestFailedShardFailsWholeReplica(t *testing.T) {
// One dead shard makes the others useless: they wait on collectives that
// will never complete while holding GPUs (plan §5.4).
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
// One shard ready, the other dead.
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
h.r.Reconcile(t.Context())
// Both shards must have been stopped, not just the failed one.
h.disp.mu.Lock()
stopped := slices.Clone(h.disp.stopped)
h.disp.mu.Unlock()
if len(stopped) != 2 {
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
}
// And the GPUs must be free again.
for _, id := range []string{"1", "2"} {
if got := len(h.inv.FreeGPUs(id)); got != 8 {
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
}
}
}
func TestBackoffAfterFailure(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
h.r.Reconcile(t.Context()) // reaps it
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
}
// Past the backoff, it tries again.
h.advance(DefaultConfig().MaxBackoff)
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got == 0 {
t.Error("the replica was never rebuilt after the backoff expired")
}
}
func TestSuccessClearsBackoff(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
// Fail once.
h.r.Reconcile(t.Context())
s := h.disp.startedShards()[0]
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
h.r.Reconcile(t.Context())
// Then succeed.
h.advance(DefaultConfig().MaxBackoff)
h.disp.reset()
h.r.Reconcile(t.Context())
if len(h.disp.startedShards()) == 0 {
t.Fatal("no replica was started after the backoff expired")
}
s2 := h.disp.startedShards()[0]
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
// A later failure should start from a short backoff, not a long streak.
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
h.r.Reconcile(t.Context())
h.advance(2 * DefaultConfig().BaseBackoff)
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got == 0 {
t.Error("backoff was not reset by the successful run in between")
}
}
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
// must get its replicas rebuilt elsewhere.
func TestLostNodeMovesReplicas(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Ready {
t.Fatalf("setup failed: %+v", reps)
}
original := reps[0].NodeIDs()[0]
survivor := "2"
if original == "2" {
survivor = "1"
}
// The surviving node keeps reporting; the other goes silent.
h.clock.advance(20 * time.Second)
touchAt(h.inv, survivor, h.clock.now())
h.r.Reconcile(t.Context())
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
t.Errorf("silent node is %s, expected unreachable", n.State)
}
// Unreachable is not lost: a blip must not tear down a working replica.
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
}
h.clock.advance(90 * time.Second)
touchAt(h.inv, survivor, h.clock.now())
h.disp.reset()
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
t.Fatalf("node is %s, expected lost", n.State)
}
// Past the backoff, the replica is rebuilt on the survivor. Only the
// survivor heartbeats; the lost node stays gone.
h.clock.advance(DefaultConfig().MaxBackoff)
touchAt(h.inv, survivor, h.clock.now())
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps = h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
}
if got := reps[0].NodeIDs()[0]; got != survivor {
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
}
if reps[0].State != Ready {
t.Errorf("rebuilt replica is %s", reps[0].State)
}
}
func TestHeartbeatRevivesNode(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
h.r.Reconcile(t.Context())
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
t.Fatalf("node is %s", n.State)
}
h.inv.Touch("1")
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
t.Errorf("a heartbeat did not revive the node: %s", n.State)
}
}
func TestScaleDownDrainsFirst(t *testing.T) {
// Draining before stopping lets in-flight requests finish instead of being
// cut off mid-stream.
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
if got := len(h.r.Replicas()); got != 2 {
t.Fatalf("setup: %d replicas", got)
}
// Shrink the catalog to one replica.
cat, err := catalog.New([]catalog.Model{testModel(1)})
if err != nil {
t.Fatal(err)
}
h.r.cat = cat
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained == 0 {
t.Error("the excess replica was not drained")
}
if stopped != 0 {
t.Error("a replica was stopped before it had drained")
}
// The next pass stops what has drained.
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
stopped = len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped")
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica after scale-down, got %d", got)
}
}
func TestUnplaceableModelExplains(t *testing.T) {
// A model that cannot be placed must leave a usable explanation rather than
// a silent absence.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "tp8", GPUClass: "h100-80g", TP: 8,
}), gpuNode("1", 4))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 0 {
t.Fatalf("something was placed: %d", got)
}
res, ok := h.r.Explain("llama")
if !ok {
t.Fatal("no explanation was recorded")
}
exp := res.Explain()
if !strings.Contains(exp, "needs 8") {
t.Errorf("explanation does not say what was missing:\n%s", exp)
}
}
func TestDispatcherFailureFailsReplica(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.disp.failNext = fmt.Errorf("agent unreachable")
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
}
// And the next pass must release its GPUs.
h.r.Reconcile(t.Context())
if got := len(h.inv.FreeGPUs("1")); got != 8 {
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
}
}
func TestReadyReplicasFeedTheRouter(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.ReadyReplicas()); got != 0 {
t.Errorf("a starting replica was offered to the router")
}
h.markReady()
h.r.Reconcile(t.Context())
h.markReady()
ready := h.r.ReadyReplicas()
if len(ready) != 2 {
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
}
for _, rep := range ready {
if !strings.HasPrefix(rep.Endpoint, "http://") {
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
}
if rep.Weight <= 0 {
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
}
}
}
func TestPortsDoNotCollide(t *testing.T) {
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
for range 3 {
h.r.Reconcile(t.Context())
h.markReady()
}
ports := map[int]string{}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
}
ports[s.HTTPPort] = rep.ID
}
}
}
func TestReplicaStateStrings(t *testing.T) {
for st, want := range map[ReplicaState]string{
Placing: "placing", Starting: "starting", Ready: "ready",
Draining: "draining", Stopped: "stopped", Failed: "failed",
} {
if got := st.String(); got != want {
t.Errorf("%d -> %q", int(st), got)
}
}
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
// Draining must not count, or a replacement is never started for it.
t.Error("Counts is wrong")
}
}
// --- adoption after a control plane restart (plan §6.5) ---
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
a := AdoptedShard{
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
}
if rank == 0 {
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
}
return a
}
func TestAdoptRebuildsReplica(t *testing.T) {
// The control plane restarted. Nothing is known, but a node reports a live
// replica. Without this the reconciler would place a duplicate while the
// original keeps holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
if len(unclaimed) != 0 {
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
}
rep := reps[0]
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
t.Errorf("identity wrong: %+v", rep)
}
if rep.State != Ready {
t.Errorf("state=%s; every shard reported Ready", rep.State)
}
if rep.Endpoint == "" {
t.Error("no endpoint; the router would have nowhere to send")
}
if rep.Weight <= 0 {
t.Error("weight not taken from the catalog")
}
// The GPUs must be reclaimed, or the next placement hands them out twice.
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
}
}
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
// The whole point: after adopting, a reconciliation pass must see the
// desired count already met.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica, got %d", got)
}
}
func TestAdoptMultiNodeReplica(t *testing.T) {
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
shards := []AdoptedShard{
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
}
for i := range shards {
shards[i].Profile = "2node"
shards[i].Fabric = "ib-a"
shards[i].Pool = "ib-a/h100-80g"
}
if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 {
t.Fatalf("unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 || len(reps[0].Shards) != 2 {
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
}
if reps[0].Fabric != "ib-a" {
t.Errorf("fabric lost: %q", reps[0].Fabric)
}
// Both nodes' GPUs must be reclaimed.
for _, id := range []string{"1", "2"} {
if free := len(h.inv.FreeGPUs(id)); free != 0 {
t.Errorf("node %s has %d free GPUs after adoption", id, free)
}
}
}
func TestAdoptPartialReplicaFails(t *testing.T) {
// A multi-node replica missing a shard is useless: the survivors wait on
// collectives that will never complete while holding GPUs.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
s.Profile = "2node"
s.State = supervisor.Failed
h.r.Adopt("1", []AdoptedShard{s})
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
}
// And the next pass releases its GPUs.
h.r.Reconcile(t.Context())
if free := len(h.inv.FreeGPUs("1")); free != 8 {
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
}
}
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
// The model left the catalog while the engine kept running. Nothing will
// ever route to it, and it is holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
s.Model = "a-model-that-was-removed"
unclaimed := h.r.Adopt("1", []AdoptedShard{s})
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
}
if got := len(h.r.Replicas()); got != 0 {
t.Errorf("a replica was created for an unknown model: %d", got)
}
}
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
s.Profile = "a-profile-that-was-removed"
if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 {
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
}
}
func TestAdoptIsIdempotent(t *testing.T) {
// An agent reconnecting re-reports the same shards; that must not double
// the allocation.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
h.r.Adopt("1", s)
h.r.Adopt("1", s)
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("re-reporting created %d replicas", got)
}
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
}
}
// --- draining a node (ctl drain) ---
func TestDrainingNodeMovesWorkOff(t *testing.T) {
// The operator's view: a node is about to be worked on, its replicas must
// end up elsewhere, and nothing may be cut off mid-request on the way.
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("setup: %d replicas", len(reps))
}
victim := reps[0].NodeIDs()[0]
survivor := "2"
if victim == "2" {
survivor = "1"
}
h.inv.SetState(victim, inventory.Draining)
h.disp.reset()
h.r.Reconcile(t.Context())
// It is drained, not stopped: the engine is healthy and still serving.
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained == 0 {
t.Error("the replica on the draining node was not drained")
}
if stopped != 0 {
t.Error("the replica was stopped before a replacement existed; in-flight requests would be cut")
}
// A draining replica does not count, so a replacement is placed elsewhere.
if got := len(h.disp.startedShards()); got == 0 {
t.Fatal("no replacement was placed")
}
for _, cmd := range h.disp.startedShards() {
if cmd.NodeID == victim {
t.Errorf("the replacement was placed back on the draining node %s", victim)
}
}
h.markReady()
h.r.Reconcile(t.Context())
// Now that the replacement serves, the original can go.
h.disp.mu.Lock()
stopped = len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped once its replacement was ready")
}
for _, rep := range h.r.Replicas() {
if rep.OnNode(victim) {
t.Errorf("replica %s is still on the drained node", rep.ID)
}
if !slices.Contains(rep.NodeIDs(), survivor) {
t.Errorf("replica %s did not move to the surviving node", rep.ID)
}
}
}
func TestDrainGraceStopsWithoutReplacement(t *testing.T) {
// Draining the only node that can host a model would otherwise never
// finish, leaving the node undrainable.
h := newHarness(t, testModel(1), gpuNode("only", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.inv.SetState("only", inventory.Draining)
h.r.Reconcile(t.Context()) // drains it; nowhere to place a replacement
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Draining {
t.Fatalf("expected a draining replica, got %+v", got)
}
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
early := len(h.disp.stopped)
h.disp.mu.Unlock()
if early != 0 {
t.Error("the replica was stopped before the grace period expired")
}
h.advance(DefaultConfig().DrainGrace + time.Second)
h.inv.SetState("only", inventory.Draining) // advance() resets node state
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
stopped := len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped; the node cannot be drained")
}
}
func TestCordonedNodeKeepsItsWork(t *testing.T) {
// Cordon is not drain: it stops new placement and leaves what is running.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.inv.SetState("1", inventory.Cordoned)
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained != 0 || stopped != 0 {
t.Errorf("cordoning disturbed a running replica (drained=%d stopped=%d)", drained, stopped)
}
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Ready {
t.Errorf("the replica did not survive a cordon: %+v", got)
}
}