Files
llmgw/internal/control/reconciler/reconciler_test.go
T
Ilker ManapandClaude Opus 5 aef1c143c5
CI / test (push) Successful in 1m19s
Let go of shards an agent says it is not running
Adoption handled "the agent has something we do not know about". The mirror
image was missing: "we think the agent has something it does not".

Both follow from the same rule — the fleet is authoritative about what is
running (plan §6.5) — and without the second half an engine that dies while
its agent survives leaves the control plane believing a replica is Ready
forever. Measured on the cluster: the engine was killed, the agent restarted
and registered with nothing running, and the replica stayed Ready pointing at
a dead port for two and a half minutes until I stopped watching. Traffic
survived, because the breaker took the endpoint out and retries covered it —
24 requests, none failed — so the only symptom was a cluster quietly running a
third short.

A registration follows a fresh connection, so anything dispatched over a
previous one is not running by definition; there is no window where a shard is
legitimately started but unreported.

The previous commit's disown rule was too broad, which this turned up: when a
known replica had a live shard it disowned everything the agent reported for
it, including the shards we already knew. A reconnecting agent would have been
told to stop the replica that was serving. It now disowns only what it does
not recognise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 10:30:49 +02:00

901 lines
26 KiB
Go

package reconciler
import (
"context"
"fmt"
"log/slog"
"slices"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// fakeDispatcher records commands instead of sending them.
type fakeDispatcher struct {
mu sync.Mutex
started []ShardCommand
stopped []string
drained []string
failNext error
}
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.failNext != nil {
err := d.failNext
d.failNext = nil
return err
}
d.started = append(d.started, cmd)
return nil
}
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.stopped = append(d.stopped, shardID)
return nil
}
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.drained = append(d.drained, shardID)
return nil
}
func (d *fakeDispatcher) startedShards() []ShardCommand {
d.mu.Lock()
defer d.mu.Unlock()
return slices.Clone(d.started)
}
func (d *fakeDispatcher) reset() {
d.mu.Lock()
defer d.mu.Unlock()
d.started, d.stopped, d.drained = nil, nil, nil
}
// clock lets tests move time without sleeping.
type clock struct {
mu sync.Mutex
t time.Time
}
func newClock() *clock {
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) advance(d time.Duration) {
c.mu.Lock()
c.t = c.t.Add(d)
c.mu.Unlock()
}
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
for i := range n {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
})
}
for _, f := range fabrics {
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
})
}
return nd
}
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
if len(profiles) == 0 {
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
}
return catalog.Model{
Name: "llama", Source: "/models/llama",
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
}
}
type harness struct {
r *Reconciler
inv *inventory.Store
disp *fakeDispatcher
clock *clock
}
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
t.Helper()
cat, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatal(err)
}
inv := inventory.NewStore()
c := newClock()
for _, n := range nodes {
inv.Upsert(n)
inv.Touch(n.ID)
}
disp := &fakeDispatcher{}
cfg := DefaultConfig()
cfg.UnreachableAfter = 15 * time.Second
cfg.LostAfter = 60 * time.Second
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
// Nodes were just registered, so their LastSeen is wall-clock now, which is
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
for _, n := range nodes {
touchAt(inv, n.ID, c.now())
}
return &harness{r: r, inv: inv, disp: disp, clock: c}
}
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
func touchAt(inv *inventory.Store, id string, at time.Time) {
n, ok := inv.Get(id)
if !ok {
return
}
n.LastSeen = at
n.State = inventory.Ready
inv.Upsert(n)
// Upsert stamps LastSeen with the wall clock, so set it again directly.
inv.SetLastSeen(id, at)
}
// advance moves the clock and keeps every node heartbeating.
//
// Without this, advancing time past UnreachableAfter makes every node look
// silent and placement stops — which is correct behaviour, but not what the
// backoff tests are trying to exercise.
func (h *harness) advance(d time.Duration) {
h.clock.advance(d)
for _, n := range h.inv.Nodes() {
touchAt(h.inv, n.ID, h.clock.now())
}
}
// markReady walks every started shard to Ready.
func (h *harness) markReady() {
for _, cmd := range h.disp.startedShards() {
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
}
}
func TestPlacesMinimumReplicas(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.markReady()
reps := h.r.Replicas()
if len(reps) != 2 {
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
}
for _, rep := range reps {
if rep.State != Ready {
t.Errorf("replica %s is %s", rep.ID, rep.State)
}
if rep.Endpoint == "" {
t.Errorf("replica %s has no endpoint", rep.ID)
}
}
}
func TestOneReplicaPerPass(t *testing.T) {
// Placement depends on what is already allocated, so starting several from
// one inventory snapshot would hand them the same GPUs.
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 1 {
t.Fatalf("one pass created %d replicas", got)
}
seen := map[int]string{}
for range 3 {
h.markReady()
h.r.Reconcile(t.Context())
}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
for _, g := range s.GPUIndices {
key := g
if prev, dup := seen[key]; dup && prev != s.NodeID {
continue // same index on a different node is fine
}
seen[key] = s.NodeID
}
}
}
if got := len(h.r.Replicas()); got != 3 {
t.Errorf("expected 3 replicas, got %d", got)
}
}
func TestSpreadsAcrossNodes(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
nodes := map[string]struct{}{}
for _, rep := range h.r.Replicas() {
for _, n := range rep.NodeIDs() {
nodes[n] = struct{}{}
}
}
if len(nodes) != 2 {
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
}
}
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
// Spreading is a preference, not a rule: a second replica on one node still
// survives an engine crash, which is the common failure.
h := newHarness(t, testModel(2), gpuNode("only", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
if got := len(h.r.Replicas()); got != 2 {
t.Fatalf("expected 2 replicas on the single node, got %d", got)
}
}
func TestShardOrderLeaderFirst(t *testing.T) {
// Both engines put the rendezvous point on rank 0 — SGLang's dist-init
// address and vLLM's Ray head — so a worker started first has nothing to
// connect to and spends its startup budget retrying.
//
// This test previously asserted the opposite, on the theory that the head
// should wait for its workers. That is backwards: vLLM's server blocks
// until the Ray cluster has the GPUs it needs, so the leader can start
// immediately and waits on its own.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
if len(started) != 2 {
t.Fatalf("expected 2 shards, got %d", len(started))
}
if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 {
t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+
"point and must start first",
started[0].Spec.Rank, started[1].Spec.Rank)
}
// Every rank must be given the same rendezvous point.
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
t.Errorf("ranks got different leader addresses: %q and %q",
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
}
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
t.Error("ranks got different leader ports")
}
for _, cmd := range started {
if len(cmd.Spec.HCAs) == 0 {
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
}
}
}
func TestFailedShardFailsWholeReplica(t *testing.T) {
// One dead shard makes the others useless: they wait on collectives that
// will never complete while holding GPUs (plan §5.4).
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
// One shard ready, the other dead.
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
h.r.Reconcile(t.Context())
// Both shards must have been stopped, not just the failed one.
h.disp.mu.Lock()
stopped := slices.Clone(h.disp.stopped)
h.disp.mu.Unlock()
if len(stopped) != 2 {
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
}
// And the GPUs must be free again.
for _, id := range []string{"1", "2"} {
if got := len(h.inv.FreeGPUs(id)); got != 8 {
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
}
}
}
func TestBackoffAfterFailure(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
h.r.Reconcile(t.Context()) // reaps it
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
}
// Past the backoff, it tries again.
h.advance(DefaultConfig().MaxBackoff)
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got == 0 {
t.Error("the replica was never rebuilt after the backoff expired")
}
}
func TestSuccessClearsBackoff(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
// Fail once.
h.r.Reconcile(t.Context())
s := h.disp.startedShards()[0]
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
h.r.Reconcile(t.Context())
// Then succeed.
h.advance(DefaultConfig().MaxBackoff)
h.disp.reset()
h.r.Reconcile(t.Context())
if len(h.disp.startedShards()) == 0 {
t.Fatal("no replica was started after the backoff expired")
}
s2 := h.disp.startedShards()[0]
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
// A later failure should start from a short backoff, not a long streak.
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
h.r.Reconcile(t.Context())
h.advance(2 * DefaultConfig().BaseBackoff)
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got == 0 {
t.Error("backoff was not reset by the successful run in between")
}
}
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
// must get its replicas rebuilt elsewhere.
func TestLostNodeMovesReplicas(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Ready {
t.Fatalf("setup failed: %+v", reps)
}
original := reps[0].NodeIDs()[0]
survivor := "2"
if original == "2" {
survivor = "1"
}
// The surviving node keeps reporting; the other goes silent.
h.clock.advance(20 * time.Second)
touchAt(h.inv, survivor, h.clock.now())
h.r.Reconcile(t.Context())
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
t.Errorf("silent node is %s, expected unreachable", n.State)
}
// Unreachable is not lost: a blip must not tear down a working replica.
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
}
h.clock.advance(90 * time.Second)
touchAt(h.inv, survivor, h.clock.now())
h.disp.reset()
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
t.Fatalf("node is %s, expected lost", n.State)
}
// Past the backoff, the replica is rebuilt on the survivor. Only the
// survivor heartbeats; the lost node stays gone.
h.clock.advance(DefaultConfig().MaxBackoff)
touchAt(h.inv, survivor, h.clock.now())
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps = h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
}
if got := reps[0].NodeIDs()[0]; got != survivor {
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
}
if reps[0].State != Ready {
t.Errorf("rebuilt replica is %s", reps[0].State)
}
}
func TestHeartbeatRevivesNode(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
h.r.Reconcile(t.Context())
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
t.Fatalf("node is %s", n.State)
}
h.inv.Touch("1")
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
t.Errorf("a heartbeat did not revive the node: %s", n.State)
}
}
func TestScaleDownDrainsFirst(t *testing.T) {
// Draining before stopping lets in-flight requests finish instead of being
// cut off mid-stream.
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
if got := len(h.r.Replicas()); got != 2 {
t.Fatalf("setup: %d replicas", got)
}
// Shrink the catalog to one replica.
cat, err := catalog.New([]catalog.Model{testModel(1)})
if err != nil {
t.Fatal(err)
}
h.r.cat = cat
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained == 0 {
t.Error("the excess replica was not drained")
}
if stopped != 0 {
t.Error("a replica was stopped before it had drained")
}
// The next pass stops what has drained.
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
stopped = len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped")
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica after scale-down, got %d", got)
}
}
func TestUnplaceableModelExplains(t *testing.T) {
// A model that cannot be placed must leave a usable explanation rather than
// a silent absence.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "tp8", GPUClass: "h100-80g", TP: 8,
}), gpuNode("1", 4))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 0 {
t.Fatalf("something was placed: %d", got)
}
res, ok := h.r.Explain("llama")
if !ok {
t.Fatal("no explanation was recorded")
}
exp := res.Explain()
if !strings.Contains(exp, "needs 8") {
t.Errorf("explanation does not say what was missing:\n%s", exp)
}
}
func TestDispatcherFailureFailsReplica(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.disp.failNext = fmt.Errorf("agent unreachable")
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
}
// And the next pass must release its GPUs.
h.r.Reconcile(t.Context())
if got := len(h.inv.FreeGPUs("1")); got != 8 {
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
}
}
func TestReadyReplicasFeedTheRouter(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.ReadyReplicas()); got != 0 {
t.Errorf("a starting replica was offered to the router")
}
h.markReady()
h.r.Reconcile(t.Context())
h.markReady()
ready := h.r.ReadyReplicas()
if len(ready) != 2 {
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
}
for _, rep := range ready {
if !strings.HasPrefix(rep.Endpoint, "http://") {
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
}
if rep.Weight <= 0 {
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
}
}
}
func TestPortsDoNotCollide(t *testing.T) {
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
for range 3 {
h.r.Reconcile(t.Context())
h.markReady()
}
ports := map[int]string{}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
}
ports[s.HTTPPort] = rep.ID
}
}
}
func TestReplicaStateStrings(t *testing.T) {
for st, want := range map[ReplicaState]string{
Placing: "placing", Starting: "starting", Ready: "ready",
Draining: "draining", Stopped: "stopped", Failed: "failed",
} {
if got := st.String(); got != want {
t.Errorf("%d -> %q", int(st), got)
}
}
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
// Draining must not count, or a replacement is never started for it.
t.Error("Counts is wrong")
}
}
// --- adoption after a control plane restart (plan §6.5) ---
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
a := AdoptedShard{
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
}
if rank == 0 {
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
}
return a
}
func TestAdoptRebuildsReplica(t *testing.T) {
// The control plane restarted. Nothing is known, but a node reports a live
// replica. Without this the reconciler would place a duplicate while the
// original keeps holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
if len(unclaimed) != 0 {
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
}
rep := reps[0]
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
t.Errorf("identity wrong: %+v", rep)
}
if rep.State != Ready {
t.Errorf("state=%s; every shard reported Ready", rep.State)
}
if rep.Endpoint == "" {
t.Error("no endpoint; the router would have nowhere to send")
}
if rep.Weight <= 0 {
t.Error("weight not taken from the catalog")
}
// The GPUs must be reclaimed, or the next placement hands them out twice.
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
}
}
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
// The whole point: after adopting, a reconciliation pass must see the
// desired count already met.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica, got %d", got)
}
}
func TestAdoptMultiNodeReplica(t *testing.T) {
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
shards := []AdoptedShard{
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
}
for i := range shards {
shards[i].Profile = "2node"
shards[i].Fabric = "ib-a"
shards[i].Pool = "ib-a/h100-80g"
}
if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 {
t.Fatalf("unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 || len(reps[0].Shards) != 2 {
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
}
if reps[0].Fabric != "ib-a" {
t.Errorf("fabric lost: %q", reps[0].Fabric)
}
// Both nodes' GPUs must be reclaimed.
for _, id := range []string{"1", "2"} {
if free := len(h.inv.FreeGPUs(id)); free != 0 {
t.Errorf("node %s has %d free GPUs after adoption", id, free)
}
}
}
func TestAdoptPartialReplicaFails(t *testing.T) {
// A multi-node replica missing a shard is useless: the survivors wait on
// collectives that will never complete while holding GPUs.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
s.Profile = "2node"
s.State = supervisor.Failed
h.r.Adopt("1", []AdoptedShard{s})
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
}
// And the next pass releases its GPUs.
h.r.Reconcile(t.Context())
if free := len(h.inv.FreeGPUs("1")); free != 8 {
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
}
}
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
// The model left the catalog while the engine kept running. Nothing will
// ever route to it, and it is holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
s.Model = "a-model-that-was-removed"
unclaimed := h.r.Adopt("1", []AdoptedShard{s})
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
}
if got := len(h.r.Replicas()); got != 0 {
t.Errorf("a replica was created for an unknown model: %d", got)
}
}
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
s.Profile = "a-profile-that-was-removed"
if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 {
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
}
}
func TestAdoptIsIdempotent(t *testing.T) {
// An agent reconnecting re-reports the same shards; that must not double
// the allocation.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
h.r.Adopt("1", s)
h.r.Adopt("1", s)
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("re-reporting created %d replicas", got)
}
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
}
}
// --- draining a node (ctl drain) ---
func TestDrainingNodeMovesWorkOff(t *testing.T) {
// The operator's view: a node is about to be worked on, its replicas must
// end up elsewhere, and nothing may be cut off mid-request on the way.
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("setup: %d replicas", len(reps))
}
victim := reps[0].NodeIDs()[0]
survivor := "2"
if victim == "2" {
survivor = "1"
}
h.inv.SetState(victim, inventory.Draining)
h.disp.reset()
h.r.Reconcile(t.Context())
// It is drained, not stopped: the engine is healthy and still serving.
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained == 0 {
t.Error("the replica on the draining node was not drained")
}
if stopped != 0 {
t.Error("the replica was stopped before a replacement existed; in-flight requests would be cut")
}
// A draining replica does not count, so a replacement is placed elsewhere.
if got := len(h.disp.startedShards()); got == 0 {
t.Fatal("no replacement was placed")
}
for _, cmd := range h.disp.startedShards() {
if cmd.NodeID == victim {
t.Errorf("the replacement was placed back on the draining node %s", victim)
}
}
h.markReady()
h.r.Reconcile(t.Context())
// Now that the replacement serves, the original can go.
h.disp.mu.Lock()
stopped = len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped once its replacement was ready")
}
for _, rep := range h.r.Replicas() {
if rep.OnNode(victim) {
t.Errorf("replica %s is still on the drained node", rep.ID)
}
if !slices.Contains(rep.NodeIDs(), survivor) {
t.Errorf("replica %s did not move to the surviving node", rep.ID)
}
}
}
func TestDrainGraceStopsWithoutReplacement(t *testing.T) {
// Draining the only node that can host a model would otherwise never
// finish, leaving the node undrainable.
h := newHarness(t, testModel(1), gpuNode("only", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.inv.SetState("only", inventory.Draining)
h.r.Reconcile(t.Context()) // drains it; nowhere to place a replacement
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Draining {
t.Fatalf("expected a draining replica, got %+v", got)
}
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
early := len(h.disp.stopped)
h.disp.mu.Unlock()
if early != 0 {
t.Error("the replica was stopped before the grace period expired")
}
h.advance(DefaultConfig().DrainGrace + time.Second)
h.inv.SetState("only", inventory.Draining) // advance() resets node state
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
stopped := len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped; the node cannot be drained")
}
}
func TestCordonedNodeKeepsItsWork(t *testing.T) {
// Cordon is not drain: it stops new placement and leaves what is running.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.inv.SetState("1", inventory.Cordoned)
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained != 0 || stopped != 0 {
t.Errorf("cordoning disturbed a running replica (drained=%d stopped=%d)", drained, stopped)
}
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Ready {
t.Errorf("the replica did not survive a cordon: %+v", got)
}
}