Files
llmgw/internal/control/reconciler/reconciler_test.go
T
ilkerandClaude Opus 5 75a8e76706 Survive a control plane restart without orphaning live engines
Not persisting replica state left one hole: restarting the control plane made it
forget every replica while the engines kept running. They held GPUs the
inventory believed were free, so the next placement failed to allocate, with an
error naming the new process rather than the old one.

The fix belongs on the agent, not in a database. A row can say "running" about a
process that died; the fleet cannot.

- the supervisor writes the full shard spec, not just a pid and command line. A
  pid identifies a process; the spec is what lets it be described back to the
  control plane and matched to a replica.
- Adopt takes over an engine a previous agent left running. It is not this
  process's child, so it cannot be waited on: it is watched by polling and
  stopped by signalling its group directly. Its earlier output is gone and the
  log says so rather than presenting emptiness as silence.
- every Register reports what the agent is running, not only the first, because
  a reconnect may be a control plane restart and has to recover the same way.
- the reconciler rebuilds replicas from those reports and reclaims their GPUs.
  Anything whose model or profile left the catalog is stopped; nothing will
  route to it and it is holding cards.
- an engine the agent cannot identify is killed. A process holding GPUs that
  nothing can describe is worse than no process, because placement hands those
  GPUs out again.

Stop was also wrong for adopted shards: it checked for an exec.Cmd, which they
do not have, and reported them stopped without killing anything.

TestAgentReconnects had been passing for no reason. httptest's
CloseClientConnections does nothing to an h2c stream — prior-knowledge h2c
hijacks the connection and httptest stops tracking hijacked ones — so the agent
never reconnected, and the assertion on the connection map passed on a stale
entry. Tests now sever connections through their own listener and count opened
streams.

Plan §11 also records what multi-node still needs, since placement landing early
makes it look further along than it is: workers never become Ready because the
agent health-checks every shard while only rank 0 serves HTTP; RayCommand is
written and tested but never called; and startup is fire-and-forget.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 06:15:04 +02:00

767 lines
22 KiB
Go

package reconciler
import (
"context"
"fmt"
"log/slog"
"slices"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// fakeDispatcher records commands instead of sending them.
type fakeDispatcher struct {
mu sync.Mutex
started []ShardCommand
stopped []string
drained []string
failNext error
}
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.failNext != nil {
err := d.failNext
d.failNext = nil
return err
}
d.started = append(d.started, cmd)
return nil
}
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.stopped = append(d.stopped, shardID)
return nil
}
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
d.mu.Lock()
defer d.mu.Unlock()
d.drained = append(d.drained, shardID)
return nil
}
func (d *fakeDispatcher) startedShards() []ShardCommand {
d.mu.Lock()
defer d.mu.Unlock()
return slices.Clone(d.started)
}
func (d *fakeDispatcher) reset() {
d.mu.Lock()
defer d.mu.Unlock()
d.started, d.stopped, d.drained = nil, nil, nil
}
// clock lets tests move time without sleeping.
type clock struct {
mu sync.Mutex
t time.Time
}
func newClock() *clock {
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) advance(d time.Duration) {
c.mu.Lock()
c.t = c.t.Add(d)
c.mu.Unlock()
}
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
for i := range n {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
})
}
for _, f := range fabrics {
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
})
}
return nd
}
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
if len(profiles) == 0 {
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
}
return catalog.Model{
Name: "llama", Source: "/models/llama",
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
}
}
type harness struct {
r *Reconciler
inv *inventory.Store
disp *fakeDispatcher
clock *clock
}
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
t.Helper()
cat, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatal(err)
}
inv := inventory.NewStore()
c := newClock()
for _, n := range nodes {
inv.Upsert(n)
inv.Touch(n.ID)
}
disp := &fakeDispatcher{}
cfg := DefaultConfig()
cfg.UnreachableAfter = 15 * time.Second
cfg.LostAfter = 60 * time.Second
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
r.SetClock(c.now)
// Nodes were just registered, so their LastSeen is wall-clock now, which is
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
for _, n := range nodes {
touchAt(inv, n.ID, c.now())
}
return &harness{r: r, inv: inv, disp: disp, clock: c}
}
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
func touchAt(inv *inventory.Store, id string, at time.Time) {
n, ok := inv.Get(id)
if !ok {
return
}
n.LastSeen = at
n.State = inventory.Ready
inv.Upsert(n)
// Upsert stamps LastSeen with the wall clock, so set it again directly.
inv.SetLastSeen(id, at)
}
// advance moves the clock and keeps every node heartbeating.
//
// Without this, advancing time past UnreachableAfter makes every node look
// silent and placement stops — which is correct behaviour, but not what the
// backoff tests are trying to exercise.
func (h *harness) advance(d time.Duration) {
h.clock.advance(d)
for _, n := range h.inv.Nodes() {
touchAt(h.inv, n.ID, h.clock.now())
}
}
// markReady walks every started shard to Ready.
func (h *harness) markReady() {
for _, cmd := range h.disp.startedShards() {
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
}
}
func TestPlacesMinimumReplicas(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
h.markReady()
reps := h.r.Replicas()
if len(reps) != 2 {
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
}
for _, rep := range reps {
if rep.State != Ready {
t.Errorf("replica %s is %s", rep.ID, rep.State)
}
if rep.Endpoint == "" {
t.Errorf("replica %s has no endpoint", rep.ID)
}
}
}
func TestOneReplicaPerPass(t *testing.T) {
// Placement depends on what is already allocated, so starting several from
// one inventory snapshot would hand them the same GPUs.
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 1 {
t.Fatalf("one pass created %d replicas", got)
}
seen := map[int]string{}
for range 3 {
h.markReady()
h.r.Reconcile(t.Context())
}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
for _, g := range s.GPUIndices {
key := g
if prev, dup := seen[key]; dup && prev != s.NodeID {
continue // same index on a different node is fine
}
seen[key] = s.NodeID
}
}
}
if got := len(h.r.Replicas()); got != 3 {
t.Errorf("expected 3 replicas, got %d", got)
}
}
func TestSpreadsAcrossNodes(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
nodes := map[string]struct{}{}
for _, rep := range h.r.Replicas() {
for _, n := range rep.NodeIDs() {
nodes[n] = struct{}{}
}
}
if len(nodes) != 2 {
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
}
}
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
// Spreading is a preference, not a rule: a second replica on one node still
// survives an engine crash, which is the common failure.
h := newHarness(t, testModel(2), gpuNode("only", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
if got := len(h.r.Replicas()); got != 2 {
t.Fatalf("expected 2 replicas on the single node, got %d", got)
}
}
func TestShardOrderWorkersFirst(t *testing.T) {
// SGLang ranks rendezvous and vLLM under Ray needs its workers attached
// before the head serves, so rank 0 goes last.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
if len(started) != 2 {
t.Fatalf("expected 2 shards, got %d", len(started))
}
if started[0].Spec.Rank != 1 || started[1].Spec.Rank != 0 {
t.Errorf("start order was rank %d then %d; workers must start first",
started[0].Spec.Rank, started[1].Spec.Rank)
}
// Every rank must be given the same rendezvous point.
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
t.Errorf("ranks got different leader addresses: %q and %q",
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
}
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
t.Error("ranks got different leader ports")
}
for _, cmd := range started {
if len(cmd.Spec.HCAs) == 0 {
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
}
}
}
func TestFailedShardFailsWholeReplica(t *testing.T) {
// One dead shard makes the others useless: they wait on collectives that
// will never complete while holding GPUs (plan §5.4).
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
// One shard ready, the other dead.
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
h.r.Reconcile(t.Context())
// Both shards must have been stopped, not just the failed one.
h.disp.mu.Lock()
stopped := slices.Clone(h.disp.stopped)
h.disp.mu.Unlock()
if len(stopped) != 2 {
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
}
// And the GPUs must be free again.
for _, id := range []string{"1", "2"} {
if got := len(h.inv.FreeGPUs(id)); got != 8 {
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
}
}
}
func TestBackoffAfterFailure(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
started := h.disp.startedShards()
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
h.r.Reconcile(t.Context()) // reaps it
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
}
// Past the backoff, it tries again.
h.advance(DefaultConfig().MaxBackoff)
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got == 0 {
t.Error("the replica was never rebuilt after the backoff expired")
}
}
func TestSuccessClearsBackoff(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
// Fail once.
h.r.Reconcile(t.Context())
s := h.disp.startedShards()[0]
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
h.r.Reconcile(t.Context())
// Then succeed.
h.advance(DefaultConfig().MaxBackoff)
h.disp.reset()
h.r.Reconcile(t.Context())
if len(h.disp.startedShards()) == 0 {
t.Fatal("no replica was started after the backoff expired")
}
s2 := h.disp.startedShards()[0]
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
// A later failure should start from a short backoff, not a long streak.
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
h.r.Reconcile(t.Context())
h.advance(2 * DefaultConfig().BaseBackoff)
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got == 0 {
t.Error("backoff was not reset by the successful run in between")
}
}
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
// must get its replicas rebuilt elsewhere.
func TestLostNodeMovesReplicas(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Ready {
t.Fatalf("setup failed: %+v", reps)
}
original := reps[0].NodeIDs()[0]
survivor := "2"
if original == "2" {
survivor = "1"
}
// The surviving node keeps reporting; the other goes silent.
h.clock.advance(20 * time.Second)
touchAt(h.inv, survivor, h.clock.now())
h.r.Reconcile(t.Context())
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
t.Errorf("silent node is %s, expected unreachable", n.State)
}
// Unreachable is not lost: a blip must not tear down a working replica.
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
}
h.clock.advance(90 * time.Second)
touchAt(h.inv, survivor, h.clock.now())
h.disp.reset()
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
t.Fatalf("node is %s, expected lost", n.State)
}
// Past the backoff, the replica is rebuilt on the survivor. Only the
// survivor heartbeats; the lost node stays gone.
h.clock.advance(DefaultConfig().MaxBackoff)
touchAt(h.inv, survivor, h.clock.now())
h.r.Reconcile(t.Context())
h.markReady()
h.r.Reconcile(t.Context())
reps = h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
}
if got := reps[0].NodeIDs()[0]; got != survivor {
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
}
if reps[0].State != Ready {
t.Errorf("rebuilt replica is %s", reps[0].State)
}
}
func TestHeartbeatRevivesNode(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
h.r.Reconcile(t.Context())
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
t.Fatalf("node is %s", n.State)
}
h.inv.Touch("1")
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
t.Errorf("a heartbeat did not revive the node: %s", n.State)
}
}
func TestScaleDownDrainsFirst(t *testing.T) {
// Draining before stopping lets in-flight requests finish instead of being
// cut off mid-stream.
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
for range 2 {
h.r.Reconcile(t.Context())
h.markReady()
}
if got := len(h.r.Replicas()); got != 2 {
t.Fatalf("setup: %d replicas", got)
}
// Shrink the catalog to one replica.
cat, err := catalog.New([]catalog.Model{testModel(1)})
if err != nil {
t.Fatal(err)
}
h.r.cat = cat
h.disp.reset()
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
h.disp.mu.Unlock()
if drained == 0 {
t.Error("the excess replica was not drained")
}
if stopped != 0 {
t.Error("a replica was stopped before it had drained")
}
// The next pass stops what has drained.
h.r.Reconcile(t.Context())
h.disp.mu.Lock()
stopped = len(h.disp.stopped)
h.disp.mu.Unlock()
if stopped == 0 {
t.Error("the drained replica was never stopped")
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica after scale-down, got %d", got)
}
}
func TestUnplaceableModelExplains(t *testing.T) {
// A model that cannot be placed must leave a usable explanation rather than
// a silent absence.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "tp8", GPUClass: "h100-80g", TP: 8,
}), gpuNode("1", 4))
h.r.Reconcile(t.Context())
if got := len(h.r.Replicas()); got != 0 {
t.Fatalf("something was placed: %d", got)
}
res, ok := h.r.Explain("llama")
if !ok {
t.Fatal("no explanation was recorded")
}
exp := res.Explain()
if !strings.Contains(exp, "needs 8") {
t.Errorf("explanation does not say what was missing:\n%s", exp)
}
}
func TestDispatcherFailureFailsReplica(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.disp.failNext = fmt.Errorf("agent unreachable")
h.r.Reconcile(t.Context())
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
}
// And the next pass must release its GPUs.
h.r.Reconcile(t.Context())
if got := len(h.inv.FreeGPUs("1")); got != 8 {
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
}
}
func TestReadyReplicasFeedTheRouter(t *testing.T) {
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
h.r.Reconcile(t.Context())
if got := len(h.r.ReadyReplicas()); got != 0 {
t.Errorf("a starting replica was offered to the router")
}
h.markReady()
h.r.Reconcile(t.Context())
h.markReady()
ready := h.r.ReadyReplicas()
if len(ready) != 2 {
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
}
for _, rep := range ready {
if !strings.HasPrefix(rep.Endpoint, "http://") {
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
}
if rep.Weight <= 0 {
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
}
}
}
func TestPortsDoNotCollide(t *testing.T) {
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
for range 3 {
h.r.Reconcile(t.Context())
h.markReady()
}
ports := map[int]string{}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
}
ports[s.HTTPPort] = rep.ID
}
}
}
func TestReplicaStateStrings(t *testing.T) {
for st, want := range map[ReplicaState]string{
Placing: "placing", Starting: "starting", Ready: "ready",
Draining: "draining", Stopped: "stopped", Failed: "failed",
} {
if got := st.String(); got != want {
t.Errorf("%d -> %q", int(st), got)
}
}
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
// Draining must not count, or a replacement is never started for it.
t.Error("Counts is wrong")
}
}
// --- adoption after a control plane restart (plan §6.5) ---
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
a := AdoptedShard{
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
}
if rank == 0 {
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
}
return a
}
func TestAdoptRebuildsReplica(t *testing.T) {
// The control plane restarted. Nothing is known, but a node reports a live
// replica. Without this the reconciler would place a duplicate while the
// original keeps holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
if len(unclaimed) != 0 {
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
}
rep := reps[0]
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
t.Errorf("identity wrong: %+v", rep)
}
if rep.State != Ready {
t.Errorf("state=%s; every shard reported Ready", rep.State)
}
if rep.Endpoint == "" {
t.Error("no endpoint; the router would have nowhere to send")
}
if rep.Weight <= 0 {
t.Error("weight not taken from the catalog")
}
// The GPUs must be reclaimed, or the next placement hands them out twice.
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
}
}
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
// The whole point: after adopting, a reconciliation pass must see the
// desired count already met.
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
h.disp.reset()
h.r.Reconcile(t.Context())
if got := len(h.disp.startedShards()); got != 0 {
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
}
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("expected 1 replica, got %d", got)
}
}
func TestAdoptMultiNodeReplica(t *testing.T) {
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
shards := []AdoptedShard{
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
}
for i := range shards {
shards[i].Profile = "2node"
shards[i].Fabric = "ib-a"
shards[i].Pool = "ib-a/h100-80g"
}
if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 {
t.Fatalf("unclaimed: %+v", unclaimed)
}
reps := h.r.Replicas()
if len(reps) != 1 || len(reps[0].Shards) != 2 {
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
}
if reps[0].Fabric != "ib-a" {
t.Errorf("fabric lost: %q", reps[0].Fabric)
}
// Both nodes' GPUs must be reclaimed.
for _, id := range []string{"1", "2"} {
if free := len(h.inv.FreeGPUs(id)); free != 0 {
t.Errorf("node %s has %d free GPUs after adoption", id, free)
}
}
}
func TestAdoptPartialReplicaFails(t *testing.T) {
// A multi-node replica missing a shard is useless: the survivors wait on
// collectives that will never complete while holding GPUs.
h := newHarness(t, testModel(1, catalog.Profile{
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
s.Profile = "2node"
s.State = supervisor.Failed
h.r.Adopt([]AdoptedShard{s})
reps := h.r.Replicas()
if len(reps) != 1 || reps[0].State != Failed {
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
}
// And the next pass releases its GPUs.
h.r.Reconcile(t.Context())
if free := len(h.inv.FreeGPUs("1")); free != 8 {
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
}
}
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
// The model left the catalog while the engine kept running. Nothing will
// ever route to it, and it is holding GPUs.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
s.Model = "a-model-that-was-removed"
unclaimed := h.r.Adopt([]AdoptedShard{s})
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
}
if got := len(h.r.Replicas()); got != 0 {
t.Errorf("a replica was created for an unknown model: %d", got)
}
}
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
s.Profile = "a-profile-that-was-removed"
if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 {
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
}
}
func TestAdoptIsIdempotent(t *testing.T) {
// An agent reconnecting re-reports the same shards; that must not double
// the allocation.
h := newHarness(t, testModel(1), gpuNode("1", 8))
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
h.r.Adopt(s)
h.r.Adopt(s)
if got := len(h.r.Replicas()); got != 1 {
t.Errorf("re-reporting created %d replicas", got)
}
if free := len(h.inv.FreeGPUs("1")); free != 4 {
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
}
}