CI / test (push) Failing after 53s
Placement and environment generation had landed early, which made multi-node look further along than it was. A replica spanning nodes could not come up at all, and all three reasons came down to assumptions that only hold on one node. Not every rank serves HTTP. The agent health-checked every shard at its own port, but only the leader answers; workers timed out, reported Failed and took the replica with them. Engine.ServesHTTP now says which ranks serve, and a worker is judged by staying alive. That is deliberately weaker evidence: the replica's real readiness is the leader answering, which only happens once every worker has joined. A shard is not always one process. Engine.Commands returns a sequence — every command but the last is a sidecar kept alive for the shard's lifetime and torn down with it, and the last one's exit is the shard's. vLLM under Ray is why: the leader runs a Ray head and then the server against it, while a worker runs only the join and no vLLM process at all. RayCommand had been written and tested but never called, so launch_mode: ray would have run vllm serve against a cluster that did not exist. Sidecars are killed with the shard, because a Ray head left behind holds the port the next replica needs. Start order was backwards. Workers went first, on the theory that the head should wait for them. Both engines put the rendezvous point on rank 0, so a worker started first has nothing to connect to. vLLM's server blocks until the Ray cluster has the GPUs it needs, so the leader can start immediately and waits on its own. A command can now also require that rendezvous point to be listening before the next starts, bounded by a timeout so a head that never appears fails the shard instead of hanging it. Stop was also killing only the engine, leaving sidecars behind. None of this is verified against real engines — it is tested with stub processes. The Ray sequencing, and which launch mode a given vLLM version wants, can only be confirmed on hardware. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
774 lines
22 KiB
Go
774 lines
22 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
// fakeDispatcher records commands instead of sending them.
|
|
type fakeDispatcher struct {
|
|
mu sync.Mutex
|
|
started []ShardCommand
|
|
stopped []string
|
|
drained []string
|
|
failNext error
|
|
}
|
|
|
|
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
if d.failNext != nil {
|
|
err := d.failNext
|
|
d.failNext = nil
|
|
return err
|
|
}
|
|
d.started = append(d.started, cmd)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.stopped = append(d.stopped, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.drained = append(d.drained, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) startedShards() []ShardCommand {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
return slices.Clone(d.started)
|
|
}
|
|
|
|
func (d *fakeDispatcher) reset() {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.started, d.stopped, d.drained = nil, nil, nil
|
|
}
|
|
|
|
// clock lets tests move time without sleeping.
|
|
type clock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
}
|
|
|
|
func newClock() *clock {
|
|
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
|
|
}
|
|
func (c *clock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.t
|
|
}
|
|
func (c *clock) advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
c.t = c.t.Add(d)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
|
|
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
|
|
for i := range n {
|
|
nd.GPUs = append(nd.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
|
|
})
|
|
}
|
|
for _, f := range fabrics {
|
|
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
|
|
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
|
|
})
|
|
}
|
|
return nd
|
|
}
|
|
|
|
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
|
|
if len(profiles) == 0 {
|
|
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
|
|
}
|
|
return catalog.Model{
|
|
Name: "llama", Source: "/models/llama",
|
|
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
|
|
}
|
|
}
|
|
|
|
type harness struct {
|
|
r *Reconciler
|
|
inv *inventory.Store
|
|
disp *fakeDispatcher
|
|
clock *clock
|
|
}
|
|
|
|
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
|
|
t.Helper()
|
|
|
|
cat, err := catalog.New([]catalog.Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inv := inventory.NewStore()
|
|
c := newClock()
|
|
for _, n := range nodes {
|
|
inv.Upsert(n)
|
|
inv.Touch(n.ID)
|
|
}
|
|
|
|
disp := &fakeDispatcher{}
|
|
cfg := DefaultConfig()
|
|
cfg.UnreachableAfter = 15 * time.Second
|
|
cfg.LostAfter = 60 * time.Second
|
|
|
|
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
|
|
r.SetClock(c.now)
|
|
// Nodes were just registered, so their LastSeen is wall-clock now, which is
|
|
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
|
|
for _, n := range nodes {
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
return &harness{r: r, inv: inv, disp: disp, clock: c}
|
|
}
|
|
|
|
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
|
|
func touchAt(inv *inventory.Store, id string, at time.Time) {
|
|
n, ok := inv.Get(id)
|
|
if !ok {
|
|
return
|
|
}
|
|
n.LastSeen = at
|
|
n.State = inventory.Ready
|
|
inv.Upsert(n)
|
|
// Upsert stamps LastSeen with the wall clock, so set it again directly.
|
|
inv.SetLastSeen(id, at)
|
|
}
|
|
|
|
// advance moves the clock and keeps every node heartbeating.
|
|
//
|
|
// Without this, advancing time past UnreachableAfter makes every node look
|
|
// silent and placement stops — which is correct behaviour, but not what the
|
|
// backoff tests are trying to exercise.
|
|
func (h *harness) advance(d time.Duration) {
|
|
h.clock.advance(d)
|
|
for _, n := range h.inv.Nodes() {
|
|
touchAt(h.inv, n.ID, h.clock.now())
|
|
}
|
|
}
|
|
|
|
// markReady walks every started shard to Ready.
|
|
func (h *harness) markReady() {
|
|
for _, cmd := range h.disp.startedShards() {
|
|
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
|
|
}
|
|
}
|
|
|
|
func TestPlacesMinimumReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 2 {
|
|
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
|
|
}
|
|
for _, rep := range reps {
|
|
if rep.State != Ready {
|
|
t.Errorf("replica %s is %s", rep.ID, rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Errorf("replica %s has no endpoint", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOneReplicaPerPass(t *testing.T) {
|
|
// Placement depends on what is already allocated, so starting several from
|
|
// one inventory snapshot would hand them the same GPUs.
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Fatalf("one pass created %d replicas", got)
|
|
}
|
|
|
|
seen := map[int]string{}
|
|
for range 3 {
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
for _, g := range s.GPUIndices {
|
|
key := g
|
|
if prev, dup := seen[key]; dup && prev != s.NodeID {
|
|
continue // same index on a different node is fine
|
|
}
|
|
seen[key] = s.NodeID
|
|
}
|
|
}
|
|
}
|
|
if got := len(h.r.Replicas()); got != 3 {
|
|
t.Errorf("expected 3 replicas, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestSpreadsAcrossNodes(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
nodes := map[string]struct{}{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, n := range rep.NodeIDs() {
|
|
nodes[n] = struct{}{}
|
|
}
|
|
}
|
|
if len(nodes) != 2 {
|
|
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
|
|
}
|
|
}
|
|
|
|
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
|
|
// Spreading is a preference, not a rule: a second replica on one node still
|
|
// survives an engine crash, which is the common failure.
|
|
h := newHarness(t, testModel(2), gpuNode("only", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("expected 2 replicas on the single node, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestShardOrderLeaderFirst(t *testing.T) {
|
|
// Both engines put the rendezvous point on rank 0 — SGLang's dist-init
|
|
// address and vLLM's Ray head — so a worker started first has nothing to
|
|
// connect to and spends its startup budget retrying.
|
|
//
|
|
// This test previously asserted the opposite, on the theory that the head
|
|
// should wait for its workers. That is backwards: vLLM's server blocks
|
|
// until the Ray cluster has the GPUs it needs, so the leader can start
|
|
// immediately and waits on its own.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
started := h.disp.startedShards()
|
|
if len(started) != 2 {
|
|
t.Fatalf("expected 2 shards, got %d", len(started))
|
|
}
|
|
if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 {
|
|
t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+
|
|
"point and must start first",
|
|
started[0].Spec.Rank, started[1].Spec.Rank)
|
|
}
|
|
// Every rank must be given the same rendezvous point.
|
|
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
|
|
t.Errorf("ranks got different leader addresses: %q and %q",
|
|
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
|
|
}
|
|
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
|
|
t.Error("ranks got different leader ports")
|
|
}
|
|
for _, cmd := range started {
|
|
if len(cmd.Spec.HCAs) == 0 {
|
|
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFailedShardFailsWholeReplica(t *testing.T) {
|
|
// One dead shard makes the others useless: they wait on collectives that
|
|
// will never complete while holding GPUs (plan §5.4).
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
|
|
// One shard ready, the other dead.
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
|
|
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Both shards must have been stopped, not just the failed one.
|
|
h.disp.mu.Lock()
|
|
stopped := slices.Clone(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if len(stopped) != 2 {
|
|
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
|
|
}
|
|
// And the GPUs must be free again.
|
|
for _, id := range []string{"1", "2"} {
|
|
if got := len(h.inv.FreeGPUs(id)); got != 8 {
|
|
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackoffAfterFailure(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context()) // reaps it
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
|
|
}
|
|
|
|
// Past the backoff, it tries again.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("the replica was never rebuilt after the backoff expired")
|
|
}
|
|
}
|
|
|
|
func TestSuccessClearsBackoff(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
// Fail once.
|
|
h.r.Reconcile(t.Context())
|
|
s := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Then succeed.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Fatal("no replica was started after the backoff expired")
|
|
}
|
|
s2 := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
|
|
|
|
// A later failure should start from a short backoff, not a long streak.
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(2 * DefaultConfig().BaseBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("backoff was not reset by the successful run in between")
|
|
}
|
|
}
|
|
|
|
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
|
|
// must get its replicas rebuilt elsewhere.
|
|
func TestLostNodeMovesReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("setup failed: %+v", reps)
|
|
}
|
|
original := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if original == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
// The surviving node keeps reporting; the other goes silent.
|
|
h.clock.advance(20 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
|
|
t.Errorf("silent node is %s, expected unreachable", n.State)
|
|
}
|
|
// Unreachable is not lost: a blip must not tear down a working replica.
|
|
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
|
|
}
|
|
|
|
h.clock.advance(90 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
|
|
t.Fatalf("node is %s, expected lost", n.State)
|
|
}
|
|
|
|
// Past the backoff, the replica is rebuilt on the survivor. Only the
|
|
// survivor heartbeats; the lost node stays gone.
|
|
h.clock.advance(DefaultConfig().MaxBackoff)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps = h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
|
|
}
|
|
if got := reps[0].NodeIDs()[0]; got != survivor {
|
|
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
|
|
}
|
|
if reps[0].State != Ready {
|
|
t.Errorf("rebuilt replica is %s", reps[0].State)
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatRevivesNode(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
|
|
h.r.Reconcile(t.Context())
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
|
|
t.Fatalf("node is %s", n.State)
|
|
}
|
|
|
|
h.inv.Touch("1")
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
|
|
t.Errorf("a heartbeat did not revive the node: %s", n.State)
|
|
}
|
|
}
|
|
|
|
func TestScaleDownDrainsFirst(t *testing.T) {
|
|
// Draining before stopping lets in-flight requests finish instead of being
|
|
// cut off mid-stream.
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("setup: %d replicas", got)
|
|
}
|
|
|
|
// Shrink the catalog to one replica.
|
|
cat, err := catalog.New([]catalog.Model{testModel(1)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat = cat
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the excess replica was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("a replica was stopped before it had drained")
|
|
}
|
|
|
|
// The next pass stops what has drained.
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped")
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica after scale-down, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestUnplaceableModelExplains(t *testing.T) {
|
|
// A model that cannot be placed must leave a usable explanation rather than
|
|
// a silent absence.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "tp8", GPUClass: "h100-80g", TP: 8,
|
|
}), gpuNode("1", 4))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Fatalf("something was placed: %d", got)
|
|
}
|
|
|
|
res, ok := h.r.Explain("llama")
|
|
if !ok {
|
|
t.Fatal("no explanation was recorded")
|
|
}
|
|
exp := res.Explain()
|
|
if !strings.Contains(exp, "needs 8") {
|
|
t.Errorf("explanation does not say what was missing:\n%s", exp)
|
|
}
|
|
}
|
|
|
|
func TestDispatcherFailureFailsReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.disp.failNext = fmt.Errorf("agent unreachable")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass must release its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.inv.FreeGPUs("1")); got != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
|
|
}
|
|
}
|
|
|
|
func TestReadyReplicasFeedTheRouter(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.ReadyReplicas()); got != 0 {
|
|
t.Errorf("a starting replica was offered to the router")
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
ready := h.r.ReadyReplicas()
|
|
if len(ready) != 2 {
|
|
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
|
|
}
|
|
for _, rep := range ready {
|
|
if !strings.HasPrefix(rep.Endpoint, "http://") {
|
|
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPortsDoNotCollide(t *testing.T) {
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
ports := map[int]string{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
|
|
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
|
|
}
|
|
ports[s.HTTPPort] = rep.ID
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReplicaStateStrings(t *testing.T) {
|
|
for st, want := range map[ReplicaState]string{
|
|
Placing: "placing", Starting: "starting", Ready: "ready",
|
|
Draining: "draining", Stopped: "stopped", Failed: "failed",
|
|
} {
|
|
if got := st.String(); got != want {
|
|
t.Errorf("%d -> %q", int(st), got)
|
|
}
|
|
}
|
|
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
|
|
// Draining must not count, or a replacement is never started for it.
|
|
t.Error("Counts is wrong")
|
|
}
|
|
}
|
|
|
|
// --- adoption after a control plane restart (plan §6.5) ---
|
|
|
|
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
|
|
a := AdoptedShard{
|
|
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
|
|
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
|
|
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
|
|
}
|
|
if rank == 0 {
|
|
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
|
|
}
|
|
return a
|
|
}
|
|
|
|
func TestAdoptRebuildsReplica(t *testing.T) {
|
|
// The control plane restarted. Nothing is known, but a node reports a live
|
|
// replica. Without this the reconciler would place a duplicate while the
|
|
// original keeps holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
|
|
}
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
|
|
}
|
|
rep := reps[0]
|
|
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
|
|
t.Errorf("identity wrong: %+v", rep)
|
|
}
|
|
if rep.State != Ready {
|
|
t.Errorf("state=%s; every shard reported Ready", rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Error("no endpoint; the router would have nowhere to send")
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Error("weight not taken from the catalog")
|
|
}
|
|
|
|
// The GPUs must be reclaimed, or the next placement hands them out twice.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
|
|
// The whole point: after adopting, a reconciliation pass must see the
|
|
// desired count already met.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptMultiNodeReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
shards := []AdoptedShard{
|
|
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
}
|
|
for i := range shards {
|
|
shards[i].Profile = "2node"
|
|
shards[i].Fabric = "ib-a"
|
|
shards[i].Pool = "ib-a/h100-80g"
|
|
}
|
|
|
|
if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 {
|
|
t.Fatalf("unclaimed: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
|
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
|
|
}
|
|
if reps[0].Fabric != "ib-a" {
|
|
t.Errorf("fabric lost: %q", reps[0].Fabric)
|
|
}
|
|
// Both nodes' GPUs must be reclaimed.
|
|
for _, id := range []string{"1", "2"} {
|
|
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
|
t.Errorf("node %s has %d free GPUs after adoption", id, free)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdoptPartialReplicaFails(t *testing.T) {
|
|
// A multi-node replica missing a shard is useless: the survivors wait on
|
|
// collectives that will never complete while holding GPUs.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile = "2node"
|
|
s.State = supervisor.Failed
|
|
h.r.Adopt([]AdoptedShard{s})
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass releases its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
|
|
// The model left the catalog while the engine kept running. Nothing will
|
|
// ever route to it, and it is holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
|
|
s.Model = "a-model-that-was-removed"
|
|
|
|
unclaimed := h.r.Adopt([]AdoptedShard{s})
|
|
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
|
|
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a replica was created for an unknown model: %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
|
|
s.Profile = "a-profile-that-was-removed"
|
|
|
|
if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 {
|
|
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
|
|
}
|
|
}
|
|
|
|
func TestAdoptIsIdempotent(t *testing.T) {
|
|
// An agent reconnecting re-reports the same shards; that must not double
|
|
// the allocation.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
|
|
|
|
h.r.Adopt(s)
|
|
h.r.Adopt(s)
|
|
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("re-reporting created %d replicas", got)
|
|
}
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
|
|
}
|
|
}
|