CI / test (push) Failing after 50s
ctl talks to a management API on the admin listener, not the client-facing one. The two have different audiences: one serves inference to whoever holds an API key, the other lets a caller cordon nodes and read the cluster's shape. - nodes surfaces unhealthy GPUs and why they are excluded. That is usually the answer when a node looks idle but nothing will schedule onto it, and it was invisible before. - explain-placement prints every candidate considered and why it was rejected. - replicas shows the shards a replica is made of, with per-shard errors. - cordon, uncordon and drain are one endpoint with different targets, because the difference between cordon and drain is exactly what an operator needs to be sure of before running either. Draining had to be fixed first to make ctl drain worth having. A drained replica was stopped on the next pass, which cuts in-flight requests — the thing draining exists to avoid. Now a node marked Draining has its replicas drained, the model falls below its desired count, a replacement is placed elsewhere, and the original is stopped only once the replacement is serving. A grace period bounds it, because draining the only node that can host a model would otherwise never finish and the node could never be taken out of service. Verified against a two-node cluster of real binaries: draining moved a replica to the surviving node and traffic kept returning 200 throughout. ctl logs is deliberately absent. It needs a request/response pair the agent protocol does not have, and the failure path already ships an engine log tail with every status report — so the case that matters most is covered, and reading a healthy shard's output on demand is not worth a protocol change yet. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
901 lines
26 KiB
Go
901 lines
26 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
// fakeDispatcher records commands instead of sending them.
|
|
type fakeDispatcher struct {
|
|
mu sync.Mutex
|
|
started []ShardCommand
|
|
stopped []string
|
|
drained []string
|
|
failNext error
|
|
}
|
|
|
|
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
if d.failNext != nil {
|
|
err := d.failNext
|
|
d.failNext = nil
|
|
return err
|
|
}
|
|
d.started = append(d.started, cmd)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.stopped = append(d.stopped, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.drained = append(d.drained, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) startedShards() []ShardCommand {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
return slices.Clone(d.started)
|
|
}
|
|
|
|
func (d *fakeDispatcher) reset() {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.started, d.stopped, d.drained = nil, nil, nil
|
|
}
|
|
|
|
// clock lets tests move time without sleeping.
|
|
type clock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
}
|
|
|
|
func newClock() *clock {
|
|
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
|
|
}
|
|
func (c *clock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.t
|
|
}
|
|
func (c *clock) advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
c.t = c.t.Add(d)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
|
|
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
|
|
for i := range n {
|
|
nd.GPUs = append(nd.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
|
|
})
|
|
}
|
|
for _, f := range fabrics {
|
|
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
|
|
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
|
|
})
|
|
}
|
|
return nd
|
|
}
|
|
|
|
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
|
|
if len(profiles) == 0 {
|
|
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
|
|
}
|
|
return catalog.Model{
|
|
Name: "llama", Source: "/models/llama",
|
|
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
|
|
}
|
|
}
|
|
|
|
type harness struct {
|
|
r *Reconciler
|
|
inv *inventory.Store
|
|
disp *fakeDispatcher
|
|
clock *clock
|
|
}
|
|
|
|
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
|
|
t.Helper()
|
|
|
|
cat, err := catalog.New([]catalog.Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inv := inventory.NewStore()
|
|
c := newClock()
|
|
for _, n := range nodes {
|
|
inv.Upsert(n)
|
|
inv.Touch(n.ID)
|
|
}
|
|
|
|
disp := &fakeDispatcher{}
|
|
cfg := DefaultConfig()
|
|
cfg.UnreachableAfter = 15 * time.Second
|
|
cfg.LostAfter = 60 * time.Second
|
|
|
|
r := New(cfg, cat, inv, disp, slog.New(slog.DiscardHandler))
|
|
r.SetClock(c.now)
|
|
// Nodes were just registered, so their LastSeen is wall-clock now, which is
|
|
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
|
|
for _, n := range nodes {
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
return &harness{r: r, inv: inv, disp: disp, clock: c}
|
|
}
|
|
|
|
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
|
|
func touchAt(inv *inventory.Store, id string, at time.Time) {
|
|
n, ok := inv.Get(id)
|
|
if !ok {
|
|
return
|
|
}
|
|
n.LastSeen = at
|
|
n.State = inventory.Ready
|
|
inv.Upsert(n)
|
|
// Upsert stamps LastSeen with the wall clock, so set it again directly.
|
|
inv.SetLastSeen(id, at)
|
|
}
|
|
|
|
// advance moves the clock and keeps every node heartbeating.
|
|
//
|
|
// Without this, advancing time past UnreachableAfter makes every node look
|
|
// silent and placement stops — which is correct behaviour, but not what the
|
|
// backoff tests are trying to exercise.
|
|
func (h *harness) advance(d time.Duration) {
|
|
h.clock.advance(d)
|
|
for _, n := range h.inv.Nodes() {
|
|
touchAt(h.inv, n.ID, h.clock.now())
|
|
}
|
|
}
|
|
|
|
// markReady walks every started shard to Ready.
|
|
func (h *harness) markReady() {
|
|
for _, cmd := range h.disp.startedShards() {
|
|
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
|
|
}
|
|
}
|
|
|
|
func TestPlacesMinimumReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 2 {
|
|
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
|
|
}
|
|
for _, rep := range reps {
|
|
if rep.State != Ready {
|
|
t.Errorf("replica %s is %s", rep.ID, rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Errorf("replica %s has no endpoint", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOneReplicaPerPass(t *testing.T) {
|
|
// Placement depends on what is already allocated, so starting several from
|
|
// one inventory snapshot would hand them the same GPUs.
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Fatalf("one pass created %d replicas", got)
|
|
}
|
|
|
|
seen := map[int]string{}
|
|
for range 3 {
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
for _, g := range s.GPUIndices {
|
|
key := g
|
|
if prev, dup := seen[key]; dup && prev != s.NodeID {
|
|
continue // same index on a different node is fine
|
|
}
|
|
seen[key] = s.NodeID
|
|
}
|
|
}
|
|
}
|
|
if got := len(h.r.Replicas()); got != 3 {
|
|
t.Errorf("expected 3 replicas, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestSpreadsAcrossNodes(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
nodes := map[string]struct{}{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, n := range rep.NodeIDs() {
|
|
nodes[n] = struct{}{}
|
|
}
|
|
}
|
|
if len(nodes) != 2 {
|
|
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
|
|
}
|
|
}
|
|
|
|
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
|
|
// Spreading is a preference, not a rule: a second replica on one node still
|
|
// survives an engine crash, which is the common failure.
|
|
h := newHarness(t, testModel(2), gpuNode("only", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("expected 2 replicas on the single node, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestShardOrderLeaderFirst(t *testing.T) {
|
|
// Both engines put the rendezvous point on rank 0 — SGLang's dist-init
|
|
// address and vLLM's Ray head — so a worker started first has nothing to
|
|
// connect to and spends its startup budget retrying.
|
|
//
|
|
// This test previously asserted the opposite, on the theory that the head
|
|
// should wait for its workers. That is backwards: vLLM's server blocks
|
|
// until the Ray cluster has the GPUs it needs, so the leader can start
|
|
// immediately and waits on its own.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
started := h.disp.startedShards()
|
|
if len(started) != 2 {
|
|
t.Fatalf("expected 2 shards, got %d", len(started))
|
|
}
|
|
if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 {
|
|
t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+
|
|
"point and must start first",
|
|
started[0].Spec.Rank, started[1].Spec.Rank)
|
|
}
|
|
// Every rank must be given the same rendezvous point.
|
|
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
|
|
t.Errorf("ranks got different leader addresses: %q and %q",
|
|
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
|
|
}
|
|
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
|
|
t.Error("ranks got different leader ports")
|
|
}
|
|
for _, cmd := range started {
|
|
if len(cmd.Spec.HCAs) == 0 {
|
|
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFailedShardFailsWholeReplica(t *testing.T) {
|
|
// One dead shard makes the others useless: they wait on collectives that
|
|
// will never complete while holding GPUs (plan §5.4).
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
|
|
// One shard ready, the other dead.
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
|
|
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Both shards must have been stopped, not just the failed one.
|
|
h.disp.mu.Lock()
|
|
stopped := slices.Clone(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if len(stopped) != 2 {
|
|
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
|
|
}
|
|
// And the GPUs must be free again.
|
|
for _, id := range []string{"1", "2"} {
|
|
if got := len(h.inv.FreeGPUs(id)); got != 8 {
|
|
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackoffAfterFailure(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context()) // reaps it
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
|
|
}
|
|
|
|
// Past the backoff, it tries again.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("the replica was never rebuilt after the backoff expired")
|
|
}
|
|
}
|
|
|
|
func TestSuccessClearsBackoff(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
// Fail once.
|
|
h.r.Reconcile(t.Context())
|
|
s := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Then succeed.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Fatal("no replica was started after the backoff expired")
|
|
}
|
|
s2 := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
|
|
|
|
// A later failure should start from a short backoff, not a long streak.
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(2 * DefaultConfig().BaseBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("backoff was not reset by the successful run in between")
|
|
}
|
|
}
|
|
|
|
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
|
|
// must get its replicas rebuilt elsewhere.
|
|
func TestLostNodeMovesReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("setup failed: %+v", reps)
|
|
}
|
|
original := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if original == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
// The surviving node keeps reporting; the other goes silent.
|
|
h.clock.advance(20 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
|
|
t.Errorf("silent node is %s, expected unreachable", n.State)
|
|
}
|
|
// Unreachable is not lost: a blip must not tear down a working replica.
|
|
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
|
|
}
|
|
|
|
h.clock.advance(90 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
|
|
t.Fatalf("node is %s, expected lost", n.State)
|
|
}
|
|
|
|
// Past the backoff, the replica is rebuilt on the survivor. Only the
|
|
// survivor heartbeats; the lost node stays gone.
|
|
h.clock.advance(DefaultConfig().MaxBackoff)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps = h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
|
|
}
|
|
if got := reps[0].NodeIDs()[0]; got != survivor {
|
|
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
|
|
}
|
|
if reps[0].State != Ready {
|
|
t.Errorf("rebuilt replica is %s", reps[0].State)
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatRevivesNode(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
|
|
h.r.Reconcile(t.Context())
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
|
|
t.Fatalf("node is %s", n.State)
|
|
}
|
|
|
|
h.inv.Touch("1")
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
|
|
t.Errorf("a heartbeat did not revive the node: %s", n.State)
|
|
}
|
|
}
|
|
|
|
func TestScaleDownDrainsFirst(t *testing.T) {
|
|
// Draining before stopping lets in-flight requests finish instead of being
|
|
// cut off mid-stream.
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("setup: %d replicas", got)
|
|
}
|
|
|
|
// Shrink the catalog to one replica.
|
|
cat, err := catalog.New([]catalog.Model{testModel(1)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat = cat
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the excess replica was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("a replica was stopped before it had drained")
|
|
}
|
|
|
|
// The next pass stops what has drained.
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped")
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica after scale-down, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestUnplaceableModelExplains(t *testing.T) {
|
|
// A model that cannot be placed must leave a usable explanation rather than
|
|
// a silent absence.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "tp8", GPUClass: "h100-80g", TP: 8,
|
|
}), gpuNode("1", 4))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Fatalf("something was placed: %d", got)
|
|
}
|
|
|
|
res, ok := h.r.Explain("llama")
|
|
if !ok {
|
|
t.Fatal("no explanation was recorded")
|
|
}
|
|
exp := res.Explain()
|
|
if !strings.Contains(exp, "needs 8") {
|
|
t.Errorf("explanation does not say what was missing:\n%s", exp)
|
|
}
|
|
}
|
|
|
|
func TestDispatcherFailureFailsReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.disp.failNext = fmt.Errorf("agent unreachable")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass must release its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.inv.FreeGPUs("1")); got != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
|
|
}
|
|
}
|
|
|
|
func TestReadyReplicasFeedTheRouter(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.ReadyReplicas()); got != 0 {
|
|
t.Errorf("a starting replica was offered to the router")
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
ready := h.r.ReadyReplicas()
|
|
if len(ready) != 2 {
|
|
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
|
|
}
|
|
for _, rep := range ready {
|
|
if !strings.HasPrefix(rep.Endpoint, "http://") {
|
|
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPortsDoNotCollide(t *testing.T) {
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
ports := map[int]string{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
|
|
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
|
|
}
|
|
ports[s.HTTPPort] = rep.ID
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReplicaStateStrings(t *testing.T) {
|
|
for st, want := range map[ReplicaState]string{
|
|
Placing: "placing", Starting: "starting", Ready: "ready",
|
|
Draining: "draining", Stopped: "stopped", Failed: "failed",
|
|
} {
|
|
if got := st.String(); got != want {
|
|
t.Errorf("%d -> %q", int(st), got)
|
|
}
|
|
}
|
|
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
|
|
// Draining must not count, or a replacement is never started for it.
|
|
t.Error("Counts is wrong")
|
|
}
|
|
}
|
|
|
|
// --- adoption after a control plane restart (plan §6.5) ---
|
|
|
|
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
|
|
a := AdoptedShard{
|
|
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
|
|
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
|
|
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
|
|
}
|
|
if rank == 0 {
|
|
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
|
|
}
|
|
return a
|
|
}
|
|
|
|
func TestAdoptRebuildsReplica(t *testing.T) {
|
|
// The control plane restarted. Nothing is known, but a node reports a live
|
|
// replica. Without this the reconciler would place a duplicate while the
|
|
// original keeps holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
unclaimed := h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
|
|
}
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
|
|
}
|
|
rep := reps[0]
|
|
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
|
|
t.Errorf("identity wrong: %+v", rep)
|
|
}
|
|
if rep.State != Ready {
|
|
t.Errorf("state=%s; every shard reported Ready", rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Error("no endpoint; the router would have nowhere to send")
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Error("weight not taken from the catalog")
|
|
}
|
|
|
|
// The GPUs must be reclaimed, or the next placement hands them out twice.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
|
|
// The whole point: after adopting, a reconciliation pass must see the
|
|
// desired count already met.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Adopt([]AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptMultiNodeReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
shards := []AdoptedShard{
|
|
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
}
|
|
for i := range shards {
|
|
shards[i].Profile = "2node"
|
|
shards[i].Fabric = "ib-a"
|
|
shards[i].Pool = "ib-a/h100-80g"
|
|
}
|
|
|
|
if unclaimed := h.r.Adopt(shards); len(unclaimed) != 0 {
|
|
t.Fatalf("unclaimed: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
|
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
|
|
}
|
|
if reps[0].Fabric != "ib-a" {
|
|
t.Errorf("fabric lost: %q", reps[0].Fabric)
|
|
}
|
|
// Both nodes' GPUs must be reclaimed.
|
|
for _, id := range []string{"1", "2"} {
|
|
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
|
t.Errorf("node %s has %d free GPUs after adoption", id, free)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdoptPartialReplicaFails(t *testing.T) {
|
|
// A multi-node replica missing a shard is useless: the survivors wait on
|
|
// collectives that will never complete while holding GPUs.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile = "2node"
|
|
s.State = supervisor.Failed
|
|
h.r.Adopt([]AdoptedShard{s})
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass releases its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
|
|
// The model left the catalog while the engine kept running. Nothing will
|
|
// ever route to it, and it is holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
|
|
s.Model = "a-model-that-was-removed"
|
|
|
|
unclaimed := h.r.Adopt([]AdoptedShard{s})
|
|
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
|
|
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a replica was created for an unknown model: %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
|
|
s.Profile = "a-profile-that-was-removed"
|
|
|
|
if unclaimed := h.r.Adopt([]AdoptedShard{s}); len(unclaimed) != 1 {
|
|
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
|
|
}
|
|
}
|
|
|
|
func TestAdoptIsIdempotent(t *testing.T) {
|
|
// An agent reconnecting re-reports the same shards; that must not double
|
|
// the allocation.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
|
|
|
|
h.r.Adopt(s)
|
|
h.r.Adopt(s)
|
|
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("re-reporting created %d replicas", got)
|
|
}
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
|
|
}
|
|
}
|
|
|
|
// --- draining a node (ctl drain) ---
|
|
|
|
func TestDrainingNodeMovesWorkOff(t *testing.T) {
|
|
// The operator's view: a node is about to be worked on, its replicas must
|
|
// end up elsewhere, and nothing may be cut off mid-request on the way.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
victim := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if victim == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
h.inv.SetState(victim, inventory.Draining)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// It is drained, not stopped: the engine is healthy and still serving.
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the replica on the draining node was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("the replica was stopped before a replacement existed; in-flight requests would be cut")
|
|
}
|
|
|
|
// A draining replica does not count, so a replacement is placed elsewhere.
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Fatal("no replacement was placed")
|
|
}
|
|
for _, cmd := range h.disp.startedShards() {
|
|
if cmd.NodeID == victim {
|
|
t.Errorf("the replacement was placed back on the draining node %s", victim)
|
|
}
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Now that the replacement serves, the original can go.
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped once its replacement was ready")
|
|
}
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.OnNode(victim) {
|
|
t.Errorf("replica %s is still on the drained node", rep.ID)
|
|
}
|
|
if !slices.Contains(rep.NodeIDs(), survivor) {
|
|
t.Errorf("replica %s did not move to the surviving node", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDrainGraceStopsWithoutReplacement(t *testing.T) {
|
|
// Draining the only node that can host a model would otherwise never
|
|
// finish, leaving the node undrainable.
|
|
h := newHarness(t, testModel(1), gpuNode("only", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.inv.SetState("only", inventory.Draining)
|
|
h.r.Reconcile(t.Context()) // drains it; nowhere to place a replacement
|
|
|
|
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Draining {
|
|
t.Fatalf("expected a draining replica, got %+v", got)
|
|
}
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
early := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if early != 0 {
|
|
t.Error("the replica was stopped before the grace period expired")
|
|
}
|
|
|
|
h.advance(DefaultConfig().DrainGrace + time.Second)
|
|
h.inv.SetState("only", inventory.Draining) // advance() resets node state
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
stopped := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped; the node cannot be drained")
|
|
}
|
|
}
|
|
|
|
func TestCordonedNodeKeepsItsWork(t *testing.T) {
|
|
// Cordon is not drain: it stops new placement and leaves what is running.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.inv.SetState("1", inventory.Cordoned)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained != 0 || stopped != 0 {
|
|
t.Errorf("cordoning disturbed a running replica (drained=%d stopped=%d)", drained, stopped)
|
|
}
|
|
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Ready {
|
|
t.Errorf("the replica did not survive a cordon: %+v", got)
|
|
}
|
|
}
|