A control plane that has just started or become leader has heard no requests, and read the on-demand replicas its nodes report as wanted by nobody: it drained every one of them at every failover. An adopted replica of an on-demand model now gets a whole idle timeout from its adoption. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2226 lines
69 KiB
Go
2226 lines
69 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
// fakeDispatcher records commands instead of sending them.
|
|
type fakeDispatcher struct {
|
|
mu sync.Mutex
|
|
started []ShardCommand
|
|
stopped []string
|
|
drained []string
|
|
failNext error
|
|
// traces are the span each start was sent under, which is what the
|
|
// real dispatcher puts on the command for the agent.
|
|
traces []trace.SpanContext
|
|
}
|
|
|
|
func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
if d.failNext != nil {
|
|
err := d.failNext
|
|
d.failNext = nil
|
|
return err
|
|
}
|
|
d.started = append(d.started, cmd)
|
|
d.traces = append(d.traces, trace.SpanContextFromContext(ctx))
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.stopped = append(d.stopped, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.drained = append(d.drained, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) startedShards() []ShardCommand {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
return slices.Clone(d.started)
|
|
}
|
|
|
|
func (d *fakeDispatcher) reset() {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.started, d.stopped, d.drained = nil, nil, nil
|
|
}
|
|
|
|
// clock lets tests move time without sleeping.
|
|
type clock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
}
|
|
|
|
func newClock() *clock {
|
|
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
|
|
}
|
|
func (c *clock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.t
|
|
}
|
|
func (c *clock) advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
c.t = c.t.Add(d)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
|
|
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
|
|
for i := range n {
|
|
nd.GPUs = append(nd.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
|
|
})
|
|
}
|
|
for _, f := range fabrics {
|
|
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
|
|
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
|
|
})
|
|
}
|
|
return nd
|
|
}
|
|
|
|
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
|
|
if len(profiles) == 0 {
|
|
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
|
|
}
|
|
return catalog.Model{
|
|
Name: "llama", Source: "/models/llama",
|
|
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
|
|
}
|
|
}
|
|
|
|
type harness struct {
|
|
r *Reconciler
|
|
inv *inventory.Store
|
|
disp *fakeDispatcher
|
|
clock *clock
|
|
live *catalog.Live
|
|
}
|
|
|
|
// setModels replaces the catalog the reconciler works from, as an edit does.
|
|
func (h *harness) setModels(t *testing.T, models ...catalog.Model) {
|
|
t.Helper()
|
|
cat, err := catalog.New(models)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.live.Store(cat)
|
|
}
|
|
|
|
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
|
|
t.Helper()
|
|
|
|
cat, err := catalog.New([]catalog.Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inv := inventory.NewStore()
|
|
c := newClock()
|
|
for _, n := range nodes {
|
|
inv.Upsert(n)
|
|
inv.Touch(n.ID)
|
|
}
|
|
|
|
disp := &fakeDispatcher{}
|
|
cfg := DefaultConfig()
|
|
cfg.UnreachableAfter = 15 * time.Second
|
|
cfg.LostAfter = 60 * time.Second
|
|
|
|
live := catalog.NewLive(cat)
|
|
r := New(cfg, live, inv, disp, slog.New(slog.DiscardHandler))
|
|
r.SetClock(c.now)
|
|
// Nodes were just registered, so their LastSeen is wall-clock now, which is
|
|
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
|
|
for _, n := range nodes {
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
// Past the startup settling window. These nodes are all here — the harness
|
|
// built the whole fleet before anything ran — which is the state settling
|
|
// waits for, not one it should be made to simulate in every test. The
|
|
// window's own behaviour is tested separately.
|
|
c.advance(cfg.SettleFor)
|
|
for _, n := range nodes {
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
return &harness{r: r, inv: inv, disp: disp, clock: c, live: live}
|
|
}
|
|
|
|
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
|
|
func touchAt(inv *inventory.Store, id string, at time.Time) {
|
|
n, ok := inv.Get(id)
|
|
if !ok {
|
|
return
|
|
}
|
|
n.LastSeen = at
|
|
n.State = inventory.Ready
|
|
inv.Upsert(n)
|
|
// Upsert stamps LastSeen with the wall clock, so set it again directly.
|
|
inv.SetLastSeen(id, at)
|
|
}
|
|
|
|
// advance moves the clock and keeps every node heartbeating.
|
|
//
|
|
// Without this, advancing time past UnreachableAfter makes every node look
|
|
// silent and placement stops — which is correct behaviour, but not what the
|
|
// backoff tests are trying to exercise.
|
|
func (h *harness) advance(d time.Duration) {
|
|
h.clock.advance(d)
|
|
for _, n := range h.inv.Nodes() {
|
|
touchAt(h.inv, n.ID, h.clock.now())
|
|
}
|
|
}
|
|
|
|
// markReady walks every started shard to Ready.
|
|
func (h *harness) markReady() {
|
|
for _, cmd := range h.disp.startedShards() {
|
|
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
|
|
}
|
|
}
|
|
|
|
func TestPlacesMinimumReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 2 {
|
|
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
|
|
}
|
|
for _, rep := range reps {
|
|
if rep.State != Ready {
|
|
t.Errorf("replica %s is %s", rep.ID, rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Errorf("replica %s has no endpoint", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOneReplicaPerPass(t *testing.T) {
|
|
// Placement depends on what is already allocated, so starting several from
|
|
// one inventory snapshot would hand them the same GPUs.
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Fatalf("one pass created %d replicas", got)
|
|
}
|
|
|
|
seen := map[int]string{}
|
|
for range 3 {
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
for _, g := range s.GPUIndices {
|
|
key := g
|
|
if prev, dup := seen[key]; dup && prev != s.NodeID {
|
|
continue // same index on a different node is fine
|
|
}
|
|
seen[key] = s.NodeID
|
|
}
|
|
}
|
|
}
|
|
if got := len(h.r.Replicas()); got != 3 {
|
|
t.Errorf("expected 3 replicas, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestSpreadsAcrossNodes(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
nodes := map[string]struct{}{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, n := range rep.NodeIDs() {
|
|
nodes[n] = struct{}{}
|
|
}
|
|
}
|
|
if len(nodes) != 2 {
|
|
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
|
|
}
|
|
}
|
|
|
|
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
|
|
// Spreading is a preference, not a rule: a second replica on one node still
|
|
// survives an engine crash, which is the common failure.
|
|
h := newHarness(t, testModel(2), gpuNode("only", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("expected 2 replicas on the single node, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestShardOrderLeaderFirst(t *testing.T) {
|
|
// Both engines put the rendezvous point on rank 0 — SGLang's dist-init
|
|
// address and vLLM's Ray head — so a worker started first has nothing to
|
|
// connect to and spends its startup budget retrying.
|
|
//
|
|
// This test previously asserted the opposite, on the theory that the head
|
|
// should wait for its workers. That is backwards: vLLM's server blocks
|
|
// until the Ray cluster has the GPUs it needs, so the leader can start
|
|
// immediately and waits on its own.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
started := h.disp.startedShards()
|
|
if len(started) != 2 {
|
|
t.Fatalf("expected 2 shards, got %d", len(started))
|
|
}
|
|
if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 {
|
|
t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+
|
|
"point and must start first",
|
|
started[0].Spec.Rank, started[1].Spec.Rank)
|
|
}
|
|
// Every rank must be given the same rendezvous point.
|
|
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
|
|
t.Errorf("ranks got different leader addresses: %q and %q",
|
|
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
|
|
}
|
|
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
|
|
t.Error("ranks got different leader ports")
|
|
}
|
|
for _, cmd := range started {
|
|
if len(cmd.Spec.HCAs) == 0 {
|
|
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFailedShardFailsWholeReplica(t *testing.T) {
|
|
// One dead shard makes the others useless: they wait on collectives that
|
|
// will never complete while holding GPUs (plan §5.4).
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
|
|
// One shard ready, the other dead.
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
|
|
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Both shards must have been stopped, not just the failed one.
|
|
h.disp.mu.Lock()
|
|
stopped := slices.Clone(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if len(stopped) != 2 {
|
|
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
|
|
}
|
|
// And the GPUs must be free again.
|
|
for _, id := range []string{"1", "2"} {
|
|
if got := len(h.inv.FreeGPUs(id)); got != 8 {
|
|
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackoffAfterFailure(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context()) // reaps it
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
|
|
}
|
|
|
|
// Past the backoff, it tries again.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("the replica was never rebuilt after the backoff expired")
|
|
}
|
|
}
|
|
|
|
func TestSuccessClearsBackoff(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
// Fail once.
|
|
h.r.Reconcile(t.Context())
|
|
s := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Then succeed.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Fatal("no replica was started after the backoff expired")
|
|
}
|
|
s2 := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
|
|
|
|
// A later failure should start from a short backoff, not a long streak.
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(2 * DefaultConfig().BaseBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("backoff was not reset by the successful run in between")
|
|
}
|
|
}
|
|
|
|
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
|
|
// must get its replicas rebuilt elsewhere.
|
|
func TestLostNodeMovesReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("setup failed: %+v", reps)
|
|
}
|
|
original := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if original == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
// The surviving node keeps reporting; the other goes silent.
|
|
h.clock.advance(20 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
|
|
t.Errorf("silent node is %s, expected unreachable", n.State)
|
|
}
|
|
// Unreachable is not lost: a blip must not tear down a working replica.
|
|
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
|
|
}
|
|
|
|
h.clock.advance(90 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
|
|
t.Fatalf("node is %s, expected lost", n.State)
|
|
}
|
|
|
|
// Past the backoff, the replica is rebuilt on the survivor. Only the
|
|
// survivor heartbeats; the lost node stays gone.
|
|
h.clock.advance(DefaultConfig().MaxBackoff)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps = h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
|
|
}
|
|
if got := reps[0].NodeIDs()[0]; got != survivor {
|
|
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
|
|
}
|
|
if reps[0].State != Ready {
|
|
t.Errorf("rebuilt replica is %s", reps[0].State)
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatRevivesNode(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
|
|
h.r.Reconcile(t.Context())
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
|
|
t.Fatalf("node is %s", n.State)
|
|
}
|
|
|
|
h.inv.Touch("1")
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
|
|
t.Errorf("a heartbeat did not revive the node: %s", n.State)
|
|
}
|
|
}
|
|
|
|
func TestScaleDownDrainsFirst(t *testing.T) {
|
|
// Draining before stopping lets in-flight requests finish instead of being
|
|
// cut off mid-stream.
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("setup: %d replicas", got)
|
|
}
|
|
|
|
// Shrink the catalog to one replica.
|
|
cat, err := catalog.New([]catalog.Model{testModel(1)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(cat)
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the excess replica was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("a replica was stopped before it had drained")
|
|
}
|
|
|
|
// The next pass stops what has drained.
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped")
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica after scale-down, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestUnplaceableModelExplains(t *testing.T) {
|
|
// A model that cannot be placed must leave a usable explanation rather than
|
|
// a silent absence.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "tp8", GPUClass: "h100-80g", TP: 8,
|
|
}), gpuNode("1", 4))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Fatalf("something was placed: %d", got)
|
|
}
|
|
|
|
res, ok := h.r.Explain("llama")
|
|
if !ok {
|
|
t.Fatal("no explanation was recorded")
|
|
}
|
|
exp := res.Explain()
|
|
if !strings.Contains(exp, "needs 8") {
|
|
t.Errorf("explanation does not say what was missing:\n%s", exp)
|
|
}
|
|
}
|
|
|
|
func TestDispatcherFailureFailsReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.disp.failNext = fmt.Errorf("agent unreachable")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass must release its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.inv.FreeGPUs("1")); got != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
|
|
}
|
|
}
|
|
|
|
func TestReadyReplicasFeedTheRouter(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.ReadyReplicas()); got != 0 {
|
|
t.Errorf("a starting replica was offered to the router")
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
ready := h.r.ReadyReplicas()
|
|
if len(ready) != 2 {
|
|
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
|
|
}
|
|
for _, rep := range ready {
|
|
if !strings.HasPrefix(rep.Endpoint, "http://") {
|
|
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPortsDoNotCollide(t *testing.T) {
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
ports := map[int]string{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
|
|
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
|
|
}
|
|
ports[s.HTTPPort] = rep.ID
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReplicaStateStrings(t *testing.T) {
|
|
for st, want := range map[ReplicaState]string{
|
|
Placing: "placing", Starting: "starting", Ready: "ready",
|
|
Draining: "draining", Stopped: "stopped", Failed: "failed",
|
|
} {
|
|
if got := st.String(); got != want {
|
|
t.Errorf("%d -> %q", int(st), got)
|
|
}
|
|
}
|
|
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
|
|
// Draining must not count, or a replacement is never started for it.
|
|
t.Error("Counts is wrong")
|
|
}
|
|
}
|
|
|
|
// --- adoption after a control plane restart (plan §6.5) ---
|
|
|
|
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
|
|
a := AdoptedShard{
|
|
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
|
|
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
|
|
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
|
|
}
|
|
if rank == 0 {
|
|
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
|
|
}
|
|
return a
|
|
}
|
|
|
|
func TestAdoptRebuildsReplica(t *testing.T) {
|
|
// The control plane restarted. Nothing is known, but a node reports a live
|
|
// replica. Without this the reconciler would place a duplicate while the
|
|
// original keeps holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
|
|
}
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
|
|
}
|
|
rep := reps[0]
|
|
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
|
|
t.Errorf("identity wrong: %+v", rep)
|
|
}
|
|
if rep.State != Ready {
|
|
t.Errorf("state=%s; every shard reported Ready", rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Error("no endpoint; the router would have nowhere to send")
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Error("weight not taken from the catalog")
|
|
}
|
|
|
|
// The GPUs must be reclaimed, or the next placement hands them out twice.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
|
|
// The whole point: after adopting, a reconciliation pass must see the
|
|
// desired count already met.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptMultiNodeReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
shards := []AdoptedShard{
|
|
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
}
|
|
for i := range shards {
|
|
shards[i].Profile = "2node"
|
|
shards[i].Fabric = "ib-a"
|
|
shards[i].Pool = "ib-a/h100-80g"
|
|
}
|
|
|
|
if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 {
|
|
t.Fatalf("unclaimed: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
|
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
|
|
}
|
|
if reps[0].Fabric != "ib-a" {
|
|
t.Errorf("fabric lost: %q", reps[0].Fabric)
|
|
}
|
|
// Both nodes' GPUs must be reclaimed.
|
|
for _, id := range []string{"1", "2"} {
|
|
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
|
t.Errorf("node %s has %d free GPUs after adoption", id, free)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdoptPartialReplicaFails(t *testing.T) {
|
|
// A multi-node replica missing a shard is useless: the survivors wait on
|
|
// collectives that will never complete while holding GPUs.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile = "2node"
|
|
s.State = supervisor.Failed
|
|
h.r.Adopt("1", []AdoptedShard{s})
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass releases its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
|
|
// The model left the catalog while the engine kept running. Nothing will
|
|
// ever route to it, and it is holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
|
|
s.Model = "a-model-that-was-removed"
|
|
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{s})
|
|
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
|
|
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a replica was created for an unknown model: %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
|
|
s.Profile = "a-profile-that-was-removed"
|
|
|
|
if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 {
|
|
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
|
|
}
|
|
}
|
|
|
|
func TestAdoptIsIdempotent(t *testing.T) {
|
|
// An agent reconnecting re-reports the same shards; that must not double
|
|
// the allocation.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
|
|
|
|
h.r.Adopt("1", s)
|
|
h.r.Adopt("1", s)
|
|
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("re-reporting created %d replicas", got)
|
|
}
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
|
|
}
|
|
}
|
|
|
|
// --- draining a node (ctl drain) ---
|
|
|
|
func TestDrainingNodeMovesWorkOff(t *testing.T) {
|
|
// The operator's view: a node is about to be worked on, its replicas must
|
|
// end up elsewhere, and nothing may be cut off mid-request on the way.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
victim := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if victim == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
h.inv.SetState(victim, inventory.Draining)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// It is drained, not stopped: the engine is healthy and still serving.
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the replica on the draining node was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("the replica was stopped before a replacement existed; in-flight requests would be cut")
|
|
}
|
|
|
|
// A draining replica does not count, so a replacement is placed elsewhere.
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Fatal("no replacement was placed")
|
|
}
|
|
for _, cmd := range h.disp.startedShards() {
|
|
if cmd.NodeID == victim {
|
|
t.Errorf("the replacement was placed back on the draining node %s", victim)
|
|
}
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Now that the replacement serves, the original can go.
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped once its replacement was ready")
|
|
}
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.OnNode(victim) {
|
|
t.Errorf("replica %s is still on the drained node", rep.ID)
|
|
}
|
|
if !slices.Contains(rep.NodeIDs(), survivor) {
|
|
t.Errorf("replica %s did not move to the surviving node", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDrainGraceStopsWithoutReplacement(t *testing.T) {
|
|
// Draining the only node that can host a model would otherwise never
|
|
// finish, leaving the node undrainable.
|
|
h := newHarness(t, testModel(1), gpuNode("only", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.inv.SetState("only", inventory.Draining)
|
|
h.r.Reconcile(t.Context()) // drains it; nowhere to place a replacement
|
|
|
|
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Draining {
|
|
t.Fatalf("expected a draining replica, got %+v", got)
|
|
}
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
early := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if early != 0 {
|
|
t.Error("the replica was stopped before the grace period expired")
|
|
}
|
|
|
|
h.advance(DefaultConfig().DrainGrace + time.Second)
|
|
h.inv.SetState("only", inventory.Draining) // advance() resets node state
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
stopped := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped; the node cannot be drained")
|
|
}
|
|
}
|
|
|
|
func TestCordonedNodeKeepsItsWork(t *testing.T) {
|
|
// Cordon is not drain: it stops new placement and leaves what is running.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.inv.SetState("1", inventory.Cordoned)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained != 0 || stopped != 0 {
|
|
t.Errorf("cordoning disturbed a running replica (drained=%d stopped=%d)", drained, stopped)
|
|
}
|
|
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Ready {
|
|
t.Errorf("the replica did not survive a cordon: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptMultiNodeReplicaOneAgentAtATime(t *testing.T) {
|
|
// How it really arrives. Every other multi-node adoption test hands both
|
|
// shards to a single Adopt call, which no caller ever does: each node has
|
|
// its own agent and each agent registers separately, so a two-node replica
|
|
// is reported in two halves a second or so apart.
|
|
//
|
|
// Observed on the cluster after a control plane restart: the first half was
|
|
// adopted and immediately failed for "replica has 1 of 2 shards", and when
|
|
// the second node registered its half was stopped as unclaimed. big-model
|
|
// never came back — it rebuilt from scratch on every restart, with
|
|
// exponential backoff, forever.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
half := func(shardID, nodeID string, rank int) []AdoptedShard {
|
|
s := adopted("llama-7", shardID, nodeID, rank, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g"
|
|
return []AdoptedShard{s}
|
|
}
|
|
|
|
// Rank 1 first, so the endpoint arrives in the second group: whichever node
|
|
// registers first is not necessarily the leader.
|
|
if unclaimed := h.r.Adopt("2", half("llama-7-r1", "2", 1)); len(unclaimed) != 0 {
|
|
t.Fatalf("the first half was not claimed: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected the half to be adopted, got %d replicas", len(reps))
|
|
}
|
|
if reps[0].State == Failed {
|
|
t.Fatalf("a replica still assembling was failed: %q", reps[0].Err)
|
|
}
|
|
|
|
if unclaimed := h.r.Adopt("1", half("llama-7-r0", "1", 0)); len(unclaimed) != 0 {
|
|
t.Fatalf("the second half was stopped as unclaimed: %+v", unclaimed)
|
|
}
|
|
|
|
reps = h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("the halves became %d replicas", len(reps))
|
|
}
|
|
rep := reps[0]
|
|
if len(rep.Shards) != 2 {
|
|
t.Fatalf("the halves were not merged: %d shards", len(rep.Shards))
|
|
}
|
|
if rep.State != Ready {
|
|
t.Errorf("state=%s err=%q; both shards reported Ready", rep.State, rep.Err)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Error("no endpoint; rank 0 arrived in the second group and was dropped")
|
|
}
|
|
for _, id := range []string{"1", "2"} {
|
|
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
|
t.Errorf("node %s has %d free GPUs; its half's allocation was lost", id, free)
|
|
}
|
|
}
|
|
|
|
// And the reconciler must not then place a duplicate alongside it.
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a duplicate was placed despite the replica being whole (%d shards started)", got)
|
|
}
|
|
}
|
|
|
|
func TestAHalfThatNeverArrivesStillFails(t *testing.T) {
|
|
// The grace period must be a deadline, not an amnesty: a replica whose
|
|
// other node is simply gone has to fail so it can be replaced.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
StartupTimeout: time.Minute,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g"
|
|
h.r.Adopt("1", []AdoptedShard{s})
|
|
|
|
if reps := h.r.Replicas(); reps[0].State != Starting {
|
|
t.Fatalf("state=%s; the half should be waiting for its sibling", reps[0].State)
|
|
}
|
|
|
|
h.advance(2 * time.Minute)
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) == 1 && reps[0].State != Failed {
|
|
t.Errorf("state=%s; a half whose sibling never registered must fail", reps[0].State)
|
|
}
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("node has %d free GPUs; the abandoned half still holds them", free)
|
|
}
|
|
}
|
|
|
|
// --- the startup settling window ---
|
|
|
|
func settlingHarness(t *testing.T, m catalog.Model) (*Reconciler, *inventory.Store, *fakeDispatcher, *clock) {
|
|
t.Helper()
|
|
cat, err := catalog.New([]catalog.Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inv := inventory.NewStore()
|
|
disp := &fakeDispatcher{}
|
|
c := newClock()
|
|
r := New(DefaultConfig(), catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
|
|
r.SetClock(c.now)
|
|
return r, inv, disp, c
|
|
}
|
|
|
|
func addNode(inv *inventory.Store, c *clock, n inventory.Node) {
|
|
inv.Upsert(n)
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
|
|
func TestPlacementWaitsForTheFleetToArrive(t *testing.T) {
|
|
// A control plane that has just started knows nothing, and the first tick
|
|
// runs before any agent has registered. Placing then puts everything on
|
|
// whichever fraction reported first: after one restart both replicas of a
|
|
// model landed on the single node that had registered, when four were
|
|
// coming.
|
|
r, inv, disp, c := settlingHarness(t, testModel(2))
|
|
|
|
// Nothing registered yet.
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards on an empty fleet", got)
|
|
}
|
|
|
|
// One node of four.
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards while the fleet was still arriving", got)
|
|
}
|
|
|
|
// The rest arrive.
|
|
for _, id := range []string{"2", "3", "4"} {
|
|
addNode(inv, c, gpuNode(id, 8))
|
|
}
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards the instant the last node appeared", got)
|
|
}
|
|
|
|
// Steady for long enough that a reconnect backoff would have finished: now
|
|
// it may act, and on the whole fleet.
|
|
c.advance(DefaultConfig().SettleFor / 2)
|
|
for _, id := range []string{"1", "2", "3", "4"} {
|
|
touchAt(inv, id, c.now())
|
|
}
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got == 0 {
|
|
t.Fatal("placement never resumed after the fleet settled")
|
|
}
|
|
}
|
|
|
|
func TestPlacementDoesNotWaitForeverForNodesThatNeverCome(t *testing.T) {
|
|
// Waiting only for the count to stop changing would hang forever on a
|
|
// cluster whose nodes never all come back.
|
|
r, inv, disp, c := settlingHarness(t, testModel(1))
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
|
|
c.advance(DefaultConfig().SettleFor)
|
|
touchAt(inv, "1", c.now())
|
|
r.Reconcile(t.Context())
|
|
|
|
if got := len(disp.startedShards()); got == 0 {
|
|
t.Error("placement never resumed once the window expired")
|
|
}
|
|
}
|
|
|
|
func TestSettlingIsOnlyEverPaidOnce(t *testing.T) {
|
|
// A node joining later in the day is an ordinary event, not a reason to
|
|
// stop placing while the cluster is running.
|
|
r, inv, disp, c := settlingHarness(t, testModel(1))
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
c.advance(DefaultConfig().SettleFor)
|
|
touchAt(inv, "1", c.now())
|
|
r.Reconcile(t.Context())
|
|
if len(disp.startedShards()) == 0 {
|
|
t.Fatal("nothing placed after the window")
|
|
}
|
|
|
|
disp.reset()
|
|
addNode(inv, c, gpuNode("2", 8))
|
|
if !r.fleetHasSettled() {
|
|
t.Error("a node joining a running cluster restarted the settling window")
|
|
}
|
|
}
|
|
|
|
func TestAdoptionIsNotBlockedByTheSettlingWindow(t *testing.T) {
|
|
// Only placement waits. Adoption is how a restart recovers what is already
|
|
// running, and delaying it would leave live engines unaccounted for exactly
|
|
// when the reconciler is deciding what to build.
|
|
r, inv, _, c := settlingHarness(t, testModel(1))
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
|
|
unclaimed := r.Adopt("1", []AdoptedShard{
|
|
adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}),
|
|
})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("adoption was refused during settling: %+v", unclaimed)
|
|
}
|
|
if got := len(r.Replicas()); got != 1 {
|
|
t.Fatalf("expected the replica to be adopted, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestSettlingOutlastsAnAgentReconnectBackoff(t *testing.T) {
|
|
// The failure this was measured against. After a restart node05 registered
|
|
// in one second and the other four arrived seventeen seconds later, because
|
|
// a disconnected agent reconnects on a backoff reaching thirty seconds. A
|
|
// short steady period declared a fleet of one settled and resumed placement
|
|
// while four nodes were still coming — the exact situation the window
|
|
// exists to avoid, reached by being impatient about it.
|
|
r, inv, disp, c := settlingHarness(t, testModel(2))
|
|
|
|
addNode(inv, c, gpuNode("first", 8))
|
|
// Fifteen seconds of one node, which a short threshold would call settled.
|
|
for range 3 {
|
|
c.advance(5 * time.Second)
|
|
touchAt(inv, "first", c.now())
|
|
r.Reconcile(t.Context())
|
|
}
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards on one node while others were backing off", got)
|
|
}
|
|
|
|
// The stragglers arrive, as they did at seventeen seconds. The loop runs on
|
|
// a ticker, so the new count is noticed on the next pass, not the instant
|
|
// the nodes appear.
|
|
for _, id := range []string{"2", "3", "4"} {
|
|
addNode(inv, c, gpuNode(id, 8))
|
|
}
|
|
r.Reconcile(t.Context())
|
|
|
|
c.advance(DefaultConfig().SettleFor / 2)
|
|
for _, id := range []string{"first", "2", "3", "4"} {
|
|
touchAt(inv, id, c.now())
|
|
}
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got == 0 {
|
|
t.Fatal("placement never resumed")
|
|
}
|
|
}
|
|
|
|
// --- idle tracking ---
|
|
|
|
func TestUsageIsRecordedAgainstReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected a replica, got %d", len(reps))
|
|
}
|
|
id := reps[0].ID
|
|
|
|
if _, known := h.r.IdleFor(id); known {
|
|
t.Error("a replica nothing has used reported a known idleness")
|
|
}
|
|
|
|
used := h.clock.now()
|
|
h.r.RecordUsage(map[string]time.Time{id: used})
|
|
h.advance(5 * time.Minute)
|
|
|
|
idle, known := h.r.IdleFor(id)
|
|
if !known {
|
|
t.Fatal("usage was recorded but idleness is still unknown")
|
|
}
|
|
if idle != 5*time.Minute {
|
|
t.Errorf("idle = %s, want 5m", idle)
|
|
}
|
|
}
|
|
|
|
func TestUsageNeverMovesBackwards(t *testing.T) {
|
|
// Two routers reporting, or a snapshot arriving late, must not make a busy
|
|
// replica look abandoned.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
id := h.r.Replicas()[0].ID
|
|
|
|
recent := h.clock.now()
|
|
h.r.RecordUsage(map[string]time.Time{id: recent})
|
|
h.r.RecordUsage(map[string]time.Time{id: recent.Add(-time.Hour)})
|
|
|
|
h.advance(time.Minute)
|
|
if idle, _ := h.r.IdleFor(id); idle != time.Minute {
|
|
t.Errorf("idle = %s; a stale report moved last-use backwards", idle)
|
|
}
|
|
}
|
|
|
|
func TestUsageForReplicasWeDoNotHaveIsIgnored(t *testing.T) {
|
|
// The router's list and the reconciler's drift constantly on a live fleet:
|
|
// a replica stops between the snapshot and the read. That is ordinary, not
|
|
// an error, and must not panic or create phantom state.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
before := len(h.r.Replicas())
|
|
|
|
h.r.RecordUsage(map[string]time.Time{"a-replica-that-left": h.clock.now()})
|
|
if got := len(h.r.Replicas()); got != before {
|
|
t.Errorf("replica count changed from %d to %d", before, got)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyUsageReportLeavesEverythingAlone(t *testing.T) {
|
|
// A router that has just started knows nothing, and treating that as "never
|
|
// used" would make every replica look abandoned at once — exactly when a
|
|
// fleet is least able to afford acting on it.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
id := h.r.Replicas()[0].ID
|
|
|
|
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
|
h.advance(time.Minute)
|
|
h.r.RecordUsage(nil)
|
|
|
|
if idle, known := h.r.IdleFor(id); !known || idle != time.Minute {
|
|
t.Errorf("idle=%s known=%v; an empty report erased what was known", idle, known)
|
|
}
|
|
}
|
|
|
|
func TestAModelAddedWhileRunningIsPlaced(t *testing.T) {
|
|
// The reason the catalog stopped being fixed at startup. Adding a model was
|
|
// an operator procedure with downtime: edit a file on the control plane,
|
|
// restart it, wait out the settling window, watch adoption churn. None of
|
|
// that was necessary — the loop already compares desired against actual
|
|
// every pass, so a new catalog is simply a different desired.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Fatalf("expected the original model's replica, got %d", got)
|
|
}
|
|
|
|
second := testModel(1)
|
|
second.Name = "llama-two"
|
|
next, err := catalog.New([]catalog.Model{testModel(1), second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(next)
|
|
|
|
h.r.Reconcile(t.Context())
|
|
byModel := map[string]int{}
|
|
for _, rep := range h.r.Replicas() {
|
|
byModel[rep.Model]++
|
|
}
|
|
if byModel["llama-two"] == 0 {
|
|
t.Errorf("a model added while running was never placed: %v", byModel)
|
|
}
|
|
if byModel["llama"] == 0 {
|
|
t.Errorf("the model that was already running was disturbed: %v", byModel)
|
|
}
|
|
}
|
|
|
|
func TestAModelRemovedWhileRunningIsStopped(t *testing.T) {
|
|
// The other direction, and the one with consequences: a replica whose model
|
|
// has left the catalog holds GPUs for something nothing can ask for.
|
|
first := testModel(1)
|
|
second := testModel(1)
|
|
second.Name = "llama-two"
|
|
both, err := catalog.New([]catalog.Model{first, second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newHarness(t, first, gpuNode("1", 8), gpuNode("2", 8))
|
|
h.r.cat.Store(both)
|
|
h.r.Reconcile(t.Context())
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.r.Replicas()) < 2 {
|
|
t.Fatalf("expected both models placed, got %d", len(h.r.Replicas()))
|
|
}
|
|
|
|
only, err := catalog.New([]catalog.Model{first})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(only)
|
|
|
|
// Several passes: stopping is deliberate rather than immediate, and a
|
|
// replica is not torn down in the same breath it is noticed.
|
|
for range 4 {
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.Model == "llama-two" && rep.State.Counts() {
|
|
t.Errorf("a replica of a removed model is still counted: %+v", rep)
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- on-demand loading and idle eviction ---
|
|
|
|
func onDemandModel(idle time.Duration) catalog.Model {
|
|
m := testModel(1)
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle}
|
|
return m
|
|
}
|
|
|
|
func TestAnOnDemandModelIsNotRunningUntilItIsAskedFor(t *testing.T) {
|
|
// The point of the whole feature: a cluster with more models than GPUs can
|
|
// hold every model in its catalog and only the wanted ones in its cards.
|
|
h := newHarness(t, onDemandModel(time.Hour), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Fatalf("an on-demand model was placed before anyone asked: %d replicas", got)
|
|
}
|
|
|
|
if !h.r.Want("llama") {
|
|
t.Fatal("Want reported the model is not on-demand")
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("asking for the model produced %d replicas, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestWantSaysWhenAModelDoesNotLoadOnDemand(t *testing.T) {
|
|
// The caller needs to tell "being loaded" from "there is genuinely nothing
|
|
// serving this", because those are different answers to give a client.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8)) // min 1, not on-demand
|
|
if h.r.Want("llama") {
|
|
t.Error("an ordinary model was reported as loading on demand")
|
|
}
|
|
if h.r.Want("a-model-that-does-not-exist") {
|
|
t.Error("an unknown model was reported as loading on demand")
|
|
}
|
|
}
|
|
|
|
func TestAnIdleOnDemandModelIsStopped(t *testing.T) {
|
|
// The other half. Without it, on-demand is just a slower way to fill the
|
|
// cluster permanently.
|
|
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
if len(h.r.Replicas()) != 1 {
|
|
t.Fatalf("the model did not start")
|
|
}
|
|
|
|
h.advance(11 * time.Minute)
|
|
h.r.Reconcile(t.Context())
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
t.Errorf("a replica idle past its timeout is still counted: %+v", rep)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUseKeepsAnOnDemandModelAlive(t *testing.T) {
|
|
// Without this the timer measures time since the request that loaded the
|
|
// model rather than the last one, so a busy model would be evicted exactly
|
|
// IdleTimeout after it started however hard it was working.
|
|
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("the model did not start")
|
|
}
|
|
id := reps[0].ID
|
|
|
|
// Busy for half an hour, in six-minute steps.
|
|
for range 5 {
|
|
h.advance(6 * time.Minute)
|
|
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
|
|
var live int
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
live++
|
|
}
|
|
}
|
|
if live != 1 {
|
|
t.Errorf("a model in continuous use was evicted: %d live replicas", live)
|
|
}
|
|
}
|
|
|
|
func TestAnAwakeOnDemandModelSurvivesARestartOfTheControlPlane(t *testing.T) {
|
|
// A control plane that has just started, or just become leader, knows
|
|
// nobody asked for anything. Taking that to mean nobody wants the
|
|
// on-demand models a node reports running drained every one of them at
|
|
// every failover, under whoever was using them.
|
|
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
|
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(5 * time.Minute)
|
|
h.r.Reconcile(t.Context())
|
|
if reps := h.r.Replicas(); len(reps) != 1 || !reps[0].State.Counts() {
|
|
t.Fatalf("the adopted replica of an on-demand model was taken down: %+v", reps)
|
|
}
|
|
// Still an on-demand model: unused for a whole idle timeout, it goes.
|
|
h.advance(6 * time.Minute)
|
|
h.r.Reconcile(t.Context())
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
t.Errorf("an adopted replica nobody used outlived its idle timeout: %+v", rep)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAModelWithMaxZeroStaysOff(t *testing.T) {
|
|
// min 0 max 0 is configured and deliberately not running, which is a
|
|
// different and equally useful thing from "load it when wanted".
|
|
m := testModel(1)
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 0}
|
|
h := newHarness(t, m, gpuNode("1", 8))
|
|
|
|
if h.r.Want("llama") {
|
|
t.Error("a model with max 0 was treated as on-demand")
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a model switched off was placed: %d replicas", got)
|
|
}
|
|
}
|
|
|
|
func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) {
|
|
// Every model that exists today. A model with min 1 stays up whether anyone
|
|
// uses it or not, because that is what min 1 means.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
// Ready, or the startup timeout is what removes it twenty-four hours later
|
|
// and the test proves nothing about idleness.
|
|
h.markReady()
|
|
if len(h.r.Replicas()) != 1 {
|
|
t.Fatal("the model did not start")
|
|
}
|
|
|
|
h.advance(24 * time.Hour)
|
|
h.r.Reconcile(t.Context())
|
|
|
|
var live int
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
live++
|
|
}
|
|
}
|
|
if live != 1 {
|
|
t.Errorf("an always-on model was evicted for being idle: %d live", live)
|
|
}
|
|
}
|
|
|
|
// --- eviction under pressure ---
|
|
|
|
// twoOnDemandModels builds a catalog where each model fills the whole node, so
|
|
// the second can only run if the first stops.
|
|
func twoOnDemandModels(t *testing.T) *catalog.Catalog {
|
|
t.Helper()
|
|
mk := func(name string) catalog.Model {
|
|
m := testModel(1)
|
|
m.Name = name
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: time.Hour}
|
|
return m
|
|
}
|
|
built, err := catalog.New([]catalog.Model{mk("llama"), mk("second")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return built
|
|
}
|
|
|
|
func TestAWantedModelEvictsAnIdleOneWhenTheClusterIsFull(t *testing.T) {
|
|
// On-demand loading works right up to the moment the cluster is full and
|
|
// then stops meaning anything: the model somebody just asked for cannot be
|
|
// placed, and the cards are held by one nobody has asked for in an hour.
|
|
// That is the only situation in which any of this matters.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
first := h.r.Replicas()
|
|
if len(first) != 1 {
|
|
t.Fatalf("the first model did not start: %d replicas", len(first))
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{first[0].ID: h.clock.now()})
|
|
|
|
// It goes quiet. Past the eviction grace and the minimum lifetime, so it is
|
|
// genuinely a candidate rather than merely recent.
|
|
h.advance(30 * time.Minute)
|
|
|
|
// Asked for directly, rather than through a Reconcile pass, because two
|
|
// on-demand models both wanting the only free node is a race the scheduler
|
|
// resolves in whatever order the catalog iterates — and what is being
|
|
// tested here is that room gets made, not who wins.
|
|
second := mustResolve(t, h, "second")
|
|
if !h.r.makeRoomFor(t.Context(), second) {
|
|
t.Fatal("no room was made for a model that could not be placed")
|
|
}
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.Model == "llama" && rep.State.Counts() {
|
|
t.Error("the idle replica was not evicted")
|
|
}
|
|
}
|
|
// And its GPUs are back, which is what the wanted model needed.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs after the eviction, want 4", free)
|
|
}
|
|
}
|
|
|
|
func TestPlacementFailureTriggersEviction(t *testing.T) {
|
|
// The wiring: scaleUp is what notices there is nowhere to put a model, and
|
|
// is the only place that knows to try making room.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(30 * time.Minute)
|
|
|
|
// The full node has no room, so placing this must free some.
|
|
h.r.scaleUp(t.Context(), mustResolve(t, h, "second"), nil)
|
|
|
|
if free := len(h.inv.FreeGPUs("1")); free == 0 {
|
|
t.Error("a placement failure did not make room")
|
|
}
|
|
}
|
|
|
|
func TestAnAlwaysOnModelIsNeverEvicted(t *testing.T) {
|
|
// min 1 is a promise that it stays up. Quietly breaking it to serve another
|
|
// request would make every guarantee in the catalog conditional on load.
|
|
alwaysOn := testModel(1) // min 1
|
|
onDemand := testModel(1)
|
|
onDemand.Name = "second"
|
|
onDemand.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
|
built, err := catalog.New([]catalog.Model{alwaysOn, onDemand})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newHarness(t, alwaysOn, gpuNode("1", 4))
|
|
h.r.cat.Store(built)
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("the always-on model did not start")
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(24 * time.Hour)
|
|
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("an always-on model was offered for eviction: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaNothingHasReportedIsNeverEvicted(t *testing.T) {
|
|
// Unknown is not idle. After a router restart that is every replica, and
|
|
// evicting on the strength of a fact nobody has reported would empty the
|
|
// cluster at exactly the wrong moment.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
h.advance(24 * time.Hour) // idle by any measure, but nothing has said so
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("a replica with no reported usage was offered for eviction: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAMultiNodeReplicaIsLeftAlone(t *testing.T) {
|
|
// Evicting one frees several machines at once and takes tens of minutes to
|
|
// rebuild — a decision worth making deliberately rather than as a side
|
|
// effect of one request.
|
|
m := testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
})
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
|
other := testModel(1)
|
|
other.Name = "second"
|
|
other.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
|
built, err := catalog.New([]catalog.Model{m, other})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
h.r.cat.Store(built)
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
|
t.Fatalf("expected a two-node replica, got %+v", reps)
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(24 * time.Hour)
|
|
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("a multi-node replica was offered for eviction: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestOnlyOneReplicaIsEvictedPerPass(t *testing.T) {
|
|
// Freeing everything that qualifies would empty a cluster to satisfy a
|
|
// request that may only have needed one node.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
h.r.Want("llama")
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
reps := h.r.Replicas()
|
|
used := map[string]time.Time{}
|
|
for _, rep := range reps {
|
|
used[rep.ID] = h.clock.now()
|
|
}
|
|
h.r.RecordUsage(used)
|
|
h.advance(time.Hour)
|
|
|
|
before := len(h.r.evictable("second"))
|
|
if before < 1 {
|
|
t.Skip("nothing evictable to begin with")
|
|
}
|
|
h.r.makeRoomFor(t.Context(), mustResolve(t, h, "second"))
|
|
|
|
var stopped int
|
|
for _, rep := range h.r.Replicas() {
|
|
if !rep.State.Counts() {
|
|
stopped++
|
|
}
|
|
}
|
|
if stopped > 1 {
|
|
t.Errorf("%d replicas were stopped in one pass", stopped)
|
|
}
|
|
}
|
|
|
|
func mustResolve(t *testing.T, h *harness, name string) catalog.Model {
|
|
t.Helper()
|
|
m, ok := h.r.cat.Load().Resolve(name)
|
|
if !ok {
|
|
t.Fatalf("model %q is not in the catalog", name)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
|
|
// What stops two models on a cluster with room for one from taking turns
|
|
// evicting each other, paying a model load each time. A load is expensive
|
|
// enough that waiting is strictly better than winning.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
|
|
// Used once, then idle well past the eviction grace — but only just
|
|
// started, which is the case this protects.
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(DefaultConfig().EvictionGrace + time.Minute)
|
|
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("a replica younger than the minimum lifetime was offered: %+v", got)
|
|
}
|
|
|
|
// Once it has been up long enough, it is fair game.
|
|
h.advance(DefaultConfig().MinLifetime)
|
|
if got := h.r.evictable("second"); len(got) == 0 {
|
|
t.Error("a long-running idle replica was never offered for eviction")
|
|
}
|
|
}
|
|
|
|
func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) {
|
|
// The fingerprint has to leave the control plane, or it cannot come back
|
|
// after a restart — and a restart is exactly when the configuration is
|
|
// likely to have been edited.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
t.Fatal("nothing was started")
|
|
}
|
|
// From the catalog rather than the literal, because the catalog fills in
|
|
// defaults and the fingerprint is of the profile as it is actually run.
|
|
m, ok := h.r.cat.Load().Resolve("llama")
|
|
if !ok {
|
|
t.Fatal("the model is not in the catalog")
|
|
}
|
|
want := m.Profiles[0].Fingerprint()
|
|
for _, cmd := range started {
|
|
if cmd.ProfileFP != want {
|
|
t.Errorf("shard %s carries %q, expected %q", cmd.ShardID, cmd.ProfileFP, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnAdoptedReplicaKeepsTheProfileVersionItWasStartedWith(t *testing.T) {
|
|
// Adoption used to leave this empty, which made every replica read as
|
|
// unknown after a control plane restart — so an edited profile could never
|
|
// be noticed on the machines that were already running.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
shard := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})
|
|
shard.ProfileFP = "1122334455667788"
|
|
h.r.Adopt("1", []AdoptedShard{shard})
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
|
|
}
|
|
if reps[0].ProfileFP != "1122334455667788" {
|
|
t.Errorf("the agent's answer was dropped: %q", reps[0].ProfileFP)
|
|
}
|
|
}
|
|
|
|
func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
|
|
// Empty, not filled in from the catalog. Copying the current profile here
|
|
// would make the replica claim to match whatever the catalog says next,
|
|
// which is the opposite of what the field is for.
|
|
if fp := h.r.Replicas()[0].ProfileFP; fp != "" {
|
|
t.Errorf("an unreported fingerprint should stay empty, got %q", fp)
|
|
}
|
|
}
|
|
|
|
func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) {
|
|
// Engines listen on every address a node has. Started without a key, any
|
|
// machine that could reach the port reached the model past the gateway's
|
|
// authentication, quotas and accounting — checked from a neighbouring node
|
|
// on the live cluster.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id }
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
t.Fatal("nothing was started")
|
|
}
|
|
for _, c := range started {
|
|
if c.Spec.APIKey != "key-for-"+c.ReplicaID {
|
|
t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey)
|
|
}
|
|
}
|
|
}
|
|
|
|
func replicaByID(h *harness, id string) (Replica, bool) {
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.ID == id {
|
|
return rep, true
|
|
}
|
|
}
|
|
return Replica{}, false
|
|
}
|
|
|
|
func TestARestartedReplicaServesUntilItsReplacementIsReady(t *testing.T) {
|
|
// Restarting is how a running engine picks up a key, an edited profile or
|
|
// a new engine version. Stopping first would leave a one-replica model
|
|
// unserved for as long as its replacement takes to load.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if rep, _ := replicaByID(h, old); rep.State != Ready || h.r.RestartState(old) != "replacing" {
|
|
t.Fatalf("the old replica stopped serving before its replacement was up: %+v", rep)
|
|
}
|
|
if n := len(h.r.Replicas()); n != 2 {
|
|
t.Fatalf("no replacement was placed beside it: %d replicas", n)
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.r.Reconcile(t.Context())
|
|
if _, still := replicaByID(h, old); still {
|
|
if rep, _ := replicaByID(h, old); rep.State != Draining && rep.State != Stopped {
|
|
t.Errorf("the old replica is %s after its replacement became ready", rep.State)
|
|
}
|
|
}
|
|
if got := len(h.r.ReadyReplicas()); got != 1 {
|
|
t.Errorf("%d ready replicas at the end; the model asks for 1", got)
|
|
}
|
|
}
|
|
|
|
func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) {
|
|
// One node, room for one replica: the replacement can only start once the
|
|
// old one has gone. It is the only thing in the way, so it goes now —
|
|
// waiting out a grace period first changed nothing but the wait, which
|
|
// in the field read as a restart that never happened.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if rep, ok := replicaByID(h, old); ok && rep.State == Ready {
|
|
t.Fatal("still serving with no replacement possible beside it")
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
ready := h.r.ReadyReplicas()
|
|
if len(ready) != 1 || ready[0].ID == old {
|
|
t.Errorf("expected one new replica in its place, got %+v", ready)
|
|
}
|
|
}
|
|
|
|
func TestARestartWhoseReplacementFitsNowhereIsGivenUp(t *testing.T) {
|
|
// The profile was edited to need more cards than the node has: stopping
|
|
// the old replica would leave the model with nothing at all.
|
|
m := testModel(1)
|
|
h := newHarness(t, m, gpuNode("1", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
|
|
m.Profiles[0].TP, m.Profiles[0].GPUsPerNode = 8, 8
|
|
h.setModels(t, m)
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(h.r.cfg.DrainGrace + time.Second)
|
|
}
|
|
if rep, ok := replicaByID(h, old); !ok || rep.State != Ready {
|
|
t.Fatalf("the only replica was stopped for a replacement that fits nowhere: %+v", rep)
|
|
}
|
|
if st := h.r.RestartState(old); !strings.HasPrefix(st, "refused: ") || !strings.Contains(st, "fits on no node") {
|
|
t.Errorf("restart state %q should say it was given up, and why", st)
|
|
}
|
|
// Asked again, it is tried again.
|
|
if err := h.r.Restart(old); err != nil || h.r.RestartState(old) != "queued" {
|
|
t.Errorf("a new restart did not clear the refusal: %q, %v", h.r.RestartState(old), err)
|
|
}
|
|
}
|
|
|
|
func TestARestartAskedInPlaceStopsFirst(t *testing.T) {
|
|
// Room beside it, but in place was asked for: an engine that must not
|
|
// run twice, or memory somebody knows is short.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
if err := h.r.RestartInPlace(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if rep, ok := replicaByID(h, old); ok && rep.State == Ready {
|
|
t.Fatalf("asked in place, it kept serving beside its replacement: %+v", rep)
|
|
}
|
|
}
|
|
|
|
func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for i := 0; i < 3; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
var ids []string
|
|
for _, rep := range h.r.ReadyReplicas() {
|
|
ids = append(ids, rep.ID)
|
|
}
|
|
if err := h.r.Restart(ids...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
states := []string{h.r.RestartState(ids[0]), h.r.RestartState(ids[1])}
|
|
if !slices.Contains(states, "replacing") || !slices.Contains(states, "queued") {
|
|
t.Errorf("expected one replacing and one queued, got %v", states)
|
|
}
|
|
if err := h.r.Restart("no-such"); err == nil {
|
|
t.Error("an unknown replica was accepted")
|
|
}
|
|
}
|
|
|
|
// failRepeatedly starts and fails the model's replica n times over.
|
|
func failRepeatedly(t *testing.T, h *harness, n int, why string) {
|
|
t.Helper()
|
|
for i := 0; i < n; i++ {
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
return // parked, or still in backoff
|
|
}
|
|
s := started[0]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, why)
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
|
}
|
|
}
|
|
|
|
func TestAModelThatAlwaysFailsIsParkedNotRetriedForever(t *testing.T) {
|
|
// kev-0.5b: an architecture the node's llama.cpp did not know, started
|
|
// 155 times in eight hours, the same error every time.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
|
|
|
|
h.disp.reset()
|
|
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
|
h.r.Reconcile(t.Context())
|
|
if n := len(h.disp.startedShards()); n != 0 {
|
|
t.Fatalf("a parked model was started again (%d shards)", n)
|
|
}
|
|
var mh ModelHealth
|
|
for _, x := range h.r.ModelHealth() {
|
|
if x.Model == "llama" {
|
|
mh = x
|
|
}
|
|
}
|
|
if !mh.Parked || !strings.Contains(mh.Reason, "ggmlc") || mh.Failures < DefaultConfig().ParkAfter {
|
|
t.Errorf("health %+v", mh)
|
|
}
|
|
if why, no := h.r.Unservable("llama"); !no || !strings.Contains(why, "ggmlc") {
|
|
t.Errorf("a request would not be told why: %q %v", why, no)
|
|
}
|
|
|
|
// Retried by the operator once fixed.
|
|
if !h.r.Retry("llama") {
|
|
t.Error("Retry did not report the model as parked")
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Error("a retried model was not placed")
|
|
}
|
|
}
|
|
|
|
func TestChangingAParkedModelsDefinitionTriesItAgain(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
failRepeatedly(t, h, DefaultConfig().ParkAfter, "boom")
|
|
|
|
fixed := testModel(1)
|
|
fixed.Source = "/models/llama-fixed"
|
|
cat, err := catalog.New([]catalog.Model{fixed})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(cat)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Error("a model whose definition changed stayed parked")
|
|
}
|
|
}
|
|
|
|
func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
h := newHarness(t, testModel(1), gpuNode("1", 1)) // tp4 needs 4 cards
|
|
h.r.log = slog.New(slog.NewTextHandler(&buf, nil))
|
|
for i := 0; i < 20; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(5 * time.Second)
|
|
}
|
|
if n := strings.Count(buf.String(), "no placement for model"); n != 1 {
|
|
t.Errorf("logged %d times over 20 passes", n)
|
|
}
|
|
h.advance(placementLogEvery)
|
|
h.r.Reconcile(t.Context())
|
|
if n := strings.Count(buf.String(), "no placement for model"); n != 2 {
|
|
t.Errorf("not repeated after %s: %d", placementLogEvery, n)
|
|
}
|
|
var mh ModelHealth
|
|
for _, x := range h.r.ModelHealth() {
|
|
mh = x
|
|
}
|
|
if !mh.Unplaceable || mh.Desired != 1 {
|
|
t.Errorf("health %+v", mh)
|
|
}
|
|
}
|
|
|
|
func liveCount(h *harness) int {
|
|
n := 0
|
|
for _, r := range h.r.Replicas() {
|
|
if r.State.Counts() {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func TestLoadAddsReplicasUpToMaxAndQuietTakesThemBack(t *testing.T) {
|
|
m := testModel(1)
|
|
m.Scaling.Max = 3
|
|
h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
busy := map[string]Load{"llama": {Replicas: 1, Running: 4, Waiting: 3}}
|
|
|
|
// A spike shorter than ScaleUpAfter adds nothing.
|
|
h.r.RecordLoad(busy)
|
|
h.advance(30 * time.Second)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
if n := liveCount(h); n != 1 {
|
|
t.Fatalf("%d replicas after 30s of load", n)
|
|
}
|
|
|
|
// Held past it, one more.
|
|
h.advance(31 * time.Second)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
if n := liveCount(h); n != 2 {
|
|
t.Fatalf("%d replicas after a minute of load; expected 2", n)
|
|
}
|
|
|
|
// Not another while that one is still starting.
|
|
h.advance(2 * time.Minute)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
if n := liveCount(h); n != 2 {
|
|
t.Fatalf("stacked a replica on one still starting: %d", n)
|
|
}
|
|
|
|
// Once it is up and load holds, the third — and never a fourth.
|
|
h.markReady()
|
|
for i := 0; i < 5; i++ {
|
|
h.r.RecordLoad(busy)
|
|
h.advance(61 * time.Second)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if n := liveCount(h); n != 3 {
|
|
t.Fatalf("%d replicas under sustained load; max is 3", n)
|
|
}
|
|
|
|
// Quiet: back down one step per ScaleDownAfter, to min and no further.
|
|
quiet := map[string]Load{"llama": {Replicas: 3}}
|
|
for i := 0; i < 6; i++ {
|
|
h.r.RecordLoad(quiet)
|
|
h.advance(DefaultConfig().ScaleDownAfter + time.Second)
|
|
h.r.RecordLoad(quiet)
|
|
for j := 0; j < 3; j++ {
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
}
|
|
if n := liveCount(h); n != 1 {
|
|
t.Errorf("%d replicas after a long quiet spell; min is 1", n)
|
|
}
|
|
}
|
|
|
|
func TestReplicasRestartSeveralAtOnceWhenAsked(t *testing.T) {
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8), gpuNode("4", 8))
|
|
for i := 0; i < 3; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
var ids []string
|
|
for _, rep := range h.r.ReadyReplicas() {
|
|
ids = append(ids, rep.ID)
|
|
}
|
|
if err := h.r.RestartPaced(RestartPace{MaxSurge: 2}, false, ids...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
replacing, queued := 0, 0
|
|
for _, id := range ids {
|
|
switch h.r.RestartState(id) {
|
|
case "replacing":
|
|
replacing++
|
|
case "queued":
|
|
queued++
|
|
}
|
|
}
|
|
if replacing != 2 || queued != 1 {
|
|
t.Errorf("max surge 2: %d replacing, %d queued", replacing, queued)
|
|
}
|
|
}
|
|
|
|
func TestMaxUnavailableStopsThatManyFirst(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for i := 0; i < 3; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
var ids []string
|
|
for _, rep := range h.r.ReadyReplicas() {
|
|
ids = append(ids, rep.ID)
|
|
}
|
|
if err := h.r.RestartPaced(RestartPace{MaxSurge: 1, MaxUnavailable: 1}, false, ids...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
stopped, beside := 0, 0
|
|
for _, id := range ids {
|
|
rep, ok := replicaByID(h, id)
|
|
switch {
|
|
case !ok || rep.State != Ready:
|
|
stopped++
|
|
case h.r.RestartState(id) == "replacing":
|
|
beside++
|
|
}
|
|
}
|
|
if stopped != 1 || beside != 1 {
|
|
t.Errorf("one stopped first and one replaced beside itself: %d stopped, %d beside", stopped, beside)
|
|
}
|
|
}
|
|
|
|
func TestAnAdoptedShardKeepsWhatItComputesOn(t *testing.T) {
|
|
// A control plane that took over, or restarted, learns where a replica
|
|
// computes from what its agent kept, not from a READY it never saw.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
a := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})
|
|
a.Compute = &Compute{Backend: "CUDA", Devices: []string{"CUDA0: H100"}, Offloaded: 81, Total: 81}
|
|
h.r.Adopt("1", []AdoptedShard{a})
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].Shards[0].Compute == nil || reps[0].Shards[0].Compute.Offloaded != 81 {
|
|
t.Fatalf("%+v", reps)
|
|
}
|
|
}
|
|
|
|
func TestStoppingTheServiceTakesEveryModelDownAndStartingBringsItBack(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
if len(h.r.ReadyReplicas()) != 1 {
|
|
t.Fatal("nothing ready to stop")
|
|
}
|
|
h.r.SetStopped(true)
|
|
h.r.Reconcile(t.Context())
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State == Ready {
|
|
t.Fatalf("stopped, a replica still serves: %+v", rep)
|
|
}
|
|
}
|
|
h.r.SetStopped(false)
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if len(h.r.ReadyReplicas()) != 1 {
|
|
t.Errorf("started again, %d ready", len(h.r.ReadyReplicas()))
|
|
}
|
|
}
|
|
|
|
func TestTheRoomAnInPlaceRestartFreesIsItsReplacements(t *testing.T) {
|
|
// Seen on the reference cluster: a model restarted in place on the only
|
|
// GPU, a judge waiting for that card, and the judge placed on it in the
|
|
// same pass, before the replacement — the model was gone until someone
|
|
// stopped the judge by hand.
|
|
for _, waitingFirst := range []bool{false, true} {
|
|
m := testModel(1)
|
|
waiting := testModel(1)
|
|
waiting.Name, waiting.Source = "judge", "/models/judge"
|
|
models := []catalog.Model{m, waiting}
|
|
if waitingFirst {
|
|
models = []catalog.Model{waiting, m}
|
|
}
|
|
h := newHarness(t, m, gpuNode("1", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
h.setModels(t, models...)
|
|
h.r.Reconcile(t.Context()) // the judge finds no room
|
|
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
var ofModel, ofWaiting int
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.ID == old || !rep.State.Counts() {
|
|
continue
|
|
}
|
|
switch rep.Model {
|
|
case m.Name:
|
|
ofModel++
|
|
case waiting.Name:
|
|
ofWaiting++
|
|
}
|
|
}
|
|
if ofModel != 1 || ofWaiting != 0 {
|
|
t.Errorf("waiting first=%v: the freed GPUs went to %d replacement(s) and %d of the waiting model",
|
|
waitingFirst, ofModel, ofWaiting)
|
|
}
|
|
}
|
|
}
|