A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2066 lines
64 KiB
Go
2066 lines
64 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
// fakeDispatcher records commands instead of sending them.
|
|
type fakeDispatcher struct {
|
|
mu sync.Mutex
|
|
started []ShardCommand
|
|
stopped []string
|
|
drained []string
|
|
failNext error
|
|
// traces are the span each start was sent under, which is what the
|
|
// real dispatcher puts on the command for the agent.
|
|
traces []trace.SpanContext
|
|
}
|
|
|
|
func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
if d.failNext != nil {
|
|
err := d.failNext
|
|
d.failNext = nil
|
|
return err
|
|
}
|
|
d.started = append(d.started, cmd)
|
|
d.traces = append(d.traces, trace.SpanContextFromContext(ctx))
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) StopShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.stopped = append(d.stopped, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) DrainShard(_ context.Context, _, shardID string) error {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.drained = append(d.drained, shardID)
|
|
return nil
|
|
}
|
|
|
|
func (d *fakeDispatcher) startedShards() []ShardCommand {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
return slices.Clone(d.started)
|
|
}
|
|
|
|
func (d *fakeDispatcher) reset() {
|
|
d.mu.Lock()
|
|
defer d.mu.Unlock()
|
|
d.started, d.stopped, d.drained = nil, nil, nil
|
|
}
|
|
|
|
// clock lets tests move time without sleeping.
|
|
type clock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
}
|
|
|
|
func newClock() *clock {
|
|
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
|
|
}
|
|
func (c *clock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.t
|
|
}
|
|
func (c *clock) advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
c.t = c.t.Add(d)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func gpuNode(id string, n int, fabrics ...string) inventory.Node {
|
|
nd := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready}
|
|
for i := range n {
|
|
nd.GPUs = append(nd.GPUs, discovery.GPU{
|
|
Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559, ComputeCapability: "9.0",
|
|
})
|
|
}
|
|
for _, f := range fabrics {
|
|
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
|
|
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"},
|
|
})
|
|
}
|
|
return nd
|
|
}
|
|
|
|
func testModel(min int, profiles ...catalog.Profile) catalog.Model {
|
|
if len(profiles) == 0 {
|
|
profiles = []catalog.Profile{{Name: "tp4", GPUClass: "h100-80g", TP: 4}}
|
|
}
|
|
return catalog.Model{
|
|
Name: "llama", Source: "/models/llama",
|
|
Scaling: catalog.Scaling{Min: min, Max: min + 2}, Profiles: profiles,
|
|
}
|
|
}
|
|
|
|
type harness struct {
|
|
r *Reconciler
|
|
inv *inventory.Store
|
|
disp *fakeDispatcher
|
|
clock *clock
|
|
live *catalog.Live
|
|
}
|
|
|
|
// setModels replaces the catalog the reconciler works from, as an edit does.
|
|
func (h *harness) setModels(t *testing.T, models ...catalog.Model) {
|
|
t.Helper()
|
|
cat, err := catalog.New(models)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.live.Store(cat)
|
|
}
|
|
|
|
func newHarness(t *testing.T, m catalog.Model, nodes ...inventory.Node) *harness {
|
|
t.Helper()
|
|
|
|
cat, err := catalog.New([]catalog.Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inv := inventory.NewStore()
|
|
c := newClock()
|
|
for _, n := range nodes {
|
|
inv.Upsert(n)
|
|
inv.Touch(n.ID)
|
|
}
|
|
|
|
disp := &fakeDispatcher{}
|
|
cfg := DefaultConfig()
|
|
cfg.UnreachableAfter = 15 * time.Second
|
|
cfg.LostAfter = 60 * time.Second
|
|
|
|
live := catalog.NewLive(cat)
|
|
r := New(cfg, live, inv, disp, slog.New(slog.DiscardHandler))
|
|
r.SetClock(c.now)
|
|
// Nodes were just registered, so their LastSeen is wall-clock now, which is
|
|
// far from the fake clock's epoch. Re-stamp them onto the fake timeline.
|
|
for _, n := range nodes {
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
// Past the startup settling window. These nodes are all here — the harness
|
|
// built the whole fleet before anything ran — which is the state settling
|
|
// waits for, not one it should be made to simulate in every test. The
|
|
// window's own behaviour is tested separately.
|
|
c.advance(cfg.SettleFor)
|
|
for _, n := range nodes {
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
return &harness{r: r, inv: inv, disp: disp, clock: c, live: live}
|
|
}
|
|
|
|
// touchAt rewrites a node's LastSeen onto the fake clock's timeline.
|
|
func touchAt(inv *inventory.Store, id string, at time.Time) {
|
|
n, ok := inv.Get(id)
|
|
if !ok {
|
|
return
|
|
}
|
|
n.LastSeen = at
|
|
n.State = inventory.Ready
|
|
inv.Upsert(n)
|
|
// Upsert stamps LastSeen with the wall clock, so set it again directly.
|
|
inv.SetLastSeen(id, at)
|
|
}
|
|
|
|
// advance moves the clock and keeps every node heartbeating.
|
|
//
|
|
// Without this, advancing time past UnreachableAfter makes every node look
|
|
// silent and placement stops — which is correct behaviour, but not what the
|
|
// backoff tests are trying to exercise.
|
|
func (h *harness) advance(d time.Duration) {
|
|
h.clock.advance(d)
|
|
for _, n := range h.inv.Nodes() {
|
|
touchAt(h.inv, n.ID, h.clock.now())
|
|
}
|
|
}
|
|
|
|
// markReady walks every started shard to Ready.
|
|
func (h *harness) markReady() {
|
|
for _, cmd := range h.disp.startedShards() {
|
|
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Ready, "")
|
|
}
|
|
}
|
|
|
|
func TestPlacesMinimumReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 2 {
|
|
t.Fatalf("expected 2 replicas, got %d: %+v", len(reps), reps)
|
|
}
|
|
for _, rep := range reps {
|
|
if rep.State != Ready {
|
|
t.Errorf("replica %s is %s", rep.ID, rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Errorf("replica %s has no endpoint", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOneReplicaPerPass(t *testing.T) {
|
|
// Placement depends on what is already allocated, so starting several from
|
|
// one inventory snapshot would hand them the same GPUs.
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Fatalf("one pass created %d replicas", got)
|
|
}
|
|
|
|
seen := map[int]string{}
|
|
for range 3 {
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
for _, g := range s.GPUIndices {
|
|
key := g
|
|
if prev, dup := seen[key]; dup && prev != s.NodeID {
|
|
continue // same index on a different node is fine
|
|
}
|
|
seen[key] = s.NodeID
|
|
}
|
|
}
|
|
}
|
|
if got := len(h.r.Replicas()); got != 3 {
|
|
t.Errorf("expected 3 replicas, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestSpreadsAcrossNodes(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
nodes := map[string]struct{}{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, n := range rep.NodeIDs() {
|
|
nodes[n] = struct{}{}
|
|
}
|
|
}
|
|
if len(nodes) != 2 {
|
|
t.Errorf("both replicas landed on %d node(s); they should spread", len(nodes))
|
|
}
|
|
}
|
|
|
|
func TestPacksWhenSpreadingIsImpossible(t *testing.T) {
|
|
// Spreading is a preference, not a rule: a second replica on one node still
|
|
// survives an engine crash, which is the common failure.
|
|
h := newHarness(t, testModel(2), gpuNode("only", 8))
|
|
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("expected 2 replicas on the single node, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestShardOrderLeaderFirst(t *testing.T) {
|
|
// Both engines put the rendezvous point on rank 0 — SGLang's dist-init
|
|
// address and vLLM's Ray head — so a worker started first has nothing to
|
|
// connect to and spends its startup budget retrying.
|
|
//
|
|
// This test previously asserted the opposite, on the theory that the head
|
|
// should wait for its workers. That is backwards: vLLM's server blocks
|
|
// until the Ray cluster has the GPUs it needs, so the leader can start
|
|
// immediately and waits on its own.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
started := h.disp.startedShards()
|
|
if len(started) != 2 {
|
|
t.Fatalf("expected 2 shards, got %d", len(started))
|
|
}
|
|
if started[0].Spec.Rank != 0 || started[1].Spec.Rank != 1 {
|
|
t.Errorf("start order was rank %d then %d; the leader hosts the rendezvous "+
|
|
"point and must start first",
|
|
started[0].Spec.Rank, started[1].Spec.Rank)
|
|
}
|
|
// Every rank must be given the same rendezvous point.
|
|
if started[0].Spec.LeaderAddr != started[1].Spec.LeaderAddr {
|
|
t.Errorf("ranks got different leader addresses: %q and %q",
|
|
started[0].Spec.LeaderAddr, started[1].Spec.LeaderAddr)
|
|
}
|
|
if started[0].Spec.LeaderPort != started[1].Spec.LeaderPort {
|
|
t.Error("ranks got different leader ports")
|
|
}
|
|
for _, cmd := range started {
|
|
if len(cmd.Spec.HCAs) == 0 {
|
|
t.Errorf("rank %d got no HCAs; NCCL would choose its own", cmd.Spec.Rank)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestFailedShardFailsWholeReplica(t *testing.T) {
|
|
// One dead shard makes the others useless: they wait on collectives that
|
|
// will never complete while holding GPUs (plan §5.4).
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
|
|
// One shard ready, the other dead.
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Ready, "")
|
|
h.r.OnShardStatus(started[1].ReplicaID, started[1].ShardID, supervisor.Failed, "CUDA out of memory")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Both shards must have been stopped, not just the failed one.
|
|
h.disp.mu.Lock()
|
|
stopped := slices.Clone(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if len(stopped) != 2 {
|
|
t.Errorf("stopped %d shards, expected both: %v", len(stopped), stopped)
|
|
}
|
|
// And the GPUs must be free again.
|
|
for _, id := range []string{"1", "2"} {
|
|
if got := len(h.inv.FreeGPUs(id)); got != 8 {
|
|
t.Errorf("node %s has %d free GPUs; the replica should have been released", id, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBackoffAfterFailure(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
h.r.OnShardStatus(started[0].ReplicaID, started[0].ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context()) // reaps it
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a replica was rebuilt immediately after failure; backoff is not applied")
|
|
}
|
|
|
|
// Past the backoff, it tries again.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("the replica was never rebuilt after the backoff expired")
|
|
}
|
|
}
|
|
|
|
func TestSuccessClearsBackoff(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
// Fail once.
|
|
h.r.Reconcile(t.Context())
|
|
s := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, "boom")
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Then succeed.
|
|
h.advance(DefaultConfig().MaxBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Fatal("no replica was started after the backoff expired")
|
|
}
|
|
s2 := h.disp.startedShards()[0]
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Ready, "")
|
|
|
|
// A later failure should start from a short backoff, not a long streak.
|
|
h.r.OnShardStatus(s2.ReplicaID, s2.ShardID, supervisor.Failed, "again")
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(2 * DefaultConfig().BaseBackoff)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Error("backoff was not reset by the successful run in between")
|
|
}
|
|
}
|
|
|
|
// TestLostNodeMovesReplicas is the phase 3 exit criterion: a node going away
|
|
// must get its replicas rebuilt elsewhere.
|
|
func TestLostNodeMovesReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("setup failed: %+v", reps)
|
|
}
|
|
original := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if original == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
// The surviving node keeps reporting; the other goes silent.
|
|
h.clock.advance(20 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Unreachable {
|
|
t.Errorf("silent node is %s, expected unreachable", n.State)
|
|
}
|
|
// Unreachable is not lost: a blip must not tear down a working replica.
|
|
if reps := h.r.Replicas(); len(reps) != 1 || reps[0].State != Ready {
|
|
t.Fatalf("an unreachable node already destroyed the replica: %+v", reps)
|
|
}
|
|
|
|
h.clock.advance(90 * time.Second)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context()) // declares it lost and reaps the replica
|
|
|
|
if n, _ := h.inv.Get(original); n.State != inventory.Lost {
|
|
t.Fatalf("node is %s, expected lost", n.State)
|
|
}
|
|
|
|
// Past the backoff, the replica is rebuilt on the survivor. Only the
|
|
// survivor heartbeats; the lost node stays gone.
|
|
h.clock.advance(DefaultConfig().MaxBackoff)
|
|
touchAt(h.inv, survivor, h.clock.now())
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps = h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected the replica to be rebuilt, got %d", len(reps))
|
|
}
|
|
if got := reps[0].NodeIDs()[0]; got != survivor {
|
|
t.Errorf("replica was rebuilt on %q, expected the surviving node %q", got, survivor)
|
|
}
|
|
if reps[0].State != Ready {
|
|
t.Errorf("rebuilt replica is %s", reps[0].State)
|
|
}
|
|
}
|
|
|
|
func TestHeartbeatRevivesNode(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.clock.advance(20 * time.Second) // deliberately without a heartbeat
|
|
h.r.Reconcile(t.Context())
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Unreachable {
|
|
t.Fatalf("node is %s", n.State)
|
|
}
|
|
|
|
h.inv.Touch("1")
|
|
if n, _ := h.inv.Get("1"); n.State != inventory.Ready {
|
|
t.Errorf("a heartbeat did not revive the node: %s", n.State)
|
|
}
|
|
}
|
|
|
|
func TestScaleDownDrainsFirst(t *testing.T) {
|
|
// Draining before stopping lets in-flight requests finish instead of being
|
|
// cut off mid-stream.
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
for range 2 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if got := len(h.r.Replicas()); got != 2 {
|
|
t.Fatalf("setup: %d replicas", got)
|
|
}
|
|
|
|
// Shrink the catalog to one replica.
|
|
cat, err := catalog.New([]catalog.Model{testModel(1)})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(cat)
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the excess replica was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("a replica was stopped before it had drained")
|
|
}
|
|
|
|
// The next pass stops what has drained.
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped")
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica after scale-down, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestUnplaceableModelExplains(t *testing.T) {
|
|
// A model that cannot be placed must leave a usable explanation rather than
|
|
// a silent absence.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "tp8", GPUClass: "h100-80g", TP: 8,
|
|
}), gpuNode("1", 4))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Fatalf("something was placed: %d", got)
|
|
}
|
|
|
|
res, ok := h.r.Explain("llama")
|
|
if !ok {
|
|
t.Fatal("no explanation was recorded")
|
|
}
|
|
exp := res.Explain()
|
|
if !strings.Contains(exp, "needs 8") {
|
|
t.Errorf("explanation does not say what was missing:\n%s", exp)
|
|
}
|
|
}
|
|
|
|
func TestDispatcherFailureFailsReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.disp.failNext = fmt.Errorf("agent unreachable")
|
|
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed dispatch did not fail the replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass must release its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.inv.FreeGPUs("1")); got != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed replica still holds some", got)
|
|
}
|
|
}
|
|
|
|
func TestReadyReplicasFeedTheRouter(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.ReadyReplicas()); got != 0 {
|
|
t.Errorf("a starting replica was offered to the router")
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
ready := h.r.ReadyReplicas()
|
|
if len(ready) != 2 {
|
|
t.Fatalf("expected 2 ready replicas, got %d", len(ready))
|
|
}
|
|
for _, rep := range ready {
|
|
if !strings.HasPrefix(rep.Endpoint, "http://") {
|
|
t.Errorf("endpoint %q is not a URL", rep.Endpoint)
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Errorf("replica %s has no weight; the router needs one", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPortsDoNotCollide(t *testing.T) {
|
|
h := newHarness(t, testModel(3), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
|
|
ports := map[int]string{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
if prev, dup := ports[s.HTTPPort]; dup && prev != rep.ID {
|
|
t.Errorf("replicas %s and %s both use port %d", prev, rep.ID, s.HTTPPort)
|
|
}
|
|
ports[s.HTTPPort] = rep.ID
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReplicaStateStrings(t *testing.T) {
|
|
for st, want := range map[ReplicaState]string{
|
|
Placing: "placing", Starting: "starting", Ready: "ready",
|
|
Draining: "draining", Stopped: "stopped", Failed: "failed",
|
|
} {
|
|
if got := st.String(); got != want {
|
|
t.Errorf("%d -> %q", int(st), got)
|
|
}
|
|
}
|
|
if !Ready.Counts() || Draining.Counts() || Failed.Counts() {
|
|
// Draining must not count, or a replacement is never started for it.
|
|
t.Error("Counts is wrong")
|
|
}
|
|
}
|
|
|
|
// --- adoption after a control plane restart (plan §6.5) ---
|
|
|
|
func adopted(replicaID, shardID, nodeID string, rank int, gpus []int) AdoptedShard {
|
|
a := AdoptedShard{
|
|
ReplicaID: replicaID, ShardID: shardID, NodeID: nodeID, Rank: rank,
|
|
GPUIndices: gpus, HTTPPort: 20000, State: supervisor.Ready,
|
|
Model: "llama", Profile: "tp4", Pool: "none/h100-80g",
|
|
}
|
|
if rank == 0 {
|
|
a.Endpoint = "http://10.0.0." + nodeID + ":20000"
|
|
}
|
|
return a
|
|
}
|
|
|
|
func TestAdoptRebuildsReplica(t *testing.T) {
|
|
// The control plane restarted. Nothing is known, but a node reports a live
|
|
// replica. Without this the reconciler would place a duplicate while the
|
|
// original keeps holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("a valid shard was reported unclaimed: %+v", unclaimed)
|
|
}
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
|
|
}
|
|
rep := reps[0]
|
|
if rep.ID != "llama-99" || rep.Model != "llama" || rep.Profile != "tp4" {
|
|
t.Errorf("identity wrong: %+v", rep)
|
|
}
|
|
if rep.State != Ready {
|
|
t.Errorf("state=%s; every shard reported Ready", rep.State)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Error("no endpoint; the router would have nowhere to send")
|
|
}
|
|
if rep.Weight <= 0 {
|
|
t.Error("weight not taken from the catalog")
|
|
}
|
|
|
|
// The GPUs must be reclaimed, or the next placement hands them out twice.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs, expected 4 after adoption", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptPreventsDuplicatePlacement(t *testing.T) {
|
|
// The whole point: after adopting, a reconciliation pass must see the
|
|
// desired count already met.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a duplicate replica was placed despite adoption (%d shards started)", got)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("expected 1 replica, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptMultiNodeReplica(t *testing.T) {
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
shards := []AdoptedShard{
|
|
adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
adopted("llama-7", "llama-7-r1", "2", 1, []int{0, 1, 2, 3, 4, 5, 6, 7}),
|
|
}
|
|
for i := range shards {
|
|
shards[i].Profile = "2node"
|
|
shards[i].Fabric = "ib-a"
|
|
shards[i].Pool = "ib-a/h100-80g"
|
|
}
|
|
|
|
if unclaimed := h.r.Adopt("1", shards); len(unclaimed) != 0 {
|
|
t.Fatalf("unclaimed: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
|
t.Fatalf("the two shards were not grouped into one replica: %+v", reps)
|
|
}
|
|
if reps[0].Fabric != "ib-a" {
|
|
t.Errorf("fabric lost: %q", reps[0].Fabric)
|
|
}
|
|
// Both nodes' GPUs must be reclaimed.
|
|
for _, id := range []string{"1", "2"} {
|
|
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
|
t.Errorf("node %s has %d free GPUs after adoption", id, free)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAdoptPartialReplicaFails(t *testing.T) {
|
|
// A multi-node replica missing a shard is useless: the survivors wait on
|
|
// collectives that will never complete while holding GPUs.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile = "2node"
|
|
s.State = supervisor.Failed
|
|
h.r.Adopt("1", []AdoptedShard{s})
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || reps[0].State != Failed {
|
|
t.Fatalf("a failed adopted shard did not fail its replica: %+v", reps)
|
|
}
|
|
|
|
// And the next pass releases its GPUs.
|
|
h.r.Reconcile(t.Context())
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("node has %d free GPUs; the failed adopted replica still holds some", free)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownModelIsUnclaimed(t *testing.T) {
|
|
// The model left the catalog while the engine kept running. Nothing will
|
|
// ever route to it, and it is holding GPUs.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("ghost-1", "ghost-1-r0", "1", 0, []int{0})
|
|
s.Model = "a-model-that-was-removed"
|
|
|
|
unclaimed := h.r.Adopt("1", []AdoptedShard{s})
|
|
if len(unclaimed) != 1 || unclaimed[0].ShardID != "ghost-1-r0" {
|
|
t.Fatalf("an orphaned shard was not reported unclaimed: %+v", unclaimed)
|
|
}
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a replica was created for an unknown model: %d", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptUnknownProfileIsUnclaimed(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
s := adopted("llama-9", "llama-9-r0", "1", 0, []int{0})
|
|
s.Profile = "a-profile-that-was-removed"
|
|
|
|
if unclaimed := h.r.Adopt("1", []AdoptedShard{s}); len(unclaimed) != 1 {
|
|
t.Errorf("a shard with an unknown profile was claimed: %+v", unclaimed)
|
|
}
|
|
}
|
|
|
|
func TestAdoptIsIdempotent(t *testing.T) {
|
|
// An agent reconnecting re-reports the same shards; that must not double
|
|
// the allocation.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
s := []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})}
|
|
|
|
h.r.Adopt("1", s)
|
|
h.r.Adopt("1", s)
|
|
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("re-reporting created %d replicas", got)
|
|
}
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs; the allocation was applied twice", free)
|
|
}
|
|
}
|
|
|
|
// --- draining a node (ctl drain) ---
|
|
|
|
func TestDrainingNodeMovesWorkOff(t *testing.T) {
|
|
// The operator's view: a node is about to be worked on, its replicas must
|
|
// end up elsewhere, and nothing may be cut off mid-request on the way.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
victim := reps[0].NodeIDs()[0]
|
|
survivor := "2"
|
|
if victim == "2" {
|
|
survivor = "1"
|
|
}
|
|
|
|
h.inv.SetState(victim, inventory.Draining)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// It is drained, not stopped: the engine is healthy and still serving.
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained == 0 {
|
|
t.Error("the replica on the draining node was not drained")
|
|
}
|
|
if stopped != 0 {
|
|
t.Error("the replica was stopped before a replacement existed; in-flight requests would be cut")
|
|
}
|
|
|
|
// A draining replica does not count, so a replacement is placed elsewhere.
|
|
if got := len(h.disp.startedShards()); got == 0 {
|
|
t.Fatal("no replacement was placed")
|
|
}
|
|
for _, cmd := range h.disp.startedShards() {
|
|
if cmd.NodeID == victim {
|
|
t.Errorf("the replacement was placed back on the draining node %s", victim)
|
|
}
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
// Now that the replacement serves, the original can go.
|
|
h.disp.mu.Lock()
|
|
stopped = len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped once its replacement was ready")
|
|
}
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.OnNode(victim) {
|
|
t.Errorf("replica %s is still on the drained node", rep.ID)
|
|
}
|
|
if !slices.Contains(rep.NodeIDs(), survivor) {
|
|
t.Errorf("replica %s did not move to the surviving node", rep.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDrainGraceStopsWithoutReplacement(t *testing.T) {
|
|
// Draining the only node that can host a model would otherwise never
|
|
// finish, leaving the node undrainable.
|
|
h := newHarness(t, testModel(1), gpuNode("only", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.inv.SetState("only", inventory.Draining)
|
|
h.r.Reconcile(t.Context()) // drains it; nowhere to place a replacement
|
|
|
|
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Draining {
|
|
t.Fatalf("expected a draining replica, got %+v", got)
|
|
}
|
|
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
h.disp.mu.Lock()
|
|
early := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if early != 0 {
|
|
t.Error("the replica was stopped before the grace period expired")
|
|
}
|
|
|
|
h.advance(DefaultConfig().DrainGrace + time.Second)
|
|
h.inv.SetState("only", inventory.Draining) // advance() resets node state
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
stopped := len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if stopped == 0 {
|
|
t.Error("the drained replica was never stopped; the node cannot be drained")
|
|
}
|
|
}
|
|
|
|
func TestCordonedNodeKeepsItsWork(t *testing.T) {
|
|
// Cordon is not drain: it stops new placement and leaves what is running.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.inv.SetState("1", inventory.Cordoned)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
|
|
h.disp.mu.Lock()
|
|
drained, stopped := len(h.disp.drained), len(h.disp.stopped)
|
|
h.disp.mu.Unlock()
|
|
if drained != 0 || stopped != 0 {
|
|
t.Errorf("cordoning disturbed a running replica (drained=%d stopped=%d)", drained, stopped)
|
|
}
|
|
if got := h.r.Replicas(); len(got) != 1 || got[0].State != Ready {
|
|
t.Errorf("the replica did not survive a cordon: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAdoptMultiNodeReplicaOneAgentAtATime(t *testing.T) {
|
|
// How it really arrives. Every other multi-node adoption test hands both
|
|
// shards to a single Adopt call, which no caller ever does: each node has
|
|
// its own agent and each agent registers separately, so a two-node replica
|
|
// is reported in two halves a second or so apart.
|
|
//
|
|
// Observed on the cluster after a control plane restart: the first half was
|
|
// adopted and immediately failed for "replica has 1 of 2 shards", and when
|
|
// the second node registered its half was stopped as unclaimed. big-model
|
|
// never came back — it rebuilt from scratch on every restart, with
|
|
// exponential backoff, forever.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
half := func(shardID, nodeID string, rank int) []AdoptedShard {
|
|
s := adopted("llama-7", shardID, nodeID, rank, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g"
|
|
return []AdoptedShard{s}
|
|
}
|
|
|
|
// Rank 1 first, so the endpoint arrives in the second group: whichever node
|
|
// registers first is not necessarily the leader.
|
|
if unclaimed := h.r.Adopt("2", half("llama-7-r1", "2", 1)); len(unclaimed) != 0 {
|
|
t.Fatalf("the first half was not claimed: %+v", unclaimed)
|
|
}
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected the half to be adopted, got %d replicas", len(reps))
|
|
}
|
|
if reps[0].State == Failed {
|
|
t.Fatalf("a replica still assembling was failed: %q", reps[0].Err)
|
|
}
|
|
|
|
if unclaimed := h.r.Adopt("1", half("llama-7-r0", "1", 0)); len(unclaimed) != 0 {
|
|
t.Fatalf("the second half was stopped as unclaimed: %+v", unclaimed)
|
|
}
|
|
|
|
reps = h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("the halves became %d replicas", len(reps))
|
|
}
|
|
rep := reps[0]
|
|
if len(rep.Shards) != 2 {
|
|
t.Fatalf("the halves were not merged: %d shards", len(rep.Shards))
|
|
}
|
|
if rep.State != Ready {
|
|
t.Errorf("state=%s err=%q; both shards reported Ready", rep.State, rep.Err)
|
|
}
|
|
if rep.Endpoint == "" {
|
|
t.Error("no endpoint; rank 0 arrived in the second group and was dropped")
|
|
}
|
|
for _, id := range []string{"1", "2"} {
|
|
if free := len(h.inv.FreeGPUs(id)); free != 0 {
|
|
t.Errorf("node %s has %d free GPUs; its half's allocation was lost", id, free)
|
|
}
|
|
}
|
|
|
|
// And the reconciler must not then place a duplicate alongside it.
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.disp.startedShards()); got != 0 {
|
|
t.Errorf("a duplicate was placed despite the replica being whole (%d shards started)", got)
|
|
}
|
|
}
|
|
|
|
func TestAHalfThatNeverArrivesStillFails(t *testing.T) {
|
|
// The grace period must be a deadline, not an amnesty: a replica whose
|
|
// other node is simply gone has to fail so it can be replaced.
|
|
h := newHarness(t, testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
StartupTimeout: time.Minute,
|
|
}), gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
|
|
s := adopted("llama-7", "llama-7-r0", "1", 0, []int{0, 1, 2, 3, 4, 5, 6, 7})
|
|
s.Profile, s.Fabric, s.Pool = "2node", "ib-a", "ib-a/h100-80g"
|
|
h.r.Adopt("1", []AdoptedShard{s})
|
|
|
|
if reps := h.r.Replicas(); reps[0].State != Starting {
|
|
t.Fatalf("state=%s; the half should be waiting for its sibling", reps[0].State)
|
|
}
|
|
|
|
h.advance(2 * time.Minute)
|
|
h.r.Reconcile(t.Context())
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) == 1 && reps[0].State != Failed {
|
|
t.Errorf("state=%s; a half whose sibling never registered must fail", reps[0].State)
|
|
}
|
|
if free := len(h.inv.FreeGPUs("1")); free != 8 {
|
|
t.Errorf("node has %d free GPUs; the abandoned half still holds them", free)
|
|
}
|
|
}
|
|
|
|
// --- the startup settling window ---
|
|
|
|
func settlingHarness(t *testing.T, m catalog.Model) (*Reconciler, *inventory.Store, *fakeDispatcher, *clock) {
|
|
t.Helper()
|
|
cat, err := catalog.New([]catalog.Model{m})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
inv := inventory.NewStore()
|
|
disp := &fakeDispatcher{}
|
|
c := newClock()
|
|
r := New(DefaultConfig(), catalog.NewLive(cat), inv, disp, slog.New(slog.DiscardHandler))
|
|
r.SetClock(c.now)
|
|
return r, inv, disp, c
|
|
}
|
|
|
|
func addNode(inv *inventory.Store, c *clock, n inventory.Node) {
|
|
inv.Upsert(n)
|
|
touchAt(inv, n.ID, c.now())
|
|
}
|
|
|
|
func TestPlacementWaitsForTheFleetToArrive(t *testing.T) {
|
|
// A control plane that has just started knows nothing, and the first tick
|
|
// runs before any agent has registered. Placing then puts everything on
|
|
// whichever fraction reported first: after one restart both replicas of a
|
|
// model landed on the single node that had registered, when four were
|
|
// coming.
|
|
r, inv, disp, c := settlingHarness(t, testModel(2))
|
|
|
|
// Nothing registered yet.
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards on an empty fleet", got)
|
|
}
|
|
|
|
// One node of four.
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards while the fleet was still arriving", got)
|
|
}
|
|
|
|
// The rest arrive.
|
|
for _, id := range []string{"2", "3", "4"} {
|
|
addNode(inv, c, gpuNode(id, 8))
|
|
}
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards the instant the last node appeared", got)
|
|
}
|
|
|
|
// Steady for long enough that a reconnect backoff would have finished: now
|
|
// it may act, and on the whole fleet.
|
|
c.advance(DefaultConfig().SettleFor / 2)
|
|
for _, id := range []string{"1", "2", "3", "4"} {
|
|
touchAt(inv, id, c.now())
|
|
}
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got == 0 {
|
|
t.Fatal("placement never resumed after the fleet settled")
|
|
}
|
|
}
|
|
|
|
func TestPlacementDoesNotWaitForeverForNodesThatNeverCome(t *testing.T) {
|
|
// Waiting only for the count to stop changing would hang forever on a
|
|
// cluster whose nodes never all come back.
|
|
r, inv, disp, c := settlingHarness(t, testModel(1))
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
|
|
c.advance(DefaultConfig().SettleFor)
|
|
touchAt(inv, "1", c.now())
|
|
r.Reconcile(t.Context())
|
|
|
|
if got := len(disp.startedShards()); got == 0 {
|
|
t.Error("placement never resumed once the window expired")
|
|
}
|
|
}
|
|
|
|
func TestSettlingIsOnlyEverPaidOnce(t *testing.T) {
|
|
// A node joining later in the day is an ordinary event, not a reason to
|
|
// stop placing while the cluster is running.
|
|
r, inv, disp, c := settlingHarness(t, testModel(1))
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
c.advance(DefaultConfig().SettleFor)
|
|
touchAt(inv, "1", c.now())
|
|
r.Reconcile(t.Context())
|
|
if len(disp.startedShards()) == 0 {
|
|
t.Fatal("nothing placed after the window")
|
|
}
|
|
|
|
disp.reset()
|
|
addNode(inv, c, gpuNode("2", 8))
|
|
if !r.fleetHasSettled() {
|
|
t.Error("a node joining a running cluster restarted the settling window")
|
|
}
|
|
}
|
|
|
|
func TestAdoptionIsNotBlockedByTheSettlingWindow(t *testing.T) {
|
|
// Only placement waits. Adoption is how a restart recovers what is already
|
|
// running, and delaying it would leave live engines unaccounted for exactly
|
|
// when the reconciler is deciding what to build.
|
|
r, inv, _, c := settlingHarness(t, testModel(1))
|
|
addNode(inv, c, gpuNode("1", 8))
|
|
|
|
unclaimed := r.Adopt("1", []AdoptedShard{
|
|
adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3}),
|
|
})
|
|
if len(unclaimed) != 0 {
|
|
t.Fatalf("adoption was refused during settling: %+v", unclaimed)
|
|
}
|
|
if got := len(r.Replicas()); got != 1 {
|
|
t.Fatalf("expected the replica to be adopted, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestSettlingOutlastsAnAgentReconnectBackoff(t *testing.T) {
|
|
// The failure this was measured against. After a restart node05 registered
|
|
// in one second and the other four arrived seventeen seconds later, because
|
|
// a disconnected agent reconnects on a backoff reaching thirty seconds. A
|
|
// short steady period declared a fleet of one settled and resumed placement
|
|
// while four nodes were still coming — the exact situation the window
|
|
// exists to avoid, reached by being impatient about it.
|
|
r, inv, disp, c := settlingHarness(t, testModel(2))
|
|
|
|
addNode(inv, c, gpuNode("first", 8))
|
|
// Fifteen seconds of one node, which a short threshold would call settled.
|
|
for range 3 {
|
|
c.advance(5 * time.Second)
|
|
touchAt(inv, "first", c.now())
|
|
r.Reconcile(t.Context())
|
|
}
|
|
if got := len(disp.startedShards()); got != 0 {
|
|
t.Fatalf("placed %d shards on one node while others were backing off", got)
|
|
}
|
|
|
|
// The stragglers arrive, as they did at seventeen seconds. The loop runs on
|
|
// a ticker, so the new count is noticed on the next pass, not the instant
|
|
// the nodes appear.
|
|
for _, id := range []string{"2", "3", "4"} {
|
|
addNode(inv, c, gpuNode(id, 8))
|
|
}
|
|
r.Reconcile(t.Context())
|
|
|
|
c.advance(DefaultConfig().SettleFor / 2)
|
|
for _, id := range []string{"first", "2", "3", "4"} {
|
|
touchAt(inv, id, c.now())
|
|
}
|
|
r.Reconcile(t.Context())
|
|
if got := len(disp.startedShards()); got == 0 {
|
|
t.Fatal("placement never resumed")
|
|
}
|
|
}
|
|
|
|
// --- idle tracking ---
|
|
|
|
func TestUsageIsRecordedAgainstReplicas(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected a replica, got %d", len(reps))
|
|
}
|
|
id := reps[0].ID
|
|
|
|
if _, known := h.r.IdleFor(id); known {
|
|
t.Error("a replica nothing has used reported a known idleness")
|
|
}
|
|
|
|
used := h.clock.now()
|
|
h.r.RecordUsage(map[string]time.Time{id: used})
|
|
h.advance(5 * time.Minute)
|
|
|
|
idle, known := h.r.IdleFor(id)
|
|
if !known {
|
|
t.Fatal("usage was recorded but idleness is still unknown")
|
|
}
|
|
if idle != 5*time.Minute {
|
|
t.Errorf("idle = %s, want 5m", idle)
|
|
}
|
|
}
|
|
|
|
func TestUsageNeverMovesBackwards(t *testing.T) {
|
|
// Two routers reporting, or a snapshot arriving late, must not make a busy
|
|
// replica look abandoned.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
id := h.r.Replicas()[0].ID
|
|
|
|
recent := h.clock.now()
|
|
h.r.RecordUsage(map[string]time.Time{id: recent})
|
|
h.r.RecordUsage(map[string]time.Time{id: recent.Add(-time.Hour)})
|
|
|
|
h.advance(time.Minute)
|
|
if idle, _ := h.r.IdleFor(id); idle != time.Minute {
|
|
t.Errorf("idle = %s; a stale report moved last-use backwards", idle)
|
|
}
|
|
}
|
|
|
|
func TestUsageForReplicasWeDoNotHaveIsIgnored(t *testing.T) {
|
|
// The router's list and the reconciler's drift constantly on a live fleet:
|
|
// a replica stops between the snapshot and the read. That is ordinary, not
|
|
// an error, and must not panic or create phantom state.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
before := len(h.r.Replicas())
|
|
|
|
h.r.RecordUsage(map[string]time.Time{"a-replica-that-left": h.clock.now()})
|
|
if got := len(h.r.Replicas()); got != before {
|
|
t.Errorf("replica count changed from %d to %d", before, got)
|
|
}
|
|
}
|
|
|
|
func TestAnEmptyUsageReportLeavesEverythingAlone(t *testing.T) {
|
|
// A router that has just started knows nothing, and treating that as "never
|
|
// used" would make every replica look abandoned at once — exactly when a
|
|
// fleet is least able to afford acting on it.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
id := h.r.Replicas()[0].ID
|
|
|
|
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
|
h.advance(time.Minute)
|
|
h.r.RecordUsage(nil)
|
|
|
|
if idle, known := h.r.IdleFor(id); !known || idle != time.Minute {
|
|
t.Errorf("idle=%s known=%v; an empty report erased what was known", idle, known)
|
|
}
|
|
}
|
|
|
|
func TestAModelAddedWhileRunningIsPlaced(t *testing.T) {
|
|
// The reason the catalog stopped being fixed at startup. Adding a model was
|
|
// an operator procedure with downtime: edit a file on the control plane,
|
|
// restart it, wait out the settling window, watch adoption churn. None of
|
|
// that was necessary — the loop already compares desired against actual
|
|
// every pass, so a new catalog is simply a different desired.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Fatalf("expected the original model's replica, got %d", got)
|
|
}
|
|
|
|
second := testModel(1)
|
|
second.Name = "llama-two"
|
|
next, err := catalog.New([]catalog.Model{testModel(1), second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(next)
|
|
|
|
h.r.Reconcile(t.Context())
|
|
byModel := map[string]int{}
|
|
for _, rep := range h.r.Replicas() {
|
|
byModel[rep.Model]++
|
|
}
|
|
if byModel["llama-two"] == 0 {
|
|
t.Errorf("a model added while running was never placed: %v", byModel)
|
|
}
|
|
if byModel["llama"] == 0 {
|
|
t.Errorf("the model that was already running was disturbed: %v", byModel)
|
|
}
|
|
}
|
|
|
|
func TestAModelRemovedWhileRunningIsStopped(t *testing.T) {
|
|
// The other direction, and the one with consequences: a replica whose model
|
|
// has left the catalog holds GPUs for something nothing can ask for.
|
|
first := testModel(1)
|
|
second := testModel(1)
|
|
second.Name = "llama-two"
|
|
both, err := catalog.New([]catalog.Model{first, second})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newHarness(t, first, gpuNode("1", 8), gpuNode("2", 8))
|
|
h.r.cat.Store(both)
|
|
h.r.Reconcile(t.Context())
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.r.Replicas()) < 2 {
|
|
t.Fatalf("expected both models placed, got %d", len(h.r.Replicas()))
|
|
}
|
|
|
|
only, err := catalog.New([]catalog.Model{first})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(only)
|
|
|
|
// Several passes: stopping is deliberate rather than immediate, and a
|
|
// replica is not torn down in the same breath it is noticed.
|
|
for range 4 {
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.Model == "llama-two" && rep.State.Counts() {
|
|
t.Errorf("a replica of a removed model is still counted: %+v", rep)
|
|
}
|
|
}
|
|
}
|
|
|
|
// --- on-demand loading and idle eviction ---
|
|
|
|
func onDemandModel(idle time.Duration) catalog.Model {
|
|
m := testModel(1)
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: idle}
|
|
return m
|
|
}
|
|
|
|
func TestAnOnDemandModelIsNotRunningUntilItIsAskedFor(t *testing.T) {
|
|
// The point of the whole feature: a cluster with more models than GPUs can
|
|
// hold every model in its catalog and only the wanted ones in its cards.
|
|
h := newHarness(t, onDemandModel(time.Hour), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Fatalf("an on-demand model was placed before anyone asked: %d replicas", got)
|
|
}
|
|
|
|
if !h.r.Want("llama") {
|
|
t.Fatal("Want reported the model is not on-demand")
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 1 {
|
|
t.Errorf("asking for the model produced %d replicas, want 1", got)
|
|
}
|
|
}
|
|
|
|
func TestWantSaysWhenAModelDoesNotLoadOnDemand(t *testing.T) {
|
|
// The caller needs to tell "being loaded" from "there is genuinely nothing
|
|
// serving this", because those are different answers to give a client.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8)) // min 1, not on-demand
|
|
if h.r.Want("llama") {
|
|
t.Error("an ordinary model was reported as loading on demand")
|
|
}
|
|
if h.r.Want("a-model-that-does-not-exist") {
|
|
t.Error("an unknown model was reported as loading on demand")
|
|
}
|
|
}
|
|
|
|
func TestAnIdleOnDemandModelIsStopped(t *testing.T) {
|
|
// The other half. Without it, on-demand is just a slower way to fill the
|
|
// cluster permanently.
|
|
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
if len(h.r.Replicas()) != 1 {
|
|
t.Fatalf("the model did not start")
|
|
}
|
|
|
|
h.advance(11 * time.Minute)
|
|
h.r.Reconcile(t.Context())
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
t.Errorf("a replica idle past its timeout is still counted: %+v", rep)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUseKeepsAnOnDemandModelAlive(t *testing.T) {
|
|
// Without this the timer measures time since the request that loaded the
|
|
// model rather than the last one, so a busy model would be evicted exactly
|
|
// IdleTimeout after it started however hard it was working.
|
|
h := newHarness(t, onDemandModel(10*time.Minute), gpuNode("1", 8))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("the model did not start")
|
|
}
|
|
id := reps[0].ID
|
|
|
|
// Busy for half an hour, in six-minute steps.
|
|
for range 5 {
|
|
h.advance(6 * time.Minute)
|
|
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
|
|
var live int
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
live++
|
|
}
|
|
}
|
|
if live != 1 {
|
|
t.Errorf("a model in continuous use was evicted: %d live replicas", live)
|
|
}
|
|
}
|
|
|
|
func TestAModelWithMaxZeroStaysOff(t *testing.T) {
|
|
// min 0 max 0 is configured and deliberately not running, which is a
|
|
// different and equally useful thing from "load it when wanted".
|
|
m := testModel(1)
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 0}
|
|
h := newHarness(t, m, gpuNode("1", 8))
|
|
|
|
if h.r.Want("llama") {
|
|
t.Error("a model with max 0 was treated as on-demand")
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if got := len(h.r.Replicas()); got != 0 {
|
|
t.Errorf("a model switched off was placed: %d replicas", got)
|
|
}
|
|
}
|
|
|
|
func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) {
|
|
// Every model that exists today. A model with min 1 stays up whether anyone
|
|
// uses it or not, because that is what min 1 means.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
// Ready, or the startup timeout is what removes it twenty-four hours later
|
|
// and the test proves nothing about idleness.
|
|
h.markReady()
|
|
if len(h.r.Replicas()) != 1 {
|
|
t.Fatal("the model did not start")
|
|
}
|
|
|
|
h.advance(24 * time.Hour)
|
|
h.r.Reconcile(t.Context())
|
|
|
|
var live int
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() {
|
|
live++
|
|
}
|
|
}
|
|
if live != 1 {
|
|
t.Errorf("an always-on model was evicted for being idle: %d live", live)
|
|
}
|
|
}
|
|
|
|
// --- eviction under pressure ---
|
|
|
|
// twoOnDemandModels builds a catalog where each model fills the whole node, so
|
|
// the second can only run if the first stops.
|
|
func twoOnDemandModels(t *testing.T) *catalog.Catalog {
|
|
t.Helper()
|
|
mk := func(name string) catalog.Model {
|
|
m := testModel(1)
|
|
m.Name = name
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: time.Hour}
|
|
return m
|
|
}
|
|
built, err := catalog.New([]catalog.Model{mk("llama"), mk("second")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return built
|
|
}
|
|
|
|
func TestAWantedModelEvictsAnIdleOneWhenTheClusterIsFull(t *testing.T) {
|
|
// On-demand loading works right up to the moment the cluster is full and
|
|
// then stops meaning anything: the model somebody just asked for cannot be
|
|
// placed, and the cards are held by one nobody has asked for in an hour.
|
|
// That is the only situation in which any of this matters.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
first := h.r.Replicas()
|
|
if len(first) != 1 {
|
|
t.Fatalf("the first model did not start: %d replicas", len(first))
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{first[0].ID: h.clock.now()})
|
|
|
|
// It goes quiet. Past the eviction grace and the minimum lifetime, so it is
|
|
// genuinely a candidate rather than merely recent.
|
|
h.advance(30 * time.Minute)
|
|
|
|
// Asked for directly, rather than through a Reconcile pass, because two
|
|
// on-demand models both wanting the only free node is a race the scheduler
|
|
// resolves in whatever order the catalog iterates — and what is being
|
|
// tested here is that room gets made, not who wins.
|
|
second := mustResolve(t, h, "second")
|
|
if !h.r.makeRoomFor(t.Context(), second) {
|
|
t.Fatal("no room was made for a model that could not be placed")
|
|
}
|
|
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.Model == "llama" && rep.State.Counts() {
|
|
t.Error("the idle replica was not evicted")
|
|
}
|
|
}
|
|
// And its GPUs are back, which is what the wanted model needed.
|
|
if free := len(h.inv.FreeGPUs("1")); free != 4 {
|
|
t.Errorf("node has %d free GPUs after the eviction, want 4", free)
|
|
}
|
|
}
|
|
|
|
func TestPlacementFailureTriggersEviction(t *testing.T) {
|
|
// The wiring: scaleUp is what notices there is nowhere to put a model, and
|
|
// is the only place that knows to try making room.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(30 * time.Minute)
|
|
|
|
// The full node has no room, so placing this must free some.
|
|
h.r.scaleUp(t.Context(), mustResolve(t, h, "second"), nil)
|
|
|
|
if free := len(h.inv.FreeGPUs("1")); free == 0 {
|
|
t.Error("a placement failure did not make room")
|
|
}
|
|
}
|
|
|
|
func TestAnAlwaysOnModelIsNeverEvicted(t *testing.T) {
|
|
// min 1 is a promise that it stays up. Quietly breaking it to serve another
|
|
// request would make every guarantee in the catalog conditional on load.
|
|
alwaysOn := testModel(1) // min 1
|
|
onDemand := testModel(1)
|
|
onDemand.Name = "second"
|
|
onDemand.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
|
built, err := catalog.New([]catalog.Model{alwaysOn, onDemand})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h := newHarness(t, alwaysOn, gpuNode("1", 4))
|
|
h.r.cat.Store(built)
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("the always-on model did not start")
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(24 * time.Hour)
|
|
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("an always-on model was offered for eviction: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaNothingHasReportedIsNeverEvicted(t *testing.T) {
|
|
// Unknown is not idle. After a router restart that is every replica, and
|
|
// evicting on the strength of a fact nobody has reported would empty the
|
|
// cluster at exactly the wrong moment.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
|
|
h.advance(24 * time.Hour) // idle by any measure, but nothing has said so
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("a replica with no reported usage was offered for eviction: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAMultiNodeReplicaIsLeftAlone(t *testing.T) {
|
|
// Evicting one frees several machines at once and takes tens of minutes to
|
|
// rebuild — a decision worth making deliberately rather than as a side
|
|
// effect of one request.
|
|
m := testModel(1, catalog.Profile{
|
|
Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
|
|
})
|
|
m.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
|
other := testModel(1)
|
|
other.Name = "second"
|
|
other.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
|
built, err := catalog.New([]catalog.Model{m, other})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a"))
|
|
h.r.cat.Store(built)
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 || len(reps[0].Shards) != 2 {
|
|
t.Fatalf("expected a two-node replica, got %+v", reps)
|
|
}
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(24 * time.Hour)
|
|
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("a multi-node replica was offered for eviction: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestOnlyOneReplicaIsEvictedPerPass(t *testing.T) {
|
|
// Freeing everything that qualifies would empty a cluster to satisfy a
|
|
// request that may only have needed one node.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
h.r.Want("llama")
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
reps := h.r.Replicas()
|
|
used := map[string]time.Time{}
|
|
for _, rep := range reps {
|
|
used[rep.ID] = h.clock.now()
|
|
}
|
|
h.r.RecordUsage(used)
|
|
h.advance(time.Hour)
|
|
|
|
before := len(h.r.evictable("second"))
|
|
if before < 1 {
|
|
t.Skip("nothing evictable to begin with")
|
|
}
|
|
h.r.makeRoomFor(t.Context(), mustResolve(t, h, "second"))
|
|
|
|
var stopped int
|
|
for _, rep := range h.r.Replicas() {
|
|
if !rep.State.Counts() {
|
|
stopped++
|
|
}
|
|
}
|
|
if stopped > 1 {
|
|
t.Errorf("%d replicas were stopped in one pass", stopped)
|
|
}
|
|
}
|
|
|
|
func mustResolve(t *testing.T, h *harness, name string) catalog.Model {
|
|
t.Helper()
|
|
m, ok := h.r.cat.Load().Resolve(name)
|
|
if !ok {
|
|
t.Fatalf("model %q is not in the catalog", name)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
|
|
// What stops two models on a cluster with room for one from taking turns
|
|
// evicting each other, paying a model load each time. A load is expensive
|
|
// enough that waiting is strictly better than winning.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.cat.Store(twoOnDemandModels(t))
|
|
h.r.Want("llama")
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("setup: %d replicas", len(reps))
|
|
}
|
|
|
|
// Used once, then idle well past the eviction grace — but only just
|
|
// started, which is the case this protects.
|
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
|
h.advance(DefaultConfig().EvictionGrace + time.Minute)
|
|
|
|
if got := h.r.evictable("second"); len(got) != 0 {
|
|
t.Errorf("a replica younger than the minimum lifetime was offered: %+v", got)
|
|
}
|
|
|
|
// Once it has been up long enough, it is fair game.
|
|
h.advance(DefaultConfig().MinLifetime)
|
|
if got := h.r.evictable("second"); len(got) == 0 {
|
|
t.Error("a long-running idle replica was never offered for eviction")
|
|
}
|
|
}
|
|
|
|
func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) {
|
|
// The fingerprint has to leave the control plane, or it cannot come back
|
|
// after a restart — and a restart is exactly when the configuration is
|
|
// likely to have been edited.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Reconcile(t.Context())
|
|
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
t.Fatal("nothing was started")
|
|
}
|
|
// From the catalog rather than the literal, because the catalog fills in
|
|
// defaults and the fingerprint is of the profile as it is actually run.
|
|
m, ok := h.r.cat.Load().Resolve("llama")
|
|
if !ok {
|
|
t.Fatal("the model is not in the catalog")
|
|
}
|
|
want := m.Profiles[0].Fingerprint()
|
|
for _, cmd := range started {
|
|
if cmd.ProfileFP != want {
|
|
t.Errorf("shard %s carries %q, expected %q", cmd.ShardID, cmd.ProfileFP, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnAdoptedReplicaKeepsTheProfileVersionItWasStartedWith(t *testing.T) {
|
|
// Adoption used to leave this empty, which made every replica read as
|
|
// unknown after a control plane restart — so an edited profile could never
|
|
// be noticed on the machines that were already running.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
|
|
shard := adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})
|
|
shard.ProfileFP = "1122334455667788"
|
|
h.r.Adopt("1", []AdoptedShard{shard})
|
|
|
|
reps := h.r.Replicas()
|
|
if len(reps) != 1 {
|
|
t.Fatalf("expected 1 adopted replica, got %d", len(reps))
|
|
}
|
|
if reps[0].ProfileFP != "1122334455667788" {
|
|
t.Errorf("the agent's answer was dropped: %q", reps[0].ProfileFP)
|
|
}
|
|
}
|
|
|
|
func TestAnAgentTooOldToReportTheProfileVersionIsNotGuessedAt(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.Adopt("1", []AdoptedShard{adopted("llama-99", "llama-99-r0", "1", 0, []int{0, 1, 2, 3})})
|
|
|
|
// Empty, not filled in from the catalog. Copying the current profile here
|
|
// would make the replica claim to match whatever the catalog says next,
|
|
// which is the opposite of what the field is for.
|
|
if fp := h.r.Replicas()[0].ProfileFP; fp != "" {
|
|
t.Errorf("an unreported fingerprint should stay empty, got %q", fp)
|
|
}
|
|
}
|
|
|
|
func TestAnEngineIsStartedWithItsReplicasKey(t *testing.T) {
|
|
// Engines listen on every address a node has. Started without a key, any
|
|
// machine that could reach the port reached the model past the gateway's
|
|
// authentication, quotas and accounting — checked from a neighbouring node
|
|
// on the live cluster.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
h.r.cfg.EngineKey = func(id string) string { return "key-for-" + id }
|
|
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
t.Fatal("nothing was started")
|
|
}
|
|
for _, c := range started {
|
|
if c.Spec.APIKey != "key-for-"+c.ReplicaID {
|
|
t.Errorf("shard %s started with key %q", c.ShardID, c.Spec.APIKey)
|
|
}
|
|
}
|
|
}
|
|
|
|
func replicaByID(h *harness, id string) (Replica, bool) {
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.ID == id {
|
|
return rep, true
|
|
}
|
|
}
|
|
return Replica{}, false
|
|
}
|
|
|
|
func TestARestartedReplicaServesUntilItsReplacementIsReady(t *testing.T) {
|
|
// Restarting is how a running engine picks up a key, an edited profile or
|
|
// a new engine version. Stopping first would leave a one-replica model
|
|
// unserved for as long as its replacement takes to load.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if rep, _ := replicaByID(h, old); rep.State != Ready || h.r.RestartState(old) != "replacing" {
|
|
t.Fatalf("the old replica stopped serving before its replacement was up: %+v", rep)
|
|
}
|
|
if n := len(h.r.Replicas()); n != 2 {
|
|
t.Fatalf("no replacement was placed beside it: %d replicas", n)
|
|
}
|
|
|
|
h.markReady()
|
|
h.r.Reconcile(t.Context())
|
|
h.r.Reconcile(t.Context())
|
|
if _, still := replicaByID(h, old); still {
|
|
if rep, _ := replicaByID(h, old); rep.State != Draining && rep.State != Stopped {
|
|
t.Errorf("the old replica is %s after its replacement became ready", rep.State)
|
|
}
|
|
}
|
|
if got := len(h.r.ReadyReplicas()); got != 1 {
|
|
t.Errorf("%d ready replicas at the end; the model asks for 1", got)
|
|
}
|
|
}
|
|
|
|
func TestARestartWithNoRoomBesideItHappensInPlace(t *testing.T) {
|
|
// One node, room for one replica: the replacement can only start once the
|
|
// old one has gone. It is the only thing in the way, so it goes now —
|
|
// waiting out a grace period first changed nothing but the wait, which
|
|
// in the field read as a restart that never happened.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if rep, ok := replicaByID(h, old); ok && rep.State == Ready {
|
|
t.Fatal("still serving with no replacement possible beside it")
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
ready := h.r.ReadyReplicas()
|
|
if len(ready) != 1 || ready[0].ID == old {
|
|
t.Errorf("expected one new replica in its place, got %+v", ready)
|
|
}
|
|
}
|
|
|
|
func TestARestartWhoseReplacementFitsNowhereIsGivenUp(t *testing.T) {
|
|
// The profile was edited to need more cards than the node has: stopping
|
|
// the old replica would leave the model with nothing at all.
|
|
m := testModel(1)
|
|
h := newHarness(t, m, gpuNode("1", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
|
|
m.Profiles[0].TP, m.Profiles[0].GPUsPerNode = 8, 8
|
|
h.setModels(t, m)
|
|
if err := h.r.Restart(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for range 3 {
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(h.r.cfg.DrainGrace + time.Second)
|
|
}
|
|
if rep, ok := replicaByID(h, old); !ok || rep.State != Ready {
|
|
t.Fatalf("the only replica was stopped for a replacement that fits nowhere: %+v", rep)
|
|
}
|
|
if st := h.r.RestartState(old); !strings.HasPrefix(st, "refused: ") || !strings.Contains(st, "fits on no node") {
|
|
t.Errorf("restart state %q should say it was given up, and why", st)
|
|
}
|
|
// Asked again, it is tried again.
|
|
if err := h.r.Restart(old); err != nil || h.r.RestartState(old) != "queued" {
|
|
t.Errorf("a new restart did not clear the refusal: %q, %v", h.r.RestartState(old), err)
|
|
}
|
|
}
|
|
|
|
func TestARestartAskedInPlaceStopsFirst(t *testing.T) {
|
|
// Room beside it, but in place was asked for: an engine that must not
|
|
// run twice, or memory somebody knows is short.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
old := h.r.ReadyReplicas()[0].ID
|
|
if err := h.r.RestartInPlace(old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if rep, ok := replicaByID(h, old); ok && rep.State == Ready {
|
|
t.Fatalf("asked in place, it kept serving beside its replacement: %+v", rep)
|
|
}
|
|
}
|
|
|
|
func TestReplicasOfAModelAreRestartedOneAtATime(t *testing.T) {
|
|
h := newHarness(t, testModel(2), gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
for i := 0; i < 3; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
var ids []string
|
|
for _, rep := range h.r.ReadyReplicas() {
|
|
ids = append(ids, rep.ID)
|
|
}
|
|
if err := h.r.Restart(ids...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
states := []string{h.r.RestartState(ids[0]), h.r.RestartState(ids[1])}
|
|
if !slices.Contains(states, "replacing") || !slices.Contains(states, "queued") {
|
|
t.Errorf("expected one replacing and one queued, got %v", states)
|
|
}
|
|
if err := h.r.Restart("no-such"); err == nil {
|
|
t.Error("an unknown replica was accepted")
|
|
}
|
|
}
|
|
|
|
// failRepeatedly starts and fails the model's replica n times over.
|
|
func failRepeatedly(t *testing.T, h *harness, n int, why string) {
|
|
t.Helper()
|
|
for i := 0; i < n; i++ {
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
started := h.disp.startedShards()
|
|
if len(started) == 0 {
|
|
return // parked, or still in backoff
|
|
}
|
|
s := started[0]
|
|
h.r.OnShardStatus(s.ReplicaID, s.ShardID, supervisor.Failed, why)
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
|
}
|
|
}
|
|
|
|
func TestAModelThatAlwaysFailsIsParkedNotRetriedForever(t *testing.T) {
|
|
// kev-0.5b: an architecture the node's llama.cpp did not know, started
|
|
// 155 times in eight hours, the same error every time.
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
failRepeatedly(t, h, DefaultConfig().ParkAfter, "unknown model architecture: 'ggmlc'")
|
|
|
|
h.disp.reset()
|
|
h.advance(DefaultConfig().MaxBackoff + time.Second)
|
|
h.r.Reconcile(t.Context())
|
|
if n := len(h.disp.startedShards()); n != 0 {
|
|
t.Fatalf("a parked model was started again (%d shards)", n)
|
|
}
|
|
var mh ModelHealth
|
|
for _, x := range h.r.ModelHealth() {
|
|
if x.Model == "llama" {
|
|
mh = x
|
|
}
|
|
}
|
|
if !mh.Parked || !strings.Contains(mh.Reason, "ggmlc") || mh.Failures < DefaultConfig().ParkAfter {
|
|
t.Errorf("health %+v", mh)
|
|
}
|
|
if why, no := h.r.Unservable("llama"); !no || !strings.Contains(why, "ggmlc") {
|
|
t.Errorf("a request would not be told why: %q %v", why, no)
|
|
}
|
|
|
|
// Retried by the operator once fixed.
|
|
if !h.r.Retry("llama") {
|
|
t.Error("Retry did not report the model as parked")
|
|
}
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Error("a retried model was not placed")
|
|
}
|
|
}
|
|
|
|
func TestChangingAParkedModelsDefinitionTriesItAgain(t *testing.T) {
|
|
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
|
failRepeatedly(t, h, DefaultConfig().ParkAfter, "boom")
|
|
|
|
fixed := testModel(1)
|
|
fixed.Source = "/models/llama-fixed"
|
|
cat, err := catalog.New([]catalog.Model{fixed})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.r.cat.Store(cat)
|
|
h.disp.reset()
|
|
h.r.Reconcile(t.Context())
|
|
if len(h.disp.startedShards()) == 0 {
|
|
t.Error("a model whose definition changed stayed parked")
|
|
}
|
|
}
|
|
|
|
func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
h := newHarness(t, testModel(1), gpuNode("1", 1)) // tp4 needs 4 cards
|
|
h.r.log = slog.New(slog.NewTextHandler(&buf, nil))
|
|
for i := 0; i < 20; i++ {
|
|
h.r.Reconcile(t.Context())
|
|
h.advance(5 * time.Second)
|
|
}
|
|
if n := strings.Count(buf.String(), "no placement for model"); n != 1 {
|
|
t.Errorf("logged %d times over 20 passes", n)
|
|
}
|
|
h.advance(placementLogEvery)
|
|
h.r.Reconcile(t.Context())
|
|
if n := strings.Count(buf.String(), "no placement for model"); n != 2 {
|
|
t.Errorf("not repeated after %s: %d", placementLogEvery, n)
|
|
}
|
|
var mh ModelHealth
|
|
for _, x := range h.r.ModelHealth() {
|
|
mh = x
|
|
}
|
|
if !mh.Unplaceable || mh.Desired != 1 {
|
|
t.Errorf("health %+v", mh)
|
|
}
|
|
}
|
|
|
|
func liveCount(h *harness) int {
|
|
n := 0
|
|
for _, r := range h.r.Replicas() {
|
|
if r.State.Counts() {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func TestLoadAddsReplicasUpToMaxAndQuietTakesThemBack(t *testing.T) {
|
|
m := testModel(1)
|
|
m.Scaling.Max = 3
|
|
h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
busy := map[string]Load{"llama": {Replicas: 1, Running: 4, Waiting: 3}}
|
|
|
|
// A spike shorter than ScaleUpAfter adds nothing.
|
|
h.r.RecordLoad(busy)
|
|
h.advance(30 * time.Second)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
if n := liveCount(h); n != 1 {
|
|
t.Fatalf("%d replicas after 30s of load", n)
|
|
}
|
|
|
|
// Held past it, one more.
|
|
h.advance(31 * time.Second)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
if n := liveCount(h); n != 2 {
|
|
t.Fatalf("%d replicas after a minute of load; expected 2", n)
|
|
}
|
|
|
|
// Not another while that one is still starting.
|
|
h.advance(2 * time.Minute)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
if n := liveCount(h); n != 2 {
|
|
t.Fatalf("stacked a replica on one still starting: %d", n)
|
|
}
|
|
|
|
// Once it is up and load holds, the third — and never a fourth.
|
|
h.markReady()
|
|
for i := 0; i < 5; i++ {
|
|
h.r.RecordLoad(busy)
|
|
h.advance(61 * time.Second)
|
|
h.r.RecordLoad(busy)
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
}
|
|
if n := liveCount(h); n != 3 {
|
|
t.Fatalf("%d replicas under sustained load; max is 3", n)
|
|
}
|
|
|
|
// Quiet: back down one step per ScaleDownAfter, to min and no further.
|
|
quiet := map[string]Load{"llama": {Replicas: 3}}
|
|
for i := 0; i < 6; i++ {
|
|
h.r.RecordLoad(quiet)
|
|
h.advance(DefaultConfig().ScaleDownAfter + time.Second)
|
|
h.r.RecordLoad(quiet)
|
|
for j := 0; j < 3; j++ {
|
|
h.r.Reconcile(t.Context())
|
|
}
|
|
}
|
|
if n := liveCount(h); n != 1 {
|
|
t.Errorf("%d replicas after a long quiet spell; min is 1", n)
|
|
}
|
|
}
|