From 9b32531ce30f91686ccc46ed03e7ba22817850af Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 17 Sep 2026 17:51:07 +0200 Subject: [PATCH] Stop an idle model to make room for one somebody asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-demand loading worked right up to the moment the cluster was full and then stopped meaning anything: the model somebody had just asked for could not be placed, and the cards were held by a model nobody had asked for in an hour. That is the only situation in which any of it matters. Deliberately narrow. Only idle replicas of other on-demand models — a model at min 1 is a promise that it stays up, and breaking it to serve a different request would make every guarantee in the catalog conditional on load. Only single-node replicas, since evicting a multi-node one frees several machines and costs tens of minutes to rebuild. One at a time, because freeing everything that qualifies would empty a cluster for a request that needed one node. And never a replica with no reported usage: after a router restart that is all of them, and unknown is not idle. Writing the test found a gap the feature had rather than a bug in the test. Two on-demand models on a cluster with room for one would take turns evicting each other, at a model load per cycle — worse for both than one of them waiting. A replica is now safe from eviction for a few minutes after it starts serving, which bounds that. Which in turn found a latent bug: deriveState stamped ReadyAt with time.Now() inside a package that has an injectable clock, so the new lifetime rule could not be tested at all — the fake clock said three minutes and the wall clock said eight months. Invisible while nothing compared ReadyAt to anything. Co-Authored-By: Claude Opus 5 (1M context) --- docs/install.md | 9 + internal/control/reconciler/evict.go | 120 +++++++++ internal/control/reconciler/reconciler.go | 40 ++- .../control/reconciler/reconciler_test.go | 234 ++++++++++++++++++ internal/control/reconciler/replica.go | 11 +- internal/control/reconciler/stuck_test.go | 6 +- 6 files changed, 412 insertions(+), 8 deletions(-) create mode 100644 internal/control/reconciler/evict.go diff --git a/docs/install.md b/docs/install.md index f6dc7e5..437bba1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -515,6 +515,15 @@ models used daily — keep those at `min: 1`. And stopping a replica throws away the KV cache prefix routing spent its effort filling, so an idle timeout wants to be minutes rather than seconds. +When the cluster is full, a model somebody asks for will stop an idle one to +make room. Only idle replicas of other on-demand models: a model at `min: 1` is +a promise that it stays up, and breaking that to serve a different request would +make every guarantee in the catalog conditional on load. Only single-node +replicas, because evicting a multi-node one frees several machines and costs +tens of minutes to rebuild — a decision worth making deliberately. One at a +time. And never a replica that has just started, which is what stops two models +on a cluster with room for one from taking turns evicting each other. + For a GPU model, name the class and the width instead: ```yaml diff --git a/internal/control/reconciler/evict.go b/internal/control/reconciler/evict.go new file mode 100644 index 0000000..d7a8bc9 --- /dev/null +++ b/internal/control/reconciler/evict.go @@ -0,0 +1,120 @@ +package reconciler + +import ( + "context" + "fmt" + "sort" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +// evictionCandidate is a replica that could be stopped to make room. +type evictionCandidate struct { + ID string + Model string + Idle time.Duration + GPUs int + // Nodes is how many machines the replica spans. More than one is left + // alone — see makeRoomFor. + Nodes int +} + +// makeRoomFor stops an idle replica so a wanted model can be placed. +// +// Without this, on-demand loading works right up to the moment the cluster is +// full and then stops meaning anything: the model somebody just asked for +// cannot be placed, and the cards are held by a model nobody has asked for in +// an hour. That is the exact situation the feature exists to resolve, and it is +// the only situation in which it matters. +// +// Deliberately conservative, on three counts. +// +// Only idle replicas of on-demand models. A model at min 1 is a promise that it +// stays up, and quietly breaking that to serve a different request would make +// every guarantee in the catalog conditional on load. +// +// Only single-node replicas. Evicting a multi-node replica frees several +// machines at once and takes tens of minutes to rebuild, which is a decision +// worth making deliberately rather than as a side effect of one request. +// +// One at a time. Freeing everything that qualifies would empty a cluster to +// satisfy a request that may only have needed one node, and the next pass can +// always free more. +func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool { + candidates := r.evictable(m.Name) + if len(candidates) == 0 { + return false + } + + // The longest idle first. It is the replica least likely to be wanted + // again, and the one whose KV cache is worth the least. + best := candidates[0] + r.log.Info("evicting an idle replica to make room", + "stopping", best.ID, "idle_model", best.Model, + "idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name) + r.stopReplica(ctx, best.ID) + return true +} + +// evictable lists replicas that could be stopped for a model that needs room, +// longest idle first. +func (r *Reconciler) evictable(wantedBy string) []evictionCandidate { + cat := r.cat.Load() + now := r.now() + + r.mu.Lock() + defer r.mu.Unlock() + + var out []evictionCandidate + for _, rep := range r.replicas { + if rep.Model == wantedBy || !rep.State.Counts() { + continue + } + other, ok := cat.Resolve(rep.Model) + if !ok || !other.Scaling.OnDemand() { + continue + } + // Unknown is not idle. A replica the router has said nothing about — + // after a restart, that is all of them — must not be evicted on the + // strength of a fact nobody has reported. + if rep.LastUsedAt.IsZero() { + continue + } + idle := now.Sub(rep.LastUsedAt) + if idle < r.cfg.EvictionGrace { + continue + } + // A replica that has only just started is left alone however idle it + // looks. Two on-demand models on a cluster with room for one would + // otherwise take turns evicting each other, paying a model load each + // time — and a load is expensive enough that waiting is strictly + // better than winning. + if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime { + continue + } + nodes := map[string]struct{}{} + gpus := 0 + for _, sh := range rep.Shards { + nodes[sh.NodeID] = struct{}{} + gpus += len(sh.GPUIndices) + } + if len(nodes) > 1 { + continue + } + out = append(out, evictionCandidate{ + ID: rep.ID, Model: rep.Model, Idle: idle, GPUs: gpus, Nodes: len(nodes), + }) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Idle > out[j].Idle }) + return out +} + +// EvictionReason explains why room could not be made, for placement failures. +func (r *Reconciler) EvictionReason(model string) string { + if c := r.evictable(model); len(c) > 0 { + return fmt.Sprintf("%d idle replica(s) could be stopped to make room", len(c)) + } + return "nothing idle enough to stop" +} diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index aff6cbd..71c41dd 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -66,6 +66,21 @@ type Config struct { PortRange [2]int // MgmtInterface is the ethernet interface used for NCCL bootstrap. MgmtInterface string + // EvictionGrace is how long a replica must have gone unasked-for before it + // may be stopped to make room for another model. + // + // Shorter than IdleTimeout on purpose: a replica idle for five minutes is + // not worth reclaiming on its own, and is worth reclaiming when something + // else needs the cards. Still minutes rather than seconds, because stopping + // one throws away the KV cache prefix routing spent filling. + EvictionGrace time.Duration + // MinLifetime is how long a replica is safe from eviction after it starts + // serving, whatever else wants its cards. + // + // The thing that stops two models on a cluster with room for one from + // taking turns evicting each other. A load costs minutes; oscillating at a + // load per cycle is worse for both models than one of them waiting. + MinLifetime time.Duration // IdleTimeout is how long an on-demand model stays loaded with nobody // asking for it. A model may override it. IdleTimeout time.Duration @@ -99,6 +114,8 @@ func DefaultConfig() Config { LostAfter: 60 * time.Second, SettleFor: 60 * time.Second, IdleTimeout: 15 * time.Minute, + EvictionGrace: 2 * time.Minute, + MinLifetime: 5 * time.Minute, DrainGrace: 5 * time.Minute, PortRange: [2]int{20000, 20999}, MgmtInterface: "eth0", @@ -119,6 +136,12 @@ func (c *Config) normalize() { if c.IdleTimeout <= 0 { c.IdleTimeout = d.IdleTimeout } + if c.EvictionGrace <= 0 { + c.EvictionGrace = d.EvictionGrace + } + if c.MinLifetime <= 0 { + c.MinLifetime = d.MinLifetime + } if c.SettleFor < 0 { c.SettleFor = 0 } else if c.SettleFor == 0 { @@ -729,6 +752,17 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin r.mu.Unlock() if res.Placement == nil { + // Full, with a model somebody has just asked for and cards held by one + // nobody has asked for in a while. Stopping one is the whole point of + // on-demand loading; without it the feature works until the cluster is + // full, which is the only time it matters. + // + // Not retried here. The replica is stopped, its GPUs are released on + // the pass that reaps it, and the next pass places this model — which + // keeps the stop-then-start ordering in one place rather than two. + if m.Scaling.OnDemand() && r.makeRoomFor(ctx, m) { + return + } r.log.Warn("no placement for model", "model", m.Name, "reasons", res.Explain()) return } @@ -1076,7 +1110,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) { } } slices.SortFunc(rep.Shards, func(a, b Shard) int { return a.Rank - b.Rank }) - rep.deriveState() + rep.deriveState(r.now()) r.log.Info("adopted replica from the fleet", "replica", rep.ID, "model", rep.Model, @@ -1132,7 +1166,7 @@ func (r *Reconciler) forgetShardsTheAgentDoesNotHave(nodeID string, running []Ad lost = append(lost, sh.ID) } if changed { - rep.deriveState() + rep.deriveState(r.now()) } } r.mu.Unlock() @@ -1203,7 +1237,7 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e } prev := rep.State - rep.deriveState() + rep.deriveState(r.now()) if rep.State == Ready && prev != Ready { // A model that comes up clears its backoff; the next failure starts // counting from scratch rather than from an old streak. diff --git a/internal/control/reconciler/reconciler_test.go b/internal/control/reconciler/reconciler_test.go index ee4cda3..c0dc7ad 100644 --- a/internal/control/reconciler/reconciler_test.go +++ b/internal/control/reconciler/reconciler_test.go @@ -1431,3 +1431,237 @@ func TestAnOrdinaryModelIsUnaffectedByIdleness(t *testing.T) { t.Errorf("an always-on model was evicted for being idle: %d live", live) } } + +// --- eviction under pressure --- + +// twoOnDemandModels builds a catalog where each model fills the whole node, so +// the second can only run if the first stops. +func twoOnDemandModels(t *testing.T) *catalog.Catalog { + t.Helper() + mk := func(name string) catalog.Model { + m := testModel(1) + m.Name = name + m.Scaling = catalog.Scaling{Min: 0, Max: 1, IdleTimeout: time.Hour} + return m + } + built, err := catalog.New([]catalog.Model{mk("llama"), mk("second")}) + if err != nil { + t.Fatal(err) + } + return built +} + +func TestAWantedModelEvictsAnIdleOneWhenTheClusterIsFull(t *testing.T) { + // On-demand loading works right up to the moment the cluster is full and + // then stops meaning anything: the model somebody just asked for cannot be + // placed, and the cards are held by one nobody has asked for in an hour. + // That is the only situation in which any of this matters. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + first := h.r.Replicas() + if len(first) != 1 { + t.Fatalf("the first model did not start: %d replicas", len(first)) + } + h.r.RecordUsage(map[string]time.Time{first[0].ID: h.clock.now()}) + + // It goes quiet. Past the eviction grace and the minimum lifetime, so it is + // genuinely a candidate rather than merely recent. + h.advance(30 * time.Minute) + + // Asked for directly, rather than through a Reconcile pass, because two + // on-demand models both wanting the only free node is a race the scheduler + // resolves in whatever order the catalog iterates — and what is being + // tested here is that room gets made, not who wins. + second := mustResolve(t, h, "second") + if !h.r.makeRoomFor(t.Context(), second) { + t.Fatal("no room was made for a model that could not be placed") + } + + for _, rep := range h.r.Replicas() { + if rep.Model == "llama" && rep.State.Counts() { + t.Error("the idle replica was not evicted") + } + } + // And its GPUs are back, which is what the wanted model needed. + if free := len(h.inv.FreeGPUs("1")); free != 4 { + t.Errorf("node has %d free GPUs after the eviction, want 4", free) + } +} + +func TestPlacementFailureTriggersEviction(t *testing.T) { + // The wiring: scaleUp is what notices there is nowhere to put a model, and + // is the only place that knows to try making room. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("setup: %d replicas", len(reps)) + } + h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) + h.advance(30 * time.Minute) + + // The full node has no room, so placing this must free some. + h.r.scaleUp(t.Context(), mustResolve(t, h, "second"), nil) + + if free := len(h.inv.FreeGPUs("1")); free == 0 { + t.Error("a placement failure did not make room") + } +} + +func TestAnAlwaysOnModelIsNeverEvicted(t *testing.T) { + // min 1 is a promise that it stays up. Quietly breaking it to serve another + // request would make every guarantee in the catalog conditional on load. + alwaysOn := testModel(1) // min 1 + onDemand := testModel(1) + onDemand.Name = "second" + onDemand.Scaling = catalog.Scaling{Min: 0, Max: 1} + built, err := catalog.New([]catalog.Model{alwaysOn, onDemand}) + if err != nil { + t.Fatal(err) + } + h := newHarness(t, alwaysOn, gpuNode("1", 4)) + h.r.cat.Store(built) + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("the always-on model did not start") + } + h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) + h.advance(24 * time.Hour) + + if got := h.r.evictable("second"); len(got) != 0 { + t.Errorf("an always-on model was offered for eviction: %+v", got) + } +} + +func TestAReplicaNothingHasReportedIsNeverEvicted(t *testing.T) { + // Unknown is not idle. After a router restart that is every replica, and + // evicting on the strength of a fact nobody has reported would empty the + // cluster at exactly the wrong moment. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + + h.advance(24 * time.Hour) // idle by any measure, but nothing has said so + if got := h.r.evictable("second"); len(got) != 0 { + t.Errorf("a replica with no reported usage was offered for eviction: %+v", got) + } +} + +func TestAMultiNodeReplicaIsLeftAlone(t *testing.T) { + // Evicting one frees several machines at once and takes tens of minutes to + // rebuild — a decision worth making deliberately rather than as a side + // effect of one request. + m := testModel(1, catalog.Profile{ + Name: "2node", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8, + }) + m.Scaling = catalog.Scaling{Min: 0, Max: 1} + other := testModel(1) + other.Name = "second" + other.Scaling = catalog.Scaling{Min: 0, Max: 1} + built, err := catalog.New([]catalog.Model{m, other}) + if err != nil { + t.Fatal(err) + } + + h := newHarness(t, m, gpuNode("1", 8, "ib-a"), gpuNode("2", 8, "ib-a")) + h.r.cat.Store(built) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + if len(reps) != 1 || len(reps[0].Shards) != 2 { + t.Fatalf("expected a two-node replica, got %+v", reps) + } + h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) + h.advance(24 * time.Hour) + + if got := h.r.evictable("second"); len(got) != 0 { + t.Errorf("a multi-node replica was offered for eviction: %+v", got) + } +} + +func TestOnlyOneReplicaIsEvictedPerPass(t *testing.T) { + // Freeing everything that qualifies would empty a cluster to satisfy a + // request that may only have needed one node. + h := newHarness(t, testModel(1), gpuNode("1", 4), gpuNode("2", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + h.r.Want("llama") + for range 3 { + h.r.Reconcile(t.Context()) + h.markReady() + } + reps := h.r.Replicas() + used := map[string]time.Time{} + for _, rep := range reps { + used[rep.ID] = h.clock.now() + } + h.r.RecordUsage(used) + h.advance(time.Hour) + + before := len(h.r.evictable("second")) + if before < 1 { + t.Skip("nothing evictable to begin with") + } + h.r.makeRoomFor(t.Context(), mustResolve(t, h, "second")) + + var stopped int + for _, rep := range h.r.Replicas() { + if !rep.State.Counts() { + stopped++ + } + } + if stopped > 1 { + t.Errorf("%d replicas were stopped in one pass", stopped) + } +} + +func mustResolve(t *testing.T, h *harness, name string) catalog.Model { + t.Helper() + m, ok := h.r.cat.Load().Resolve(name) + if !ok { + t.Fatalf("model %q is not in the catalog", name) + } + return m +} + +func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) { + // What stops two models on a cluster with room for one from taking turns + // evicting each other, paying a model load each time. A load is expensive + // enough that waiting is strictly better than winning. + h := newHarness(t, testModel(1), gpuNode("1", 4)) + h.r.cat.Store(twoOnDemandModels(t)) + h.r.Want("llama") + h.r.Reconcile(t.Context()) + h.markReady() + reps := h.r.Replicas() + if len(reps) != 1 { + t.Fatalf("setup: %d replicas", len(reps)) + } + + // Used once, then idle well past the eviction grace — but only just + // started, which is the case this protects. + h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()}) + h.advance(DefaultConfig().EvictionGrace + time.Minute) + + if got := h.r.evictable("second"); len(got) != 0 { + t.Errorf("a replica younger than the minimum lifetime was offered: %+v", got) + } + + // Once it has been up long enough, it is fair game. + h.advance(DefaultConfig().MinLifetime) + if got := h.r.evictable("second"); len(got) == 0 { + t.Error("a long-running idle replica was never offered for eviction") + } +} diff --git a/internal/control/reconciler/replica.go b/internal/control/reconciler/replica.go index ac99080..746e131 100644 --- a/internal/control/reconciler/replica.go +++ b/internal/control/reconciler/replica.go @@ -157,7 +157,14 @@ func (r *Replica) hasLiveShard() bool { return false } -func (r *Replica) deriveState() { +// deriveState recomputes the replica's state from its shards. +// +// Takes the time rather than reading the clock, because ReadyAt is now +// something decisions are made against — a replica is safe from eviction for a +// while after it starts serving — and a field stamped from the wall clock +// inside a package with an injectable one cannot be reasoned about in a test. +// It was invisible while nothing compared ReadyAt to anything. +func (r *Replica) deriveState(now time.Time) { if r.State == Draining || r.State == Stopped { return } @@ -198,7 +205,7 @@ func (r *Replica) deriveState() { } if ready == len(r.Shards) && (r.WantShards == 0 || len(r.Shards) == r.WantShards) { if r.State != Ready { - r.ReadyAt = time.Now() + r.ReadyAt = now } r.State = Ready return diff --git a/internal/control/reconciler/stuck_test.go b/internal/control/reconciler/stuck_test.go index d5898f2..6ec2e07 100644 --- a/internal/control/reconciler/stuck_test.go +++ b/internal/control/reconciler/stuck_test.go @@ -350,7 +350,7 @@ func TestAReplicaMissingAShardIsNotReady(t *testing.T) { ID: "r1", WantShards: 2, Shards: []Shard{{ID: "r1-r1", Rank: 1, State: supervisor.Ready}}, } - rep.deriveState() + rep.deriveState(time.Now()) if rep.State == Ready { t.Fatal("a replica with one of two shards is Ready") @@ -368,7 +368,7 @@ func TestACompleteReplicaIsReady(t *testing.T) { {ID: "r1-r1", Rank: 1, State: supervisor.Ready}, }, } - rep.deriveState() + rep.deriveState(time.Now()) if rep.State != Ready { t.Errorf("state = %s, want ready", rep.State) } @@ -383,7 +383,7 @@ func TestAReplicaStillComingUpIsNotFailed(t *testing.T) { {ID: "r1-r1", Rank: 1, State: supervisor.Loading}, }, } - rep.deriveState() + rep.deriveState(time.Now()) if rep.State != Starting { t.Errorf("state = %s, want starting", rep.State) }