Compare commits

...
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 e7a8105aa4 Log a preempted replica's stop with the grace it had
Release / build (push) Successful in 1m7s
Release / notes (push) Successful in 2s
CI / test (push) Successful in 3m47s
It said "stopping a drained replica without a replacement after=5m0s" when
it had been given the 30-second preemption grace.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:41:43 +02:00
Ilker ManapandClaude Opus 5.5 d0e9813ce9 Of equally idle victims, try the smallest first
Release / build (push) Successful in 1m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m55s
Idle times are unknown after a control plane restart, which left the choice
between a 1-core and a 4-core replica to map order.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:23:08 +02:00
Ilker ManapandClaude Opus 5.5 0e73f37128 Stop only the replicas that make the room
Release / build (push) Successful in 1m4s
CI / test (push) Successful in 4m32s
Release / notes (push) Successful in 3s
Candidates were added in order until the model fitted, and every one added
was stopped, including those tried first that freed room on the wrong node.
A set that works is now pruned to the replicas it needs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:21:58 +02:00
4 changed files with 139 additions and 17 deletions

No files matched your search

+5 -2
View File
@@ -70,9 +70,12 @@ these rules:
loaded, nor one spanning several nodes.
- Idle on-demand replicas go first, then what lower classes grew into, then
the same within the class, then lower classes' minimums; the longest unused
first within each.
first within each, and of equally idle ones the smallest.
- The set is tried on a copy of the inventory first, at most three replicas,
and nothing is stopped unless it makes the room.
and nothing is stopped unless it makes the room. Of a set that does, each
replica whose stopping turns out not to be needed is left running, so an
idle replica tried first on the wrong node is not stopped with the one that
made the room.
A replica stopped this way is drained and given 30 seconds to finish what it
is serving. Its model is held back for up to five minutes, or until the model
+50 -12
View File
@@ -29,6 +29,9 @@ type victim struct {
AboveMin bool
Tier int
Idle time.Duration
// Size is what stopping it frees, for choosing the smaller of two
// otherwise equal victims.
Size quota.Amount
}
// preemptFor stops replicas of lower classes when a model cannot otherwise be
@@ -54,28 +57,46 @@ func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []st
if len(cands) == 0 {
return false
}
trial := r.inv.Clone()
sched := placement.New(trial)
// Candidates in order until the model fits; then every one whose
// stopping turns out not to be needed is kept after all. Without that
// second step an idle replica on the wrong node, tried first, would be
// stopped along with the one that actually made the room.
var chosen []victim
for _, v := range cands {
if len(chosen) == maxVictims {
break
}
trial.ReleaseReplica(v.ID)
chosen = append(chosen, v)
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
res = sched.Place(r.orderedProfiles(m), nil)
}
if res.Placement == nil {
if !r.fitsWithout(m, avoid, chosen) {
continue
}
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
if r.fitsWithout(m, avoid, fewer) {
chosen = fewer
}
}
r.preempt(ctx, m, chosen)
return true
}
return false
}
// fitsWithout reports whether m could be placed with these replicas gone,
// tried on a copy of the inventory.
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
trial := r.inv.Clone()
for _, v := range gone {
trial.ReleaseReplica(v.ID)
}
sched := placement.New(trial)
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
res = sched.Place(r.orderedProfiles(m), nil)
}
return res.Placement != nil
}
// preempt drains the chosen replicas and holds their models back.
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
now := r.now()
@@ -245,15 +266,32 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
if len(nodesOf(rep)) > 1 {
continue // see makeRoomFor: a multi-node replica is its own decision
}
prof, _ := other.ProfileByName(rep.Profile)
var size quota.Amount
for _, sh := range rep.Shards {
size = size.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
}
out = append(out, victim{ID: rep.ID, Model: model, Class: class,
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now)})
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now), Size: size})
}
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Tier != out[j].Tier {
return out[i].Tier < out[j].Tier
a, b := out[i], out[j]
if a.Tier != b.Tier {
return a.Tier < b.Tier
}
return out[i].Idle > out[j].Idle
if a.Idle != b.Idle {
return a.Idle > b.Idle
}
// The smaller first: the least served is lost if it is enough, and
// the pruning after keeps a larger one only when it is needed.
if a.Size.GPUs != b.Size.GPUs {
return a.Size.GPUs < b.Size.GPUs
}
if a.Size.MemoryGB != b.Size.MemoryGB {
return a.Size.MemoryGB < b.Size.MemoryGB
}
return a.ID < b.ID
})
return out
}
@@ -235,3 +235,80 @@ func TestAHoldLastsUntilTheRoomIsUsedOrItExpires(t *testing.T) {
t.Error("a hold outlived its time")
}
}
func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) {
// Two half-used nodes. "idle" on node 1 is on demand and long unused, so
// it is first in line; but the critical model needs node 2's other half,
// which only stopping "chat" frees.
idle := catalog.Model{Name: "idle", Source: "/m", Scaling: catalog.Scaling{Min: 0, Max: 1},
Profiles: []catalog.Profile{{Name: "tp4", Engine: "vllm", GPUClass: "h100-80g", TP: 4}}}
chat := catalog.Model{Name: "chat", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "tp6", Engine: "vllm", GPUClass: "h100-80g", TP: 6}}}
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "tp8", Engine: "vllm", GPUClass: "h100-80g", TP: 8}}}
h := newFleetHarness(t, []catalog.Model{idle, chat}, gpuNode("1", 6), gpuNode("2", 8))
h.r.Want("idle")
h.settle(t, 4)
var idleNode, chatNode string
for _, rep := range h.r.Replicas() {
switch rep.Model {
case "idle":
idleNode = rep.Shards[0].NodeID
case "chat":
chatNode = rep.Shards[0].NodeID
}
}
if idleNode == "" || chatNode != "2" {
t.Fatalf("setup: idle on %q, chat on %q; chat must hold node 2: %+v", idleNode, chatNode, h.r.Replicas())
}
h.r.mu.Lock()
for _, rep := range h.r.replicas {
rep.LastUsedAt = h.r.now().Add(-time.Hour)
}
h.r.mu.Unlock()
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, _ := catalog.New([]catalog.Model{idle, chat, vital})
h.r.cat.Store(cat)
h.r.Reconcile(t.Context())
var stopped []string
for _, rep := range h.r.Replicas() {
if rep.PreemptedFor == "vital" {
stopped = append(stopped, rep.Model)
}
}
if len(stopped) != 1 || stopped[0] != "chat" {
t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped)
}
}
func TestOfTwoThatWouldEachMakeRoomTheSmallerIsStopped(t *testing.T) {
cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 31 * 1024, CPUCount: 8}
big := catalog.Model{Name: "big", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12, CPUCores: 4}}}
small := catalog.Model{Name: "small", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 2, CPUCores: 1}}}
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, CPUCores: 4}}}
// 8 cores: big 4 + small 1 leaves 3, and vital needs 4. Stopping either
// is enough; stopping big loses far more.
for range 5 { // the choice must not depend on map order
h := newFleetHarness(t, []catalog.Model{big, small}, cpu)
h.settle(t, 3)
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, _ := catalog.New([]catalog.Model{big, small, vital})
h.r.cat.Store(cat)
h.r.Reconcile(t.Context())
var stopped []string
for _, rep := range h.r.Replicas() {
if rep.PreemptedFor == "vital" {
stopped = append(stopped, rep.Model)
}
}
if len(stopped) != 1 || stopped[0] != "small" {
t.Fatalf("stopped %v; the small replica alone makes room", stopped)
}
}
}
+7 -3
View File
@@ -755,10 +755,10 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
r.mu.Lock()
rep, ok := r.replicas[id]
var expired bool
grace, preemptedFor := r.cfg.DrainGrace, ""
if ok {
grace := r.cfg.DrainGrace
if rep.PreemptedFor != "" {
grace = r.cfg.PreemptGrace
grace, preemptedFor = r.cfg.PreemptGrace, rep.PreemptedFor
}
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace
}
@@ -768,11 +768,15 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
case !ok:
case replaced:
r.stopReplica(ctx, id)
case expired && preemptedFor != "":
r.log.Info("stopping a preempted replica", "replica", id,
"for", preemptedFor, "after", grace)
r.stopReplica(ctx, id)
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", r.cfg.DrainGrace)
"replica", id, "after", grace)
r.stopReplica(ctx, id)
}
}