Compare commits

...
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 e7a8105aa4 Log a preempted replica's stop with the grace it had
Release / build (push) Successful in 1m7s
Release / notes (push) Successful in 2s
CI / test (push) Successful in 3m47s
It said "stopping a drained replica without a replacement after=5m0s" when
it had been given the 30-second preemption grace.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:41:43 +02:00
Ilker ManapandClaude Opus 5.5 d0e9813ce9 Of equally idle victims, try the smallest first
Release / build (push) Successful in 1m11s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m55s
Idle times are unknown after a control plane restart, which left the choice
between a 1-core and a 4-core replica to map order.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:23:08 +02:00
4 changed files with 62 additions and 8 deletions

No files matched your search

+1 -1
View File
@@ -70,7 +70,7 @@ these rules:
loaded, nor one spanning several nodes.
- Idle on-demand replicas go first, then what lower classes grew into, then
the same within the class, then lower classes' minimums; the longest unused
first within each.
first within each, and of equally idle ones the smallest.
- The set is tried on a copy of the inventory first, at most three replicas,
and nothing is stopped unless it makes the room. Of a set that does, each
replica whose stopping turns out not to be needed is left running, so an
+24 -4
View File
@@ -29,6 +29,9 @@ type victim struct {
AboveMin bool
Tier int
Idle time.Duration
// Size is what stopping it frees, for choosing the smaller of two
// otherwise equal victims.
Size quota.Amount
}
// preemptFor stops replicas of lower classes when a model cannot otherwise be
@@ -263,15 +266,32 @@ func (r *Reconciler) victims(m catalog.Model) []victim {
if len(nodesOf(rep)) > 1 {
continue // see makeRoomFor: a multi-node replica is its own decision
}
prof, _ := other.ProfileByName(rep.Profile)
var size quota.Amount
for _, sh := range rep.Shards {
size = size.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
}
out = append(out, victim{ID: rep.ID, Model: model, Class: class,
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now)})
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now), Size: size})
}
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Tier != out[j].Tier {
return out[i].Tier < out[j].Tier
a, b := out[i], out[j]
if a.Tier != b.Tier {
return a.Tier < b.Tier
}
return out[i].Idle > out[j].Idle
if a.Idle != b.Idle {
return a.Idle > b.Idle
}
// The smaller first: the least served is lost if it is enough, and
// the pruning after keeps a larger one only when it is needed.
if a.Size.GPUs != b.Size.GPUs {
return a.Size.GPUs < b.Size.GPUs
}
if a.Size.MemoryGB != b.Size.MemoryGB {
return a.Size.MemoryGB < b.Size.MemoryGB
}
return a.ID < b.ID
})
return out
}
@@ -282,3 +282,33 @@ func TestOnlyTheReplicasThatMakeTheRoomAreStopped(t *testing.T) {
t.Errorf("stopped %v; only chat's replica makes room for a whole node", stopped)
}
}
func TestOfTwoThatWouldEachMakeRoomTheSmallerIsStopped(t *testing.T) {
cpu := inventory.Node{ID: "a", MgmtAddr: "10.0.0.1", State: inventory.Ready, MemoryTotalMB: 31 * 1024, CPUCount: 8}
big := catalog.Model{Name: "big", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 12, CPUCores: 4}}}
small := catalog.Model{Name: "small", Source: "/m", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 2, CPUCores: 1}}}
vital := catalog.Model{Name: "vital", Source: "/m", Class: catalog.ClassCritical,
Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 15, CPUCores: 4}}}
// 8 cores: big 4 + small 1 leaves 3, and vital needs 4. Stopping either
// is enough; stopping big loses far more.
for range 5 { // the choice must not depend on map order
h := newFleetHarness(t, []catalog.Model{big, small}, cpu)
h.settle(t, 3)
h.advance(h.r.cfg.MinLifetime + time.Minute)
cat, _ := catalog.New([]catalog.Model{big, small, vital})
h.r.cat.Store(cat)
h.r.Reconcile(t.Context())
var stopped []string
for _, rep := range h.r.Replicas() {
if rep.PreemptedFor == "vital" {
stopped = append(stopped, rep.Model)
}
}
if len(stopped) != 1 || stopped[0] != "small" {
t.Fatalf("stopped %v; the small replica alone makes room", stopped)
}
}
}
+7 -3
View File
@@ -755,10 +755,10 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
r.mu.Lock()
rep, ok := r.replicas[id]
var expired bool
grace, preemptedFor := r.cfg.DrainGrace, ""
if ok {
grace := r.cfg.DrainGrace
if rep.PreemptedFor != "" {
grace = r.cfg.PreemptGrace
grace, preemptedFor = r.cfg.PreemptGrace, rep.PreemptedFor
}
expired = !rep.DrainingAt.IsZero() && r.now().Sub(rep.DrainingAt) > grace
}
@@ -768,11 +768,15 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
case !ok:
case replaced:
r.stopReplica(ctx, id)
case expired && preemptedFor != "":
r.log.Info("stopping a preempted replica", "replica", id,
"for", preemptedFor, "after", grace)
r.stopReplica(ctx, id)
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", r.cfg.DrainGrace)
"replica", id, "after", grace)
r.stopReplica(ctx, id)
}
}