Files
llmgw/internal/control/reconciler/preempt.go
T
Ilker ManapandClaude Opus 5.5 0e73f37128
Release / build (push) Successful in 1m4s
CI / test (push) Successful in 4m32s
Release / notes (push) Successful in 3s
Stop only the replicas that make the room
Candidates were added in order until the model fitted, and every one added
was stopped, including those tried first that freed room on the wrong node.
A set that works is now pruned to the replicas it needs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-30 18:21:58 +02:00

305 lines
8.9 KiB
Go

package reconciler
import (
"context"
"slices"
"sort"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
)
// hold keeps a model that gave up room from taking it back.
type hold struct {
Until time.Time
For string
}
// maxVictims bounds how many replicas one placement may stop. Past this the
// model is better reported as not fitting than paid for with half a cluster.
const maxVictims = 3
// victim is a replica that could be stopped for another model.
type victim struct {
ID string
Model string
Class string
AboveMin bool
Tier int
Idle time.Duration
}
// preemptFor stops replicas of lower classes when a model cannot otherwise be
// placed, and says whether it did.
//
// What may be taken from whom:
//
// - A lower class gives way to a higher one: opportunistic to standard,
// standard to critical.
// - Within a class, only a replica above its model's minimum, and only for
// a model still below its own. Room a model grew into under load is not a
// promise; its minimum is.
// - A critical model's minimum is never taken.
//
// Victims only ever come from a lower rank, or from above a minimum for a
// model below one, so two models cannot take turns stopping each other.
//
// The set is tried on a copy of the inventory first, and only stopped if it
// would let the model be placed: stopping replicas that do not make room is
// the one outcome worse than not preempting.
func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []string) bool {
cands := r.victims(m)
if len(cands) == 0 {
return false
}
// Candidates in order until the model fits; then every one whose
// stopping turns out not to be needed is kept after all. Without that
// second step an idle replica on the wrong node, tried first, would be
// stopped along with the one that actually made the room.
var chosen []victim
for _, v := range cands {
if len(chosen) == maxVictims {
break
}
chosen = append(chosen, v)
if !r.fitsWithout(m, avoid, chosen) {
continue
}
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
if r.fitsWithout(m, avoid, fewer) {
chosen = fewer
}
}
r.preempt(ctx, m, chosen)
return true
}
return false
}
// fitsWithout reports whether m could be placed with these replicas gone,
// tried on a copy of the inventory.
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
trial := r.inv.Clone()
for _, v := range gone {
trial.ReleaseReplica(v.ID)
}
sched := placement.New(trial)
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
res = sched.Place(r.orderedProfiles(m), nil)
}
return res.Placement != nil
}
// preempt drains the chosen replicas and holds their models back.
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
now := r.now()
var ids []string
r.mu.Lock()
for _, v := range chosen {
if rep, ok := r.replicas[v.ID]; ok {
rep.PreemptedFor = m.Name
}
ids = append(ids, v.ID)
r.held[v.Model] = hold{Until: now.Add(r.cfg.PreemptHold), For: m.Name}
r.preempted[v.Model]++
}
r.preempting[m.Name] = ids
r.mu.Unlock()
for _, v := range chosen {
r.log.Warn("preempting a replica to make room",
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
"for", m.Name, "for_class", m.EffectiveClass())
}
r.scaleDown(ctx, ids)
}
// awaitingPreemption reports whether replicas stopped for a model are still
// on their way out, so it waits for them rather than stopping more.
func (r *Reconciler) awaitingPreemption(model string) bool {
r.mu.Lock()
defer r.mu.Unlock()
ids := r.preempting[model]
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool {
_, ok := r.replicas[id]
return !ok
})
if len(ids) == 0 {
delete(r.preempting, model)
return false
}
r.preempting[model] = ids
return true
}
// isHeld reports whether a model gave up room and may not take it back yet.
// The hold ends early once the model it was given to has what it asked for.
//
// Today a preempted replica is only removed by its own model's drain loop,
// which runs after that model has decided not to scale up, so the room it
// frees reaches the model it was freed for first anyway. The hold is what
// keeps that true for any other way a replica can go — a stop reported
// between passes, a reap at the start of one — rather than it resting on the
// order of lines in reconcileModel.
func (r *Reconciler) isHeld(model string) bool {
r.mu.Lock()
h, ok := r.held[model]
r.mu.Unlock()
if !ok {
return false
}
if r.now().After(h.Until) {
r.mu.Lock()
delete(r.held, model)
r.mu.Unlock()
return false
}
if to, ok := r.cat.Load().Resolve(h.For); ok && r.liveCount(h.For) >= r.desiredFor(to) {
r.mu.Lock()
delete(r.held, model)
r.mu.Unlock()
return false
}
return true
}
// liveCount is how many replicas of a model hold capacity.
func (r *Reconciler) liveCount(model string) int {
r.mu.Lock()
defer r.mu.Unlock()
n := 0
for _, rep := range r.replicas {
if rep.Model == model && rep.State.Counts() && rep.RetiringAt.IsZero() {
n++
}
}
return n
}
// victims lists the replicas that may be stopped for m, in the order they
// should be: idle on-demand ones first, then what other tenants borrowed past
// their guarantees when m's owner is within its own, then what lower classes
// grew into under load, then the same within m's class, then lower classes'
// minimums; within each, the longest unused first.
//
// A replica within its owner's guarantee is taken only for a model of the
// same owner: a guarantee that another tenant's higher class could take is
// not one.
func (r *Reconciler) victims(m catalog.Model) []victim {
cat := r.cat.Load()
now := r.now()
myRank := catalog.ClassRank(m.EffectiveClass())
belowMin := r.liveCount(m.Name) < m.Scaling.Min
pol := r.policy()
var borrowed map[string]bool
reclaim := false
if pol != nil {
hs := r.Holdings()
borrowed = pol.Borrowed(hs)
reclaim = pol.Guaranteed(m.Owner, pol.Usage(hs), profileNeed(m))
}
r.mu.Lock()
defer r.mu.Unlock()
// Per model, its live replicas newest first: the newest are the ones
// above its minimum.
byModel := map[string][]*Replica{}
for _, rep := range r.replicas {
if rep.Model == m.Name || !rep.State.Counts() || !rep.RetiringAt.IsZero() || rep.PreemptedFor != "" {
continue
}
byModel[rep.Model] = append(byModel[rep.Model], rep)
}
var out []victim
for model, reps := range byModel {
other, ok := cat.Resolve(model)
if !ok {
continue
}
class := other.EffectiveClass()
rank := catalog.ClassRank(class)
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
above := max(len(reps)-other.Scaling.Min, 0)
for i, rep := range reps {
aboveMin := i < above
tier := -1
switch {
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace:
tier = 0
case rank < myRank && aboveMin:
tier = 2
case rank == myRank && aboveMin && belowMin:
tier = 3
case rank < myRank && class != catalog.ClassCritical:
tier = 4
}
if pol != nil && other.Owner != m.Owner {
switch {
case borrowed[rep.ID] && reclaim && (class != catalog.ClassCritical || aboveMin):
// Past its owner's guarantee, wanted by an owner within
// its own: taken back whatever the classes.
if tier != 0 {
tier = 1
}
case !borrowed[rep.ID] && guaranteed(pol, other.Owner):
continue // held within its owner's guarantee
}
}
if tier < 0 {
continue
}
// Just started: stopping it throws away a model load that has
// only now been paid for.
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
continue
}
if len(nodesOf(rep)) > 1 {
continue // see makeRoomFor: a multi-node replica is its own decision
}
out = append(out, victim{ID: rep.ID, Model: model, Class: class,
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now)})
}
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Tier != out[j].Tier {
return out[i].Tier < out[j].Tier
}
return out[i].Idle > out[j].Idle
})
return out
}
// idleFor is how long a replica has gone unused; zero when nobody has said,
// which is not the same as idle.
func idleFor(rep *Replica, now time.Time) time.Duration {
if rep.LastUsedAt.IsZero() {
return 0
}
return now.Sub(rep.LastUsedAt)
}
func nodesOf(rep *Replica) map[string]struct{} {
out := map[string]struct{}{}
for _, sh := range rep.Shards {
out[sh.NodeID] = struct{}{}
}
return out
}
// guaranteed reports whether an owner has a guarantee anywhere up its tree.
func guaranteed(pol *quota.Policy, owner string) bool {
for _, q := range pol.Governing(owner) {
if q.Guarantee.Any() {
return true
}
}
return false
}