Files
llmgw/internal/control/reconciler/preempt.go
T
Ilker ManapandClaude Opus 5.5 e725adc2f0 Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it
went, or every candidate turned down), scale-up/down, drain, stop, eviction,
preemption, reaping and restarts beneath it. A replica's start is a span open
from placement until it serves; the start command carries its context in a
new ControlMessage.trace field, and the agent continues the trace through
the weights fetch, engine launch, rendezvous and readiness wait. Stops,
drains, adoption and engine installs are traced on the node, model jobs are
a trace with a span per step linked to the replica's start, and agent.yaml
takes the same tracing block as server.yaml. Off, every span is the no-op one
and commands carry nothing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 00:03:17 +02:00

340 lines
10 KiB
Go

package reconciler
import (
"context"
"fmt"
"slices"
"sort"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
)
// hold keeps a model that gave up room from taking it back.
type hold struct {
Until time.Time
For string
}
// maxVictims bounds how many replicas one placement may stop. Past this the
// model is better reported as not fitting than paid for with half a cluster.
const maxVictims = 3
// victim is a replica that could be stopped for another model.
type victim struct {
ID string
Model string
Class string
AboveMin bool
Tier int
Idle time.Duration
// Size is what stopping it frees, for choosing the smaller of two
// otherwise equal victims.
Size quota.Amount
}
// preemptFor stops replicas of lower classes when a model cannot otherwise be
// placed, and says whether it did.
//
// What may be taken from whom:
//
// - A lower class gives way to a higher one: opportunistic to standard,
// standard to critical.
// - Within a class, only a replica above its model's minimum, and only for
// a model still below its own. Room a model grew into under load is not a
// promise; its minimum is.
// - A critical model's minimum is never taken.
//
// Victims only ever come from a lower rank, or from above a minimum for a
// model below one, so two models cannot take turns stopping each other.
//
// The set is tried on a copy of the inventory first, and only stopped if it
// would let the model be placed: stopping replicas that do not make room is
// the one outcome worse than not preempting.
func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []string) bool {
cands := r.victims(m)
if len(cands) == 0 {
return false
}
// Candidates in order until the model fits; then every one whose
// stopping turns out not to be needed is kept after all. Without that
// second step an idle replica on the wrong node, tried first, would be
// stopped along with the one that actually made the room.
var chosen []victim
for _, v := range cands {
if len(chosen) == maxVictims {
break
}
chosen = append(chosen, v)
if !r.fitsWithout(m, avoid, chosen) {
continue
}
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
if r.fitsWithout(m, avoid, fewer) {
chosen = fewer
}
}
r.preempt(ctx, m, chosen)
return true
}
return false
}
// fitsWithout reports whether m could be placed with these replicas gone,
// tried on a copy of the inventory.
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
trial := r.inv.Clone()
for _, v := range gone {
trial.ReleaseReplica(v.ID)
}
sched := placement.New(trial)
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
res = sched.Place(r.orderedProfiles(m), nil)
}
return res.Placement != nil
}
// preempt drains the chosen replicas and holds their models back.
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
now := r.now()
var ids []string
r.mu.Lock()
for _, v := range chosen {
if rep, ok := r.replicas[v.ID]; ok {
rep.PreemptedFor = m.Name
}
ids = append(ids, v.ID)
r.held[v.Model] = hold{Until: now.Add(r.cfg.PreemptHold), For: m.Name}
r.preempted[v.Model]++
}
r.preempting[m.Name] = ids
r.mu.Unlock()
victims := make([]string, 0, len(chosen))
for _, v := range chosen {
r.log.Warn("preempting a replica to make room",
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
"for", m.Name, "for_class", m.EffectiveClass())
desc := fmt.Sprintf("%s (%s, %s)", v.ID, v.Model, v.Class)
if v.AboveMin {
desc = fmt.Sprintf("%s (%s, %s, above its minimum)", v.ID, v.Model, v.Class)
}
victims = append(victims, desc)
}
ctx, span := startSpan(ctx, "preempt",
attribute.String("llmgw.for", m.Name),
attribute.String("llmgw.for_class", m.EffectiveClass()),
attribute.StringSlice("llmgw.victims", victims),
attribute.String("llmgw.hold", r.cfg.PreemptHold.String()))
defer span.End()
r.scaleDown(ctx, ids, "preempted for "+m.Name)
}
// awaitingPreemption reports whether replicas stopped for a model are still
// on their way out, so it waits for them rather than stopping more.
func (r *Reconciler) awaitingPreemption(model string) bool {
r.mu.Lock()
defer r.mu.Unlock()
ids := r.preempting[model]
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool {
_, ok := r.replicas[id]
return !ok
})
if len(ids) == 0 {
delete(r.preempting, model)
return false
}
r.preempting[model] = ids
return true
}
// isHeld reports whether a model gave up room and may not take it back yet.
// The hold ends early once the model it was given to has what it asked for.
//
// Today a preempted replica is only removed by its own model's drain loop,
// which runs after that model has decided not to scale up, so the room it
// frees reaches the model it was freed for first anyway. The hold is what
// keeps that true for any other way a replica can go — a stop reported
// between passes, a reap at the start of one — rather than it resting on the
// order of lines in reconcileModel.
func (r *Reconciler) isHeld(model string) bool {
r.mu.Lock()
h, ok := r.held[model]
r.mu.Unlock()
if !ok {
return false
}
if r.now().After(h.Until) {
r.mu.Lock()
delete(r.held, model)
r.mu.Unlock()
return false
}
if to, ok := r.cat.Load().Resolve(h.For); ok && r.liveCount(h.For) >= r.desiredFor(to) {
r.mu.Lock()
delete(r.held, model)
r.mu.Unlock()
return false
}
return true
}
// liveCount is how many replicas of a model hold capacity.
func (r *Reconciler) liveCount(model string) int {
r.mu.Lock()
defer r.mu.Unlock()
n := 0
for _, rep := range r.replicas {
if rep.Model == model && rep.State.Counts() && rep.RetiringAt.IsZero() {
n++
}
}
return n
}
// victims lists the replicas that may be stopped for m, in the order they
// should be: idle on-demand ones first, then what other tenants borrowed past
// their guarantees when m's owner is within its own, then what lower classes
// grew into under load, then the same within m's class, then lower classes'
// minimums; within each, the longest unused first.
//
// A replica within its owner's guarantee is taken only for a model of the
// same owner: a guarantee that another tenant's higher class could take is
// not one.
func (r *Reconciler) victims(m catalog.Model) []victim {
cat := r.cat.Load()
now := r.now()
myRank := catalog.ClassRank(m.EffectiveClass())
belowMin := r.liveCount(m.Name) < m.Scaling.Min
pol := r.policy()
var borrowed map[string]bool
reclaim := false
if pol != nil {
hs := r.Holdings()
borrowed = pol.Borrowed(hs)
reclaim = pol.Guaranteed(m.Owner, pol.Usage(hs), profileNeed(m))
}
r.mu.Lock()
defer r.mu.Unlock()
// Per model, its live replicas newest first: the newest are the ones
// above its minimum.
byModel := map[string][]*Replica{}
for _, rep := range r.replicas {
if rep.Model == m.Name || !rep.State.Counts() || !rep.RetiringAt.IsZero() || rep.PreemptedFor != "" {
continue
}
byModel[rep.Model] = append(byModel[rep.Model], rep)
}
var out []victim
for model, reps := range byModel {
other, ok := cat.Resolve(model)
if !ok {
continue
}
class := other.EffectiveClass()
rank := catalog.ClassRank(class)
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
above := max(len(reps)-other.Scaling.Min, 0)
for i, rep := range reps {
aboveMin := i < above
tier := -1
switch {
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace:
tier = 0
case rank < myRank && aboveMin:
tier = 2
case rank == myRank && aboveMin && belowMin:
tier = 3
case rank < myRank && class != catalog.ClassCritical:
tier = 4
}
if pol != nil && other.Owner != m.Owner {
switch {
case borrowed[rep.ID] && reclaim && (class != catalog.ClassCritical || aboveMin):
// Past its owner's guarantee, wanted by an owner within
// its own: taken back whatever the classes.
if tier != 0 {
tier = 1
}
case !borrowed[rep.ID] && guaranteed(pol, other.Owner):
continue // held within its owner's guarantee
}
}
if tier < 0 {
continue
}
// Just started: stopping it throws away a model load that has
// only now been paid for.
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
continue
}
if len(nodesOf(rep)) > 1 {
continue // see makeRoomFor: a multi-node replica is its own decision
}
prof, _ := other.ProfileByName(rep.Profile)
var size quota.Amount
for _, sh := range rep.Shards {
size = size.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
}
out = append(out, victim{ID: rep.ID, Model: model, Class: class,
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now), Size: size})
}
}
sort.SliceStable(out, func(i, j int) bool {
a, b := out[i], out[j]
if a.Tier != b.Tier {
return a.Tier < b.Tier
}
if a.Idle != b.Idle {
return a.Idle > b.Idle
}
// The smaller first: the least served is lost if it is enough, and
// the pruning after keeps a larger one only when it is needed.
if a.Size.GPUs != b.Size.GPUs {
return a.Size.GPUs < b.Size.GPUs
}
if a.Size.MemoryGB != b.Size.MemoryGB {
return a.Size.MemoryGB < b.Size.MemoryGB
}
return a.ID < b.ID
})
return out
}
// idleFor is how long a replica has gone unused; zero when nobody has said,
// which is not the same as idle.
func idleFor(rep *Replica, now time.Time) time.Duration {
if rep.LastUsedAt.IsZero() {
return 0
}
return now.Sub(rep.LastUsedAt)
}
func nodesOf(rep *Replica) map[string]struct{} {
out := map[string]struct{}{}
for _, sh := range rep.Shards {
out[sh.NodeID] = struct{}{}
}
return out
}
// guaranteed reports whether an owner has a guarantee anywhere up its tree.
func guaranteed(pol *quota.Policy, owner string) bool {
for _, q := range pol.Governing(owner) {
if q.Guarantee.Any() {
return true
}
}
return false
}