A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
340 lines
10 KiB
Go
340 lines
10 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
)
|
|
|
|
// hold keeps a model that gave up room from taking it back.
|
|
type hold struct {
|
|
Until time.Time
|
|
For string
|
|
}
|
|
|
|
// maxVictims bounds how many replicas one placement may stop. Past this the
|
|
// model is better reported as not fitting than paid for with half a cluster.
|
|
const maxVictims = 3
|
|
|
|
// victim is a replica that could be stopped for another model.
|
|
type victim struct {
|
|
ID string
|
|
Model string
|
|
Class string
|
|
AboveMin bool
|
|
Tier int
|
|
Idle time.Duration
|
|
// Size is what stopping it frees, for choosing the smaller of two
|
|
// otherwise equal victims.
|
|
Size quota.Amount
|
|
}
|
|
|
|
// preemptFor stops replicas of lower classes when a model cannot otherwise be
|
|
// placed, and says whether it did.
|
|
//
|
|
// What may be taken from whom:
|
|
//
|
|
// - A lower class gives way to a higher one: opportunistic to standard,
|
|
// standard to critical.
|
|
// - Within a class, only a replica above its model's minimum, and only for
|
|
// a model still below its own. Room a model grew into under load is not a
|
|
// promise; its minimum is.
|
|
// - A critical model's minimum is never taken.
|
|
//
|
|
// Victims only ever come from a lower rank, or from above a minimum for a
|
|
// model below one, so two models cannot take turns stopping each other.
|
|
//
|
|
// The set is tried on a copy of the inventory first, and only stopped if it
|
|
// would let the model be placed: stopping replicas that do not make room is
|
|
// the one outcome worse than not preempting.
|
|
func (r *Reconciler) preemptFor(ctx context.Context, m catalog.Model, avoid []string) bool {
|
|
cands := r.victims(m)
|
|
if len(cands) == 0 {
|
|
return false
|
|
}
|
|
// Candidates in order until the model fits; then every one whose
|
|
// stopping turns out not to be needed is kept after all. Without that
|
|
// second step an idle replica on the wrong node, tried first, would be
|
|
// stopped along with the one that actually made the room.
|
|
var chosen []victim
|
|
for _, v := range cands {
|
|
if len(chosen) == maxVictims {
|
|
break
|
|
}
|
|
chosen = append(chosen, v)
|
|
if !r.fitsWithout(m, avoid, chosen) {
|
|
continue
|
|
}
|
|
for i := len(chosen) - 1; i >= 0 && len(chosen) > 1; i-- {
|
|
fewer := append(append([]victim(nil), chosen[:i]...), chosen[i+1:]...)
|
|
if r.fitsWithout(m, avoid, fewer) {
|
|
chosen = fewer
|
|
}
|
|
}
|
|
r.preempt(ctx, m, chosen)
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// fitsWithout reports whether m could be placed with these replicas gone,
|
|
// tried on a copy of the inventory.
|
|
func (r *Reconciler) fitsWithout(m catalog.Model, avoid []string, gone []victim) bool {
|
|
trial := r.inv.Clone()
|
|
for _, v := range gone {
|
|
trial.ReleaseReplica(v.ID)
|
|
}
|
|
sched := placement.New(trial)
|
|
res := sched.Place(r.orderedProfiles(m), avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
res = sched.Place(r.orderedProfiles(m), nil)
|
|
}
|
|
return res.Placement != nil
|
|
}
|
|
|
|
// preempt drains the chosen replicas and holds their models back.
|
|
func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []victim) {
|
|
now := r.now()
|
|
var ids []string
|
|
r.mu.Lock()
|
|
for _, v := range chosen {
|
|
if rep, ok := r.replicas[v.ID]; ok {
|
|
rep.PreemptedFor = m.Name
|
|
}
|
|
ids = append(ids, v.ID)
|
|
r.held[v.Model] = hold{Until: now.Add(r.cfg.PreemptHold), For: m.Name}
|
|
r.preempted[v.Model]++
|
|
}
|
|
r.preempting[m.Name] = ids
|
|
r.mu.Unlock()
|
|
|
|
victims := make([]string, 0, len(chosen))
|
|
for _, v := range chosen {
|
|
r.log.Warn("preempting a replica to make room",
|
|
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
|
|
"for", m.Name, "for_class", m.EffectiveClass())
|
|
desc := fmt.Sprintf("%s (%s, %s)", v.ID, v.Model, v.Class)
|
|
if v.AboveMin {
|
|
desc = fmt.Sprintf("%s (%s, %s, above its minimum)", v.ID, v.Model, v.Class)
|
|
}
|
|
victims = append(victims, desc)
|
|
}
|
|
ctx, span := startSpan(ctx, "preempt",
|
|
attribute.String("llmgw.for", m.Name),
|
|
attribute.String("llmgw.for_class", m.EffectiveClass()),
|
|
attribute.StringSlice("llmgw.victims", victims),
|
|
attribute.String("llmgw.hold", r.cfg.PreemptHold.String()))
|
|
defer span.End()
|
|
r.scaleDown(ctx, ids, "preempted for "+m.Name)
|
|
}
|
|
|
|
// awaitingPreemption reports whether replicas stopped for a model are still
|
|
// on their way out, so it waits for them rather than stopping more.
|
|
func (r *Reconciler) awaitingPreemption(model string) bool {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
ids := r.preempting[model]
|
|
ids = slices.DeleteFunc(slices.Clone(ids), func(id string) bool {
|
|
_, ok := r.replicas[id]
|
|
return !ok
|
|
})
|
|
if len(ids) == 0 {
|
|
delete(r.preempting, model)
|
|
return false
|
|
}
|
|
r.preempting[model] = ids
|
|
return true
|
|
}
|
|
|
|
// isHeld reports whether a model gave up room and may not take it back yet.
|
|
// The hold ends early once the model it was given to has what it asked for.
|
|
//
|
|
// Today a preempted replica is only removed by its own model's drain loop,
|
|
// which runs after that model has decided not to scale up, so the room it
|
|
// frees reaches the model it was freed for first anyway. The hold is what
|
|
// keeps that true for any other way a replica can go — a stop reported
|
|
// between passes, a reap at the start of one — rather than it resting on the
|
|
// order of lines in reconcileModel.
|
|
func (r *Reconciler) isHeld(model string) bool {
|
|
r.mu.Lock()
|
|
h, ok := r.held[model]
|
|
r.mu.Unlock()
|
|
if !ok {
|
|
return false
|
|
}
|
|
if r.now().After(h.Until) {
|
|
r.mu.Lock()
|
|
delete(r.held, model)
|
|
r.mu.Unlock()
|
|
return false
|
|
}
|
|
if to, ok := r.cat.Load().Resolve(h.For); ok && r.liveCount(h.For) >= r.desiredFor(to) {
|
|
r.mu.Lock()
|
|
delete(r.held, model)
|
|
r.mu.Unlock()
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// liveCount is how many replicas of a model hold capacity.
|
|
func (r *Reconciler) liveCount(model string) int {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
n := 0
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == model && rep.State.Counts() && rep.RetiringAt.IsZero() {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// victims lists the replicas that may be stopped for m, in the order they
|
|
// should be: idle on-demand ones first, then what other tenants borrowed past
|
|
// their guarantees when m's owner is within its own, then what lower classes
|
|
// grew into under load, then the same within m's class, then lower classes'
|
|
// minimums; within each, the longest unused first.
|
|
//
|
|
// A replica within its owner's guarantee is taken only for a model of the
|
|
// same owner: a guarantee that another tenant's higher class could take is
|
|
// not one.
|
|
func (r *Reconciler) victims(m catalog.Model) []victim {
|
|
cat := r.cat.Load()
|
|
now := r.now()
|
|
myRank := catalog.ClassRank(m.EffectiveClass())
|
|
belowMin := r.liveCount(m.Name) < m.Scaling.Min
|
|
|
|
pol := r.policy()
|
|
var borrowed map[string]bool
|
|
reclaim := false
|
|
if pol != nil {
|
|
hs := r.Holdings()
|
|
borrowed = pol.Borrowed(hs)
|
|
reclaim = pol.Guaranteed(m.Owner, pol.Usage(hs), profileNeed(m))
|
|
}
|
|
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
// Per model, its live replicas newest first: the newest are the ones
|
|
// above its minimum.
|
|
byModel := map[string][]*Replica{}
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == m.Name || !rep.State.Counts() || !rep.RetiringAt.IsZero() || rep.PreemptedFor != "" {
|
|
continue
|
|
}
|
|
byModel[rep.Model] = append(byModel[rep.Model], rep)
|
|
}
|
|
|
|
var out []victim
|
|
for model, reps := range byModel {
|
|
other, ok := cat.Resolve(model)
|
|
if !ok {
|
|
continue
|
|
}
|
|
class := other.EffectiveClass()
|
|
rank := catalog.ClassRank(class)
|
|
sort.Slice(reps, func(i, j int) bool { return reps[i].CreatedAt.After(reps[j].CreatedAt) })
|
|
above := max(len(reps)-other.Scaling.Min, 0)
|
|
for i, rep := range reps {
|
|
aboveMin := i < above
|
|
tier := -1
|
|
switch {
|
|
case rank < myRank && other.Scaling.OnDemand() && idleFor(rep, now) >= r.cfg.EvictionGrace:
|
|
tier = 0
|
|
case rank < myRank && aboveMin:
|
|
tier = 2
|
|
case rank == myRank && aboveMin && belowMin:
|
|
tier = 3
|
|
case rank < myRank && class != catalog.ClassCritical:
|
|
tier = 4
|
|
}
|
|
if pol != nil && other.Owner != m.Owner {
|
|
switch {
|
|
case borrowed[rep.ID] && reclaim && (class != catalog.ClassCritical || aboveMin):
|
|
// Past its owner's guarantee, wanted by an owner within
|
|
// its own: taken back whatever the classes.
|
|
if tier != 0 {
|
|
tier = 1
|
|
}
|
|
case !borrowed[rep.ID] && guaranteed(pol, other.Owner):
|
|
continue // held within its owner's guarantee
|
|
}
|
|
}
|
|
if tier < 0 {
|
|
continue
|
|
}
|
|
// Just started: stopping it throws away a model load that has
|
|
// only now been paid for.
|
|
if !rep.ReadyAt.IsZero() && now.Sub(rep.ReadyAt) < r.cfg.MinLifetime {
|
|
continue
|
|
}
|
|
if len(nodesOf(rep)) > 1 {
|
|
continue // see makeRoomFor: a multi-node replica is its own decision
|
|
}
|
|
prof, _ := other.ProfileByName(rep.Profile)
|
|
var size quota.Amount
|
|
for _, sh := range rep.Shards {
|
|
size = size.Plus(quota.Amount{GPUs: len(sh.GPUIndices), CPUCores: cpuCores(prof), MemoryGB: prof.MinMemoryGB})
|
|
}
|
|
out = append(out, victim{ID: rep.ID, Model: model, Class: class,
|
|
AboveMin: aboveMin, Tier: tier, Idle: idleFor(rep, now), Size: size})
|
|
}
|
|
}
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
a, b := out[i], out[j]
|
|
if a.Tier != b.Tier {
|
|
return a.Tier < b.Tier
|
|
}
|
|
if a.Idle != b.Idle {
|
|
return a.Idle > b.Idle
|
|
}
|
|
// The smaller first: the least served is lost if it is enough, and
|
|
// the pruning after keeps a larger one only when it is needed.
|
|
if a.Size.GPUs != b.Size.GPUs {
|
|
return a.Size.GPUs < b.Size.GPUs
|
|
}
|
|
if a.Size.MemoryGB != b.Size.MemoryGB {
|
|
return a.Size.MemoryGB < b.Size.MemoryGB
|
|
}
|
|
return a.ID < b.ID
|
|
})
|
|
return out
|
|
}
|
|
|
|
// idleFor is how long a replica has gone unused; zero when nobody has said,
|
|
// which is not the same as idle.
|
|
func idleFor(rep *Replica, now time.Time) time.Duration {
|
|
if rep.LastUsedAt.IsZero() {
|
|
return 0
|
|
}
|
|
return now.Sub(rep.LastUsedAt)
|
|
}
|
|
|
|
func nodesOf(rep *Replica) map[string]struct{} {
|
|
out := map[string]struct{}{}
|
|
for _, sh := range rep.Shards {
|
|
out[sh.NodeID] = struct{}{}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// guaranteed reports whether an owner has a guarantee anywhere up its tree.
|
|
func guaranteed(pol *quota.Policy, owner string) bool {
|
|
for _, q := range pol.Governing(owner) {
|
|
if q.Guarantee.Any() {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|