Files
llmgw/internal/control/reconciler/headroom.go
T
Ilker ManapandClaude Opus 5.5 0d3eb82b9b Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 01:17:49 +02:00

149 lines
4.8 KiB
Go

package reconciler
import (
"slices"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
)
// Headroom is how many more replicas of a model the cluster has room for now.
type Headroom struct {
// Current is the replicas the model has that hold capacity: starting or
// ready, not those on their way out.
Current int
// More is how many more could be placed now, at most the limit asked.
More int
// Nodes is where each of those would go, in order, a replica's nodes
// joined when it spans several.
Nodes [][]string
// Limited is set when More reached the limit asked, so there may be room
// for more than that. A CPU profile that declares no memory is always
// limited: nothing it asks for runs out.
Limited bool
// Blocked is why one more than More does not fit, a line per node that
// was considered. Empty when Limited.
Blocked []string
// TokPerSec is what one more replica was measured to serve, on the
// profile placement would give it; zero when it was never measured.
TokPerSec float64
// ByPreempting is how many more, beyond More, would fit by stopping
// replicas the model's class may take, and Preempts the models those
// replicas belong to: what "room" means for a model that outranks others.
ByPreempting int
Preempts []string
}
// Headroom tries placing up to limit more replicas of a model on a copy of
// the inventory, each one's allocation made on the copy before the next is
// tried, exactly as the reconciler would place them one pass at a time.
//
// The scheduler's own decisions, not an estimate beside them: what the page
// offers is then what the next passes will do, and a count that the
// arithmetic of a second implementation allowed but placement refuses would
// leave a model short with nobody told why.
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
m, ok := r.cat.Load().Resolve(model)
if !ok {
return Headroom{}, false
}
var h Headroom
var avoid []string
r.mu.Lock()
for _, rep := range r.replicas {
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
h.Current++
avoid = append(avoid, rep.NodeIDs()...)
}
}
r.mu.Unlock()
inv := r.inv.Clone()
sched := placement.New(inv)
pol := r.policy()
var usage map[string]quota.Amount
if pol != nil {
usage = pol.Usage(r.Holdings())
}
place := func() placement.Result {
res := sched.Place(r.orderedProfiles(m), avoid)
if res.Placement == nil && len(avoid) > 0 {
// As scaleUp does: spreading is a preference, not a rule.
res = sched.Place(r.orderedProfiles(m), nil)
}
return res
}
// What the model's class may take, in the order preemption would, each
// released on the copy only when free room has run out.
victims, nextVictim, preempting := r.victims(m), 0, false
for h.More+h.ByPreempting < limit {
res := place()
// Named only once they have made room: candidates tried on the way
// to finding there is none would otherwise read as whose replicas a
// placement would stop.
var tried []string
for res.Placement == nil && nextVictim < len(victims) {
v := victims[nextVictim]
nextVictim++
inv.ReleaseReplica(v.ID)
tried = append(tried, v.Model)
res = place()
}
if res.Placement != nil && len(tried) > 0 {
preempting = true
for _, m := range tried {
if !slices.Contains(h.Preempts, m) {
h.Preempts = append(h.Preempts, m)
}
}
}
if res.Placement == nil {
for _, rej := range res.Rejections {
h.Blocked = append(h.Blocked, rej.String())
}
slices.Sort(h.Blocked)
h.Blocked = slices.Compact(h.Blocked)
return h, true
}
// A quota limit is as much an edge as a full node.
if pol != nil && m.Owner != "" {
need := placementNeed(m, res.Placement)
if why := pol.AtLimit(m.Owner, usage, need); why != "" {
h.Blocked = []string{why}
return h, true
}
for _, s := range pol.Chain(m.Owner) {
usage[s] = usage[s].Plus(need)
}
}
mem, cores := 0, 0
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
mem, cores = prof.MinMemoryGB, cpuCores(prof)
}
if h.More == 0 {
if meas, ok := r.measured(m.Name, res.Placement.Profile.Name); ok {
h.TokPerSec = meas.TokPerSec
}
}
var nodes []string
for _, a := range res.Placement.Nodes {
inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, PowerW: a.PowerW,
})
nodes = append(nodes, a.NodeID)
}
avoid = append(avoid, nodes...)
// Once anything was stopped, what fits after is counted as fitting
// by preempting: it may be sitting in the room that was freed.
if preempting {
h.ByPreempting++
} else {
h.Nodes = append(h.Nodes, nodes)
h.More++
}
}
h.Limited = true
return h, true
}