Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
149 lines
4.8 KiB
Go
149 lines
4.8 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"slices"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
|
)
|
|
|
|
// Headroom is how many more replicas of a model the cluster has room for now.
|
|
type Headroom struct {
|
|
// Current is the replicas the model has that hold capacity: starting or
|
|
// ready, not those on their way out.
|
|
Current int
|
|
// More is how many more could be placed now, at most the limit asked.
|
|
More int
|
|
// Nodes is where each of those would go, in order, a replica's nodes
|
|
// joined when it spans several.
|
|
Nodes [][]string
|
|
// Limited is set when More reached the limit asked, so there may be room
|
|
// for more than that. A CPU profile that declares no memory is always
|
|
// limited: nothing it asks for runs out.
|
|
Limited bool
|
|
// Blocked is why one more than More does not fit, a line per node that
|
|
// was considered. Empty when Limited.
|
|
Blocked []string
|
|
// TokPerSec is what one more replica was measured to serve, on the
|
|
// profile placement would give it; zero when it was never measured.
|
|
TokPerSec float64
|
|
// ByPreempting is how many more, beyond More, would fit by stopping
|
|
// replicas the model's class may take, and Preempts the models those
|
|
// replicas belong to: what "room" means for a model that outranks others.
|
|
ByPreempting int
|
|
Preempts []string
|
|
}
|
|
|
|
// Headroom tries placing up to limit more replicas of a model on a copy of
|
|
// the inventory, each one's allocation made on the copy before the next is
|
|
// tried, exactly as the reconciler would place them one pass at a time.
|
|
//
|
|
// The scheduler's own decisions, not an estimate beside them: what the page
|
|
// offers is then what the next passes will do, and a count that the
|
|
// arithmetic of a second implementation allowed but placement refuses would
|
|
// leave a model short with nobody told why.
|
|
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return Headroom{}, false
|
|
}
|
|
var h Headroom
|
|
var avoid []string
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
|
|
h.Current++
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
inv := r.inv.Clone()
|
|
sched := placement.New(inv)
|
|
pol := r.policy()
|
|
var usage map[string]quota.Amount
|
|
if pol != nil {
|
|
usage = pol.Usage(r.Holdings())
|
|
}
|
|
place := func() placement.Result {
|
|
res := sched.Place(r.orderedProfiles(m), avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
// As scaleUp does: spreading is a preference, not a rule.
|
|
res = sched.Place(r.orderedProfiles(m), nil)
|
|
}
|
|
return res
|
|
}
|
|
// What the model's class may take, in the order preemption would, each
|
|
// released on the copy only when free room has run out.
|
|
victims, nextVictim, preempting := r.victims(m), 0, false
|
|
for h.More+h.ByPreempting < limit {
|
|
res := place()
|
|
// Named only once they have made room: candidates tried on the way
|
|
// to finding there is none would otherwise read as whose replicas a
|
|
// placement would stop.
|
|
var tried []string
|
|
for res.Placement == nil && nextVictim < len(victims) {
|
|
v := victims[nextVictim]
|
|
nextVictim++
|
|
inv.ReleaseReplica(v.ID)
|
|
tried = append(tried, v.Model)
|
|
res = place()
|
|
}
|
|
if res.Placement != nil && len(tried) > 0 {
|
|
preempting = true
|
|
for _, m := range tried {
|
|
if !slices.Contains(h.Preempts, m) {
|
|
h.Preempts = append(h.Preempts, m)
|
|
}
|
|
}
|
|
}
|
|
if res.Placement == nil {
|
|
for _, rej := range res.Rejections {
|
|
h.Blocked = append(h.Blocked, rej.String())
|
|
}
|
|
slices.Sort(h.Blocked)
|
|
h.Blocked = slices.Compact(h.Blocked)
|
|
return h, true
|
|
}
|
|
// A quota limit is as much an edge as a full node.
|
|
if pol != nil && m.Owner != "" {
|
|
need := placementNeed(m, res.Placement)
|
|
if why := pol.AtLimit(m.Owner, usage, need); why != "" {
|
|
h.Blocked = []string{why}
|
|
return h, true
|
|
}
|
|
for _, s := range pol.Chain(m.Owner) {
|
|
usage[s] = usage[s].Plus(need)
|
|
}
|
|
}
|
|
mem, cores := 0, 0
|
|
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
|
|
mem, cores = prof.MinMemoryGB, cpuCores(prof)
|
|
}
|
|
if h.More == 0 {
|
|
if meas, ok := r.measured(m.Name, res.Placement.Profile.Name); ok {
|
|
h.TokPerSec = meas.TokPerSec
|
|
}
|
|
}
|
|
var nodes []string
|
|
for _, a := range res.Placement.Nodes {
|
|
inv.Allocate(a.NodeID, inventory.Allocation{
|
|
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, PowerW: a.PowerW,
|
|
})
|
|
nodes = append(nodes, a.NodeID)
|
|
}
|
|
avoid = append(avoid, nodes...)
|
|
// Once anything was stopped, what fits after is counted as fitting
|
|
// by preempting: it may be sitting in the room that was freed.
|
|
if preempting {
|
|
h.ByPreempting++
|
|
} else {
|
|
h.Nodes = append(h.Nodes, nodes)
|
|
h.More++
|
|
}
|
|
}
|
|
h.Limited = true
|
|
return h, true
|
|
}
|