The Held tab's model name opens a card: each replica with its node, state, age, CPU and memory, the room for more found by dry-running the scheduler on a copy of the inventory, and a form for the always-running and under-load replica counts. An always-running count placement cannot meet is refused with the reason. Headroom is also on the admin API. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
87 lines
2.7 KiB
Go
87 lines
2.7 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"slices"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
|
)
|
|
|
|
// Headroom is how many more replicas of a model the cluster has room for now.
|
|
type Headroom struct {
|
|
// Current is the replicas the model has that hold capacity: starting or
|
|
// ready, not those on their way out.
|
|
Current int
|
|
// More is how many more could be placed now, at most the limit asked.
|
|
More int
|
|
// Nodes is where each of those would go, in order, a replica's nodes
|
|
// joined when it spans several.
|
|
Nodes [][]string
|
|
// Limited is set when More reached the limit asked, so there may be room
|
|
// for more than that. A CPU profile that declares no memory is always
|
|
// limited: nothing it asks for runs out.
|
|
Limited bool
|
|
// Blocked is why one more than More does not fit, a line per node that
|
|
// was considered. Empty when Limited.
|
|
Blocked []string
|
|
}
|
|
|
|
// Headroom tries placing up to limit more replicas of a model on a copy of
|
|
// the inventory, each one's allocation made on the copy before the next is
|
|
// tried, exactly as the reconciler would place them one pass at a time.
|
|
//
|
|
// The scheduler's own decisions, not an estimate beside them: what the page
|
|
// offers is then what the next passes will do, and a count that the
|
|
// arithmetic of a second implementation allowed but placement refuses would
|
|
// leave a model short with nobody told why.
|
|
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
|
m, ok := r.cat.Load().Resolve(model)
|
|
if !ok {
|
|
return Headroom{}, false
|
|
}
|
|
var h Headroom
|
|
var avoid []string
|
|
r.mu.Lock()
|
|
for _, rep := range r.replicas {
|
|
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
|
|
h.Current++
|
|
avoid = append(avoid, rep.NodeIDs()...)
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
|
|
inv := r.inv.Clone()
|
|
sched := placement.New(inv)
|
|
for h.More < limit {
|
|
res := sched.Place(m.PlacementProfiles(), avoid)
|
|
if res.Placement == nil && len(avoid) > 0 {
|
|
// As scaleUp does: spreading is a preference, not a rule.
|
|
res = sched.Place(m.PlacementProfiles(), nil)
|
|
}
|
|
if res.Placement == nil {
|
|
for _, rej := range res.Rejections {
|
|
h.Blocked = append(h.Blocked, rej.String())
|
|
}
|
|
slices.Sort(h.Blocked)
|
|
h.Blocked = slices.Compact(h.Blocked)
|
|
return h, true
|
|
}
|
|
mem := 0
|
|
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
|
|
mem = prof.MinMemoryGB
|
|
}
|
|
var nodes []string
|
|
for _, a := range res.Placement.Nodes {
|
|
inv.Allocate(a.NodeID, inventory.Allocation{
|
|
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem,
|
|
})
|
|
nodes = append(nodes, a.NodeID)
|
|
}
|
|
avoid = append(avoid, nodes...)
|
|
h.Nodes = append(h.Nodes, nodes)
|
|
h.More++
|
|
}
|
|
h.Limited = true
|
|
return h, true
|
|
}
|