Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -0,0 +1,193 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// declaredShardWatts is the profile's declared draw for one node's shard.
|
||||
func (p Profile) declaredShardWatts() float64 {
|
||||
if p.PowerWatts <= 0 {
|
||||
return 0
|
||||
}
|
||||
return p.PowerWatts / float64(max(p.Nodes, 1))
|
||||
}
|
||||
|
||||
// shardWatts is what one shard of this profile is estimated to draw on a
|
||||
// candidate: the declared figure, or its cards at the highest power limit
|
||||
// among the node's free ones of the class (the cards are not chosen yet, and
|
||||
// guessing low is the mistake that trips a breaker), or its cores.
|
||||
func (s *Scheduler) shardWatts(p Profile, c candidate) float64 {
|
||||
if w := p.declaredShardWatts(); w > 0 {
|
||||
return w
|
||||
}
|
||||
pol := s.inv.Facility()
|
||||
if p.CPUOnly() {
|
||||
return float64(p.CPUCores) * pol.CPUCoreWatts
|
||||
}
|
||||
per := 0.0
|
||||
for _, g := range c.free {
|
||||
per = max(per, inventory.GPUWatts(c.node, g.Index, pol))
|
||||
}
|
||||
if per == 0 {
|
||||
per = pol.GPUWatts
|
||||
}
|
||||
return per * float64(p.GPUsPerNode)
|
||||
}
|
||||
|
||||
// budget is the power budgets as one placement decision spends them: the
|
||||
// domains' use when it began, and what the shards it has chosen so far add.
|
||||
type budget struct {
|
||||
pol facility.Policy
|
||||
use map[facility.Domain]inventory.PowerUse
|
||||
added map[facility.Domain]float64
|
||||
}
|
||||
|
||||
// newBudget reads the domains' use, once per decision. Without budgets there
|
||||
// is nothing to read and every check passes.
|
||||
func (s *Scheduler) newBudget() *budget {
|
||||
pol := s.inv.Facility()
|
||||
b := &budget{pol: pol, added: map[facility.Domain]float64{}}
|
||||
if len(pol.Budgets) > 0 {
|
||||
b.use = s.inv.PowerUse(s.clock())
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// check says why a shard drawing need watts on a node would take one of the
|
||||
// node's domains past its limit; empty when it fits them all. A shard whose
|
||||
// draw is not known (no declared figure, no card limit, no policy figure)
|
||||
// needs nothing: missing data never blocks placement.
|
||||
func (b *budget) check(n inventory.Node, need float64) string {
|
||||
if b.use == nil {
|
||||
return ""
|
||||
}
|
||||
for _, d := range n.Domains() {
|
||||
limit := b.pol.Limit(d)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
used := b.use[d].Used + b.added[d]
|
||||
if used+need > limit {
|
||||
capW := b.pol.Budgets[d].CapWatts
|
||||
of := facility.KW(limit) + " cap"
|
||||
if limit < capW {
|
||||
of = fmt.Sprintf("%s limit (%s cap less %.0f%%)", facility.KW(limit), facility.KW(capW), 100-100*limit/capW)
|
||||
}
|
||||
return fmt.Sprintf("%s: %s of a %s in use, the shard needs ~%s",
|
||||
d.Human(), facility.KW(used), of, facility.KW(need))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// add counts a chosen shard against the node's domains.
|
||||
func (b *budget) add(n inventory.Node, need float64) {
|
||||
for _, d := range n.Domains() {
|
||||
b.added[d] += need
|
||||
}
|
||||
}
|
||||
|
||||
// clock is the scheduler's time, for the freshness of power readings.
|
||||
func (s *Scheduler) clock() time.Time {
|
||||
if s.now != nil {
|
||||
return s.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// heldReason is a hold as a rejection says it.
|
||||
func heldReason(h facility.Hold) string {
|
||||
why := h.Domain.Human() + " is held"
|
||||
if h.Reason != "" {
|
||||
why += ": " + h.Reason
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// spreadLevels are the domains replicas of one model are kept apart in:
|
||||
// the rack, which a top-of-rack switch or a door left open takes out whole,
|
||||
// and the PDU, which a breaker does.
|
||||
var spreadLevels = []facility.Level{facility.Rack, facility.PDU}
|
||||
|
||||
// spreadAcrossDomains moves to the front the candidates sharing the fewest
|
||||
// racks and PDUs with the nodes the model already runs on, keeping the order
|
||||
// given among equals.
|
||||
//
|
||||
// A preference and never a filter, as spreading across nodes is: a rack
|
||||
// holding a model's only free cards still takes its second replica, which
|
||||
// survives an engine crash if not a breaker. Nodes with no location share
|
||||
// nothing and lose nothing, so a cluster without labels places as before.
|
||||
func (s *Scheduler) spreadAcrossDomains(candidates []candidate, avoid []string) {
|
||||
if len(avoid) == 0 {
|
||||
return
|
||||
}
|
||||
taken := map[facility.Domain]bool{}
|
||||
for _, id := range avoid {
|
||||
if n, ok := s.inv.Get(id); ok {
|
||||
for _, d := range n.Domains() {
|
||||
if slices.Contains(spreadLevels, d.Level()) {
|
||||
taken[d] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(taken) == 0 {
|
||||
return
|
||||
}
|
||||
shared := func(c candidate) int {
|
||||
k := 0
|
||||
for _, d := range c.node.Domains() {
|
||||
if taken[d] {
|
||||
k++
|
||||
}
|
||||
}
|
||||
return k
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool { return shared(candidates[i]) < shared(candidates[j]) })
|
||||
}
|
||||
|
||||
// withinGroups divides a multi-node replica's candidates by the domain its
|
||||
// profile keeps it within, largest first, and rejects the nodes that do not
|
||||
// say which they are in. Without the constraint it is all of them, as one.
|
||||
func withinGroups(p Profile, pool string, cands []candidate, rejections *[]Rejection) [][]candidate {
|
||||
if p.KeepWithin == "" {
|
||||
return [][]candidate{cands}
|
||||
}
|
||||
by := map[facility.Domain][]candidate{}
|
||||
for _, c := range cands {
|
||||
d := c.node.Location().At(p.KeepWithin)
|
||||
if d == "" {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: c.node.ID,
|
||||
Reason: fmt.Sprintf("has no %s label, and the profile keeps its nodes within one %s",
|
||||
p.KeepWithin.Label(), p.KeepWithin)})
|
||||
continue
|
||||
}
|
||||
by[d] = append(by[d], c)
|
||||
}
|
||||
keys := make([]facility.Domain, 0, len(by))
|
||||
for d := range by {
|
||||
keys = append(keys, d)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(by[keys[i]]) != len(by[keys[j]]) {
|
||||
return len(by[keys[i]]) > len(by[keys[j]])
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
out := make([][]candidate, 0, len(keys))
|
||||
for _, d := range keys {
|
||||
if len(by[d]) < p.Nodes {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool,
|
||||
Reason: fmt.Sprintf("needs %d nodes within one %s, %s has %d", p.Nodes, p.KeepWithin, d.Human(), len(by[d]))})
|
||||
continue
|
||||
}
|
||||
out = append(out, by[d])
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user