Facility-aware placement: racks, PDUs, holds and power caps

Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:17:49 +02:00
1 parent d41104a452
commit 0d3eb82b9b
42 files changed
+3510 -63

No files matched your search

+193
View File
@@ -0,0 +1,193 @@
package placement
import (
"fmt"
"slices"
"sort"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// declaredShardWatts is the profile's declared draw for one node's shard.
func (p Profile) declaredShardWatts() float64 {
if p.PowerWatts <= 0 {
return 0
}
return p.PowerWatts / float64(max(p.Nodes, 1))
}
// shardWatts is what one shard of this profile is estimated to draw on a
// candidate: the declared figure, or its cards at the highest power limit
// among the node's free ones of the class (the cards are not chosen yet, and
// guessing low is the mistake that trips a breaker), or its cores.
func (s *Scheduler) shardWatts(p Profile, c candidate) float64 {
if w := p.declaredShardWatts(); w > 0 {
return w
}
pol := s.inv.Facility()
if p.CPUOnly() {
return float64(p.CPUCores) * pol.CPUCoreWatts
}
per := 0.0
for _, g := range c.free {
per = max(per, inventory.GPUWatts(c.node, g.Index, pol))
}
if per == 0 {
per = pol.GPUWatts
}
return per * float64(p.GPUsPerNode)
}
// budget is the power budgets as one placement decision spends them: the
// domains' use when it began, and what the shards it has chosen so far add.
type budget struct {
pol facility.Policy
use map[facility.Domain]inventory.PowerUse
added map[facility.Domain]float64
}
// newBudget reads the domains' use, once per decision. Without budgets there
// is nothing to read and every check passes.
func (s *Scheduler) newBudget() *budget {
pol := s.inv.Facility()
b := &budget{pol: pol, added: map[facility.Domain]float64{}}
if len(pol.Budgets) > 0 {
b.use = s.inv.PowerUse(s.clock())
}
return b
}
// check says why a shard drawing need watts on a node would take one of the
// node's domains past its limit; empty when it fits them all. A shard whose
// draw is not known (no declared figure, no card limit, no policy figure)
// needs nothing: missing data never blocks placement.
func (b *budget) check(n inventory.Node, need float64) string {
if b.use == nil {
return ""
}
for _, d := range n.Domains() {
limit := b.pol.Limit(d)
if limit <= 0 {
continue
}
used := b.use[d].Used + b.added[d]
if used+need > limit {
capW := b.pol.Budgets[d].CapWatts
of := facility.KW(limit) + " cap"
if limit < capW {
of = fmt.Sprintf("%s limit (%s cap less %.0f%%)", facility.KW(limit), facility.KW(capW), 100-100*limit/capW)
}
return fmt.Sprintf("%s: %s of a %s in use, the shard needs ~%s",
d.Human(), facility.KW(used), of, facility.KW(need))
}
}
return ""
}
// add counts a chosen shard against the node's domains.
func (b *budget) add(n inventory.Node, need float64) {
for _, d := range n.Domains() {
b.added[d] += need
}
}
// clock is the scheduler's time, for the freshness of power readings.
func (s *Scheduler) clock() time.Time {
if s.now != nil {
return s.now()
}
return time.Now()
}
// heldReason is a hold as a rejection says it.
func heldReason(h facility.Hold) string {
why := h.Domain.Human() + " is held"
if h.Reason != "" {
why += ": " + h.Reason
}
return why
}
// spreadLevels are the domains replicas of one model are kept apart in:
// the rack, which a top-of-rack switch or a door left open takes out whole,
// and the PDU, which a breaker does.
var spreadLevels = []facility.Level{facility.Rack, facility.PDU}
// spreadAcrossDomains moves to the front the candidates sharing the fewest
// racks and PDUs with the nodes the model already runs on, keeping the order
// given among equals.
//
// A preference and never a filter, as spreading across nodes is: a rack
// holding a model's only free cards still takes its second replica, which
// survives an engine crash if not a breaker. Nodes with no location share
// nothing and lose nothing, so a cluster without labels places as before.
func (s *Scheduler) spreadAcrossDomains(candidates []candidate, avoid []string) {
if len(avoid) == 0 {
return
}
taken := map[facility.Domain]bool{}
for _, id := range avoid {
if n, ok := s.inv.Get(id); ok {
for _, d := range n.Domains() {
if slices.Contains(spreadLevels, d.Level()) {
taken[d] = true
}
}
}
}
if len(taken) == 0 {
return
}
shared := func(c candidate) int {
k := 0
for _, d := range c.node.Domains() {
if taken[d] {
k++
}
}
return k
}
sort.SliceStable(candidates, func(i, j int) bool { return shared(candidates[i]) < shared(candidates[j]) })
}
// withinGroups divides a multi-node replica's candidates by the domain its
// profile keeps it within, largest first, and rejects the nodes that do not
// say which they are in. Without the constraint it is all of them, as one.
func withinGroups(p Profile, pool string, cands []candidate, rejections *[]Rejection) [][]candidate {
if p.KeepWithin == "" {
return [][]candidate{cands}
}
by := map[facility.Domain][]candidate{}
for _, c := range cands {
d := c.node.Location().At(p.KeepWithin)
if d == "" {
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: c.node.ID,
Reason: fmt.Sprintf("has no %s label, and the profile keeps its nodes within one %s",
p.KeepWithin.Label(), p.KeepWithin)})
continue
}
by[d] = append(by[d], c)
}
keys := make([]facility.Domain, 0, len(by))
for d := range by {
keys = append(keys, d)
}
sort.Slice(keys, func(i, j int) bool {
if len(by[keys[i]]) != len(by[keys[j]]) {
return len(by[keys[i]]) > len(by[keys[j]])
}
return keys[i] < keys[j]
})
out := make([][]candidate, 0, len(keys))
for _, d := range keys {
if len(by[d]) < p.Nodes {
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool,
Reason: fmt.Sprintf("needs %d nodes within one %s, %s has %d", p.Nodes, p.KeepWithin, d.Human(), len(by[d]))})
continue
}
out = append(out, by[d])
}
return out
}
+325
View File
@@ -0,0 +1,325 @@
package placement
import (
"fmt"
"math/rand/v2"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// at puts a node in a rack, and on PDUs when given.
func at(n inventory.Node, rack string, pdus ...string) inventory.Node {
if n.Labels == nil {
n.Labels = map[string]string{}
}
n.Labels[facility.Rack.Label()] = rack
if len(pdus) > 0 {
n.Labels[facility.PDU.Label()] = strings.Join(pdus, ",")
}
return n
}
// withPowerLimit has a node's cards report a power limit, and draw, as a
// heartbeat would.
func withPowerLimit(n inventory.Node, limit, draw float64) inventory.Node {
l := &inventory.Load{At: time.Now()}
for _, g := range n.GPUs {
l.GPUs = append(l.GPUs, inventory.GPUSensor{Index: g.Index, PowerLimitW: limit, PowerW: draw})
}
n.Load = l
return n
}
func onePerNode(name string, gpus int) Profile {
return Profile{Name: name, GPUClass: "h100-80g", GPUsPerNode: gpus, Nodes: 1}
}
func TestNothingIsPlacedInAHeldRackAndTheRejectionSaysWhy(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("b1", "h100-80g", 8), "r2"))
st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{
"rack:r2": {Domain: "rack:r2", Reason: "PDU A replacement"},
}})
st.Allocate("a1", inventory.Allocation{ShardID: "x", GPUIndices: []int{0, 1, 2, 3, 4, 5, 6, 7}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement != nil {
t.Fatalf("placed on %s, in a held rack", res.Placement.Nodes[0].NodeID)
}
if !strings.Contains(res.Explain(), "b1: rack r2 is held: PDU A replacement") {
t.Errorf("the rejection does not name the hold:\n%s", res.Explain())
}
}
func TestAHeldPDUHoldsEveryRackOnIt(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1", "p1", "p2"), at(node("b1", "h100-80g", 8), "r2", "p3"))
st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{"pdu:p2": {Domain: "pdu:p2"}}})
res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil)
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" {
t.Fatalf("want b1, the node on no held PDU:\n%s", res.Explain())
}
}
func TestASecondReplicaGoesToAnotherRackBeforeAFullerNodeInTheSameOne(t *testing.T) {
// Bin packing alone would choose a2, the partly used node beside the
// first replica; a rack apart is worth an emptier node.
st := store(
at(node("a1", "h100-80g", 8), "r1"),
at(node("a2", "h100-80g", 8), "r1"),
at(node("b1", "h100-80g", 8), "r2"),
)
st.Allocate("a2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
if res.Placement == nil {
t.Fatalf("not placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "b1" {
t.Errorf("placed on %s, in the rack of the first replica; want b1 in another", got)
}
}
func TestReplicasAreSpreadAcrossPDUsAsWellAsRacks(t *testing.T) {
st := store(
at(node("a1", "h100-80g", 8), "r1", "p1"),
at(node("b1", "h100-80g", 8), "r2", "p1"),
at(node("c1", "h100-80g", 8), "r3", "p2"),
)
st.Allocate("b1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "c1" {
t.Fatalf("want c1, on another rack and another PDU:\n%s", res.Explain())
}
}
func TestSpreadingAcrossRacksNeverLeavesAReplicaUnplaced(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("a2", "h100-80g", 8), "r1"))
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "a2" {
t.Fatalf("the rack's other node should still be used:\n%s", res.Explain())
}
}
func TestNodesWithoutALocationPlaceAsTheyAlwaysDid(t *testing.T) {
// Bin packing picks the partly used node; nothing about racks changes it.
st := store(node("n1", "h100-80g", 8), node("n2", "h100-80g", 8))
st.Allocate("n2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1}})
st.SetFacility(facility.Policy{GPUWatts: 700, Margin: 0.1})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "n2" {
t.Fatalf("want n2, the bin-packed choice:\n%s", res.Explain())
}
}
func TestAReplicaThatWouldTakeARackPastItsCapGoesToAnotherRack(t *testing.T) {
// Rack r1: 8 cards at 700 W, four of them placed: 2.8 kW used of a
// 4.5 kW limit (5 kW less 10%). Four more need 2.8 kW, which does not fit.
st := store(
withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0),
withPowerLimit(at(node("b1", "h100-80g", 8), "r2"), 700, 0),
)
st.Allocate("a1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
st.SetFacility(facility.Policy{Margin: 0.1, Budgets: map[facility.Domain]facility.Budget{
"rack:r1": {CapWatts: 5000},
}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" {
t.Fatalf("want b1, in the rack with room:\n%s", res.Explain())
}
want := "a1: rack r1: 2.8 kW of a 4.5 kW limit (5.0 kW cap less 10%) in use, the shard needs ~2.8 kW"
if !strings.Contains(res.Explain(), want) {
t.Errorf("the rejection should give the figures, %q:\n%s", want, res.Explain())
}
}
func TestAProfilesDeclaredDrawIsWhatIsCounted(t *testing.T) {
st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0))
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 2000}}})
derived := onePerNode("tp4", 4) // 4 × 700 W: too much for 2 kW
if res := New(st).Place([]Profile{derived}, nil); res.Placement != nil {
t.Fatalf("2.8 kW was placed under a 2 kW cap")
}
declared := derived
declared.PowerWatts = 1500
res := New(st).Place([]Profile{declared}, nil)
if res.Placement == nil {
t.Fatalf("a profile declaring 1.5 kW was refused under a 2 kW cap:\n%s", res.Explain())
}
if res.Placement.Nodes[0].PowerW != 1500 {
t.Errorf("the shard carries %v W to record, want the declared 1500", res.Placement.Nodes[0].PowerW)
}
}
func TestWhatTheSensorsReadCountsWhenItIsMoreThanWasReserved(t *testing.T) {
// Nothing placed, but the cards draw 400 W each: 3.2 kW measured, and
// another 2.8 kW would pass a 5 kW cap only on the reservations.
st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400))
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement != nil {
t.Fatalf("placed against reservations alone, with 3.2 kW measured")
}
if !strings.Contains(res.Explain(), "3.2 kW of a 5.0 kW cap in use") {
t.Errorf("the measured draw should be what is in use:\n%s", res.Explain())
}
}
func TestAStaleReadingIsNotCounted(t *testing.T) {
n := withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400)
n.Load.At = time.Now().Add(-10 * time.Minute)
st := store(n)
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}})
if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement == nil {
t.Fatalf("a ten-minute-old reading kept a replica out:\n%s", res.Explain())
}
}
func TestADrawNobodyKnowsNeverBlocksPlacement(t *testing.T) {
// A cap, but no card reports a limit and no figure is configured: the
// shard's draw is unknown, which is not a reason to refuse it.
st := store(at(node("a1", "h100-80g", 8), "r1"))
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 100}}})
if res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil); res.Placement == nil {
t.Fatalf("an unknown draw was refused:\n%s", res.Explain())
}
}
func TestTheOtherEquipmentInARackCountsAgainstItsCap(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1"))
st.SetFacility(facility.Policy{GPUWatts: 500, Budgets: map[facility.Domain]facility.Budget{
"rack:r1": {CapWatts: 5000, OtherWatts: 3500},
}})
if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement != nil {
t.Fatal("2 kW was placed where switches already take 3.5 of 5")
}
}
func TestAMultiNodeReplicasShardsAreCountedTogetherAgainstTheirRack(t *testing.T) {
// Each node alone fits r1's budget; two together do not, so the second
// shard goes to the rack with room — on the same fabric, as always.
st := store(
withPowerLimit(at(node("a1", "h100-80g", 8, "ib"), "r1"), 500, 0),
withPowerLimit(at(node("a2", "h100-80g", 8, "ib"), "r1"), 500, 0),
withPowerLimit(at(node("b1", "h100-80g", 8, "ib"), "r2"), 500, 0),
)
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}})
res := New(st).Place([]Profile{{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement == nil {
t.Fatalf("not placed:\n%s", res.Explain())
}
racks := map[string]int{}
for _, a := range res.Placement.Nodes {
n, _ := st.Get(a.NodeID)
racks[n.Location().Rack]++
}
if racks["r1"] != 1 || racks["r2"] != 1 {
t.Errorf("shards by rack %v: two in r1 draw 8 kW against a 6 kW cap", racks)
}
if !strings.Contains(res.Explain(), "with this replica's other shards") {
t.Errorf("the node passed over should say why:\n%s", res.Explain())
}
}
func TestAProfileKeptWithinARackNeverSpansTwo(t *testing.T) {
// The emptiest-first order would take a1 and b1; the profile wants one
// rack, and only r2 has two nodes.
st := store(
at(node("a1", "h100-80g", 8, "ib"), "r1"),
at(node("b1", "h100-80g", 8, "ib"), "r2"),
at(node("b2", "h100-80g", 8, "ib"), "r2"),
node("x1", "h100-80g", 8, "ib"),
)
p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack}
res := New(st).Place([]Profile{p}, nil)
if res.Placement == nil {
t.Fatalf("not placed:\n%s", res.Explain())
}
for _, a := range res.Placement.Nodes {
if a.NodeID != "b1" && a.NodeID != "b2" {
t.Errorf("a shard on %s, outside rack r2", a.NodeID)
}
}
explain := res.Explain()
if !strings.Contains(explain, "x1: has no llmgw.io/rack label, and the profile keeps its nodes within one rack") {
t.Errorf("the unlabelled node should be named:\n%s", explain)
}
if !strings.Contains(explain, "needs 2 nodes within one rack, rack r1 has 1") {
t.Errorf("the rack too small should be named:\n%s", explain)
}
}
func TestAProfileKeptWithinARackIsNotPlacedAcrossRacks(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8, "ib"), "r1"), at(node("b1", "h100-80g", 8, "ib"), "r2"))
p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack}
if res := New(st).Place([]Profile{p}, nil); res.Placement != nil {
t.Fatal("a replica kept within one rack was placed across two")
}
}
// TestNoCapIsEverExceeded fills random clusters until nothing more fits, and
// checks every capped domain stayed within its limit — and that a cluster
// whose draw nobody knows is filled as if it had no caps at all.
func TestNoCapIsEverExceeded(t *testing.T) {
rng := rand.New(rand.NewPCG(3, 4))
for iter := range 200 {
st := inventory.NewStore()
known := rng.IntN(4) != 0
cards := 0
for i := range 2 + rng.IntN(6) {
n := at(node(fmt.Sprintf("n%d", i), "h100-80g", 1+rng.IntN(8), "ib"),
fmt.Sprintf("r%d", rng.IntN(3)), fmt.Sprintf("p%d", rng.IntN(2)))
if known {
n = withPowerLimit(n, float64(300+rng.IntN(500)), 0)
}
cards += len(n.GPUs)
st.Upsert(n)
}
pol := facility.Policy{Margin: float64(rng.IntN(20)) / 100, Budgets: map[facility.Domain]facility.Budget{}}
for _, d := range []facility.Domain{"rack:r0", "rack:r1", "rack:r2", "pdu:p0", "pdu:p1"} {
if rng.IntN(2) == 0 {
pol.Budgets[d] = facility.Budget{CapWatts: float64(1000 + rng.IntN(8000))}
}
}
st.SetFacility(pol)
for k := range 60 {
p := Profile{Name: "gen", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1 + rng.IntN(2)}
res := New(st).Place([]Profile{p}, nil)
if res.Placement == nil {
continue
}
for _, a := range res.Placement.Nodes {
st.Allocate(a.NodeID, inventory.Allocation{ShardID: fmt.Sprintf("s%d-%d", k, a.Rank), GPUIndices: a.GPUIndices})
}
}
for d, u := range st.PowerUse(time.Now()) {
if u.Limit > 0 && u.Used > u.Limit+1e-6 {
t.Fatalf("iteration %d: %s uses %.0f W of a %.0f W limit", iter, d, u.Used, u.Limit)
}
}
if !known {
free := 0
for _, n := range st.Nodes() {
free += len(st.FreeGPUs(n.ID))
}
if free != 0 {
t.Fatalf("iteration %d: %d of %d cards left free with no draw known", iter, free, cards)
}
}
}
}
+85 -14
View File
@@ -16,8 +16,10 @@ import (
"slices"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -45,6 +47,13 @@ type Profile struct {
// Source is the model's weights, as the engine would be given it. Used to
// prefer nodes that already hold them; empty asks for no preference.
Source string
// PowerWatts is what a replica of this profile draws, split evenly over
// its nodes. Zero derives it from the cards' power limits and the cores.
PowerWatts float64
// KeepWithin keeps a multi-node replica's nodes inside one domain of
// this level, rack or row, where the fabric or the cabling needs them
// close. Empty is no constraint.
KeepWithin facility.Level
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
@@ -81,6 +90,9 @@ type NodeAssignment struct {
GPUIndices []int
// HCAs are the adapters of the chosen fabric only.
HCAs []string
// PowerW is the draw the profile declared for this shard, to be recorded
// with its allocation; zero when the profile declared none.
PowerW float64
}
// Placement is a decision: which profile, which fabric, which nodes.
@@ -139,6 +151,9 @@ func (r Result) Explain() string {
// Scheduler places replicas onto an inventory.
type Scheduler struct {
inv *inventory.Store
// now is the clock power readings are judged fresh by; nil is the wall
// clock.
now func() time.Time
}
// New builds a scheduler over an inventory.
@@ -148,6 +163,11 @@ func New(inv *inventory.Store) *Scheduler { return &Scheduler{inv: inv} }
//
// avoid lists nodes already hosting this model, so replicas of one model spread
// across failure domains instead of stacking on one machine.
//
// The rejections of the profile that was placed are kept with the placement,
// not only those of the profiles before it: a replica placed in the second
// rack because the first was held or at its power cap is placed somewhere
// nobody expected, and explain-placement is where the operator asks why.
func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
var res Result
@@ -168,6 +188,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
if p.Nodes == 1 {
if pl, rej := s.placeSingleNode(p, avoid); pl != nil {
res.Placement = pl
res.Rejections = append(res.Rejections, rej...)
return res
} else {
res.Rejections = append(res.Rejections, rej...)
@@ -176,6 +197,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
}
if pl, rej := s.placeMultiNode(p, avoid); pl != nil {
res.Placement = pl
res.Rejections = append(res.Rejections, rej...)
return res
} else {
res.Rejections = append(res.Rejections, rej...)
@@ -212,17 +234,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
len(s.inv.Allocations(candidates[j].node.ID))
})
preferNodesHoldingWeights(p.Source, candidates)
s.spreadAcrossDomains(candidates, avoid)
return &Placement{
Profile: p,
Pool: "none/cpu",
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}},
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0, PowerW: p.declaredShardWatts()}},
}, rejections
}
// Last, so it takes precedence: moving tens of gigabytes costs more than an
// imperfectly packed node, and with isolated fabrics it may cost a
// cross-fabric copy over the management network.
// After packing, so it takes precedence: moving tens of gigabytes costs
// more than an imperfectly packed node, and with isolated fabrics it may
// cost a cross-fabric copy over the management network.
preferNodesHoldingWeights(p.Source, candidates)
// And racks over weights: a second replica in the rack of the first is
// one breaker away from none, which no copy saved is worth.
s.spreadAcrossDomains(candidates, avoid)
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode, nil)
@@ -237,7 +263,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return &Placement{
Profile: p,
Pool: "none/" + p.classLabel(),
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}},
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus, PowerW: p.declaredShardWatts()}},
}, rejections
}
@@ -273,8 +299,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
if group.key != "" && group.split {
pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel()
}
if pl := s.placeOnFabric(p, fabric, pool, group.nodes, &rejections); pl != nil {
return pl, rejections
for _, within := range withinGroups(p, pool, group.nodes, &rejections) {
if pl := s.placeOnFabric(p, fabric, pool, within, &rejections); pl != nil {
return pl, rejections
}
}
}
}
@@ -325,8 +353,10 @@ func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejec
})
for _, d := range keys {
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
return pl
for _, within := range withinGroups(p, pool, by[d], rejections) {
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, within, rejections); pl != nil {
return pl
}
}
}
return nil
@@ -416,8 +446,22 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
// together, a degraded cable last.
candidates = byLeaf(candidates, hints(s.inv))
// Each node passed the power budgets on its own; together, two shards in
// one rack may not. Nodes whose shard would take a domain past its cap
// with the shards already chosen are passed over for the next.
budget := s.newBudget()
assignments := make([]NodeAssignment, 0, p.Nodes)
for rank, c := range candidates[:p.Nodes] {
for _, c := range candidates {
if len(assignments) == p.Nodes {
break
}
if why := budget.check(c.node, s.shardWatts(p, c)); why != "" {
*rejections = append(*rejections, Rejection{
Profile: p.Name, Pool: pool, Node: c.node.ID,
Reason: why + ", with this replica's other shards",
})
continue
}
gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric))
if gpus == nil {
*rejections = append(*rejections, Rejection{
@@ -426,14 +470,23 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
})
return nil
}
budget.add(c.node, s.shardWatts(p, c))
assignments = append(assignments, NodeAssignment{
NodeID: c.node.ID,
Rank: rank,
Rank: len(assignments),
GPUIndices: gpus,
// Only this fabric's adapters, never the node's others.
HCAs: c.node.HCAsFor(fabric),
HCAs: c.node.HCAsFor(fabric),
PowerW: p.declaredShardWatts(),
})
}
if len(assignments) < p.Nodes {
*rejections = append(*rejections, Rejection{
Profile: p.Name, Pool: pool,
Reason: fmt.Sprintf("needs %d nodes, only %d fit within the power budgets together", p.Nodes, len(assignments)),
})
return nil
}
return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}
}
@@ -508,6 +561,8 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
}
pool += "/" + p.classLabel()
pol := s.inv.Facility()
budget := s.newBudget()
var out []candidate
for _, n := range s.inv.Nodes() {
reject := func(reason string) {
@@ -518,6 +573,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
reject("node is " + n.State.String())
continue
}
// A domain held for maintenance takes nothing new, as a cordon of
// every node in it would, and says which hold and why.
if h, held := pol.HeldIn(n.Domains()); held {
reject(heldReason(h))
continue
}
if slices.Contains(avoid, n.ID) {
reject("already hosts this model; spreading across failure domains")
continue
@@ -608,7 +669,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
if p.CPUOnly() {
// Every ready node can run it; there is no card to be free.
out = append(out, candidate{node: n})
c := candidate{node: n}
if why := budget.check(n, s.shardWatts(p, c)); why != "" {
reject(why)
continue
}
out = append(out, c)
continue
}
@@ -624,7 +690,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
reject(fmt.Sprintf("has %d free %s GPUs, needs %d", len(free), p.GPUClass, p.GPUsPerNode))
continue
}
out = append(out, candidate{node: n, free: free})
c := candidate{node: n, free: free}
if why := budget.check(n, s.shardWatts(p, c)); why != "" {
reject(why)
continue
}
out = append(out, c)
}
return out
}