Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -0,0 +1,193 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// declaredShardWatts is the profile's declared draw for one node's shard.
|
||||
func (p Profile) declaredShardWatts() float64 {
|
||||
if p.PowerWatts <= 0 {
|
||||
return 0
|
||||
}
|
||||
return p.PowerWatts / float64(max(p.Nodes, 1))
|
||||
}
|
||||
|
||||
// shardWatts is what one shard of this profile is estimated to draw on a
|
||||
// candidate: the declared figure, or its cards at the highest power limit
|
||||
// among the node's free ones of the class (the cards are not chosen yet, and
|
||||
// guessing low is the mistake that trips a breaker), or its cores.
|
||||
func (s *Scheduler) shardWatts(p Profile, c candidate) float64 {
|
||||
if w := p.declaredShardWatts(); w > 0 {
|
||||
return w
|
||||
}
|
||||
pol := s.inv.Facility()
|
||||
if p.CPUOnly() {
|
||||
return float64(p.CPUCores) * pol.CPUCoreWatts
|
||||
}
|
||||
per := 0.0
|
||||
for _, g := range c.free {
|
||||
per = max(per, inventory.GPUWatts(c.node, g.Index, pol))
|
||||
}
|
||||
if per == 0 {
|
||||
per = pol.GPUWatts
|
||||
}
|
||||
return per * float64(p.GPUsPerNode)
|
||||
}
|
||||
|
||||
// budget is the power budgets as one placement decision spends them: the
|
||||
// domains' use when it began, and what the shards it has chosen so far add.
|
||||
type budget struct {
|
||||
pol facility.Policy
|
||||
use map[facility.Domain]inventory.PowerUse
|
||||
added map[facility.Domain]float64
|
||||
}
|
||||
|
||||
// newBudget reads the domains' use, once per decision. Without budgets there
|
||||
// is nothing to read and every check passes.
|
||||
func (s *Scheduler) newBudget() *budget {
|
||||
pol := s.inv.Facility()
|
||||
b := &budget{pol: pol, added: map[facility.Domain]float64{}}
|
||||
if len(pol.Budgets) > 0 {
|
||||
b.use = s.inv.PowerUse(s.clock())
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// check says why a shard drawing need watts on a node would take one of the
|
||||
// node's domains past its limit; empty when it fits them all. A shard whose
|
||||
// draw is not known (no declared figure, no card limit, no policy figure)
|
||||
// needs nothing: missing data never blocks placement.
|
||||
func (b *budget) check(n inventory.Node, need float64) string {
|
||||
if b.use == nil {
|
||||
return ""
|
||||
}
|
||||
for _, d := range n.Domains() {
|
||||
limit := b.pol.Limit(d)
|
||||
if limit <= 0 {
|
||||
continue
|
||||
}
|
||||
used := b.use[d].Used + b.added[d]
|
||||
if used+need > limit {
|
||||
capW := b.pol.Budgets[d].CapWatts
|
||||
of := facility.KW(limit) + " cap"
|
||||
if limit < capW {
|
||||
of = fmt.Sprintf("%s limit (%s cap less %.0f%%)", facility.KW(limit), facility.KW(capW), 100-100*limit/capW)
|
||||
}
|
||||
return fmt.Sprintf("%s: %s of a %s in use, the shard needs ~%s",
|
||||
d.Human(), facility.KW(used), of, facility.KW(need))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// add counts a chosen shard against the node's domains.
|
||||
func (b *budget) add(n inventory.Node, need float64) {
|
||||
for _, d := range n.Domains() {
|
||||
b.added[d] += need
|
||||
}
|
||||
}
|
||||
|
||||
// clock is the scheduler's time, for the freshness of power readings.
|
||||
func (s *Scheduler) clock() time.Time {
|
||||
if s.now != nil {
|
||||
return s.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// heldReason is a hold as a rejection says it.
|
||||
func heldReason(h facility.Hold) string {
|
||||
why := h.Domain.Human() + " is held"
|
||||
if h.Reason != "" {
|
||||
why += ": " + h.Reason
|
||||
}
|
||||
return why
|
||||
}
|
||||
|
||||
// spreadLevels are the domains replicas of one model are kept apart in:
|
||||
// the rack, which a top-of-rack switch or a door left open takes out whole,
|
||||
// and the PDU, which a breaker does.
|
||||
var spreadLevels = []facility.Level{facility.Rack, facility.PDU}
|
||||
|
||||
// spreadAcrossDomains moves to the front the candidates sharing the fewest
|
||||
// racks and PDUs with the nodes the model already runs on, keeping the order
|
||||
// given among equals.
|
||||
//
|
||||
// A preference and never a filter, as spreading across nodes is: a rack
|
||||
// holding a model's only free cards still takes its second replica, which
|
||||
// survives an engine crash if not a breaker. Nodes with no location share
|
||||
// nothing and lose nothing, so a cluster without labels places as before.
|
||||
func (s *Scheduler) spreadAcrossDomains(candidates []candidate, avoid []string) {
|
||||
if len(avoid) == 0 {
|
||||
return
|
||||
}
|
||||
taken := map[facility.Domain]bool{}
|
||||
for _, id := range avoid {
|
||||
if n, ok := s.inv.Get(id); ok {
|
||||
for _, d := range n.Domains() {
|
||||
if slices.Contains(spreadLevels, d.Level()) {
|
||||
taken[d] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(taken) == 0 {
|
||||
return
|
||||
}
|
||||
shared := func(c candidate) int {
|
||||
k := 0
|
||||
for _, d := range c.node.Domains() {
|
||||
if taken[d] {
|
||||
k++
|
||||
}
|
||||
}
|
||||
return k
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool { return shared(candidates[i]) < shared(candidates[j]) })
|
||||
}
|
||||
|
||||
// withinGroups divides a multi-node replica's candidates by the domain its
|
||||
// profile keeps it within, largest first, and rejects the nodes that do not
|
||||
// say which they are in. Without the constraint it is all of them, as one.
|
||||
func withinGroups(p Profile, pool string, cands []candidate, rejections *[]Rejection) [][]candidate {
|
||||
if p.KeepWithin == "" {
|
||||
return [][]candidate{cands}
|
||||
}
|
||||
by := map[facility.Domain][]candidate{}
|
||||
for _, c := range cands {
|
||||
d := c.node.Location().At(p.KeepWithin)
|
||||
if d == "" {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: c.node.ID,
|
||||
Reason: fmt.Sprintf("has no %s label, and the profile keeps its nodes within one %s",
|
||||
p.KeepWithin.Label(), p.KeepWithin)})
|
||||
continue
|
||||
}
|
||||
by[d] = append(by[d], c)
|
||||
}
|
||||
keys := make([]facility.Domain, 0, len(by))
|
||||
for d := range by {
|
||||
keys = append(keys, d)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(by[keys[i]]) != len(by[keys[j]]) {
|
||||
return len(by[keys[i]]) > len(by[keys[j]])
|
||||
}
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
out := make([][]candidate, 0, len(keys))
|
||||
for _, d := range keys {
|
||||
if len(by[d]) < p.Nodes {
|
||||
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool,
|
||||
Reason: fmt.Sprintf("needs %d nodes within one %s, %s has %d", p.Nodes, p.KeepWithin, d.Human(), len(by[d]))})
|
||||
continue
|
||||
}
|
||||
out = append(out, by[d])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// at puts a node in a rack, and on PDUs when given.
|
||||
func at(n inventory.Node, rack string, pdus ...string) inventory.Node {
|
||||
if n.Labels == nil {
|
||||
n.Labels = map[string]string{}
|
||||
}
|
||||
n.Labels[facility.Rack.Label()] = rack
|
||||
if len(pdus) > 0 {
|
||||
n.Labels[facility.PDU.Label()] = strings.Join(pdus, ",")
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// withPowerLimit has a node's cards report a power limit, and draw, as a
|
||||
// heartbeat would.
|
||||
func withPowerLimit(n inventory.Node, limit, draw float64) inventory.Node {
|
||||
l := &inventory.Load{At: time.Now()}
|
||||
for _, g := range n.GPUs {
|
||||
l.GPUs = append(l.GPUs, inventory.GPUSensor{Index: g.Index, PowerLimitW: limit, PowerW: draw})
|
||||
}
|
||||
n.Load = l
|
||||
return n
|
||||
}
|
||||
|
||||
func onePerNode(name string, gpus int) Profile {
|
||||
return Profile{Name: name, GPUClass: "h100-80g", GPUsPerNode: gpus, Nodes: 1}
|
||||
}
|
||||
|
||||
func TestNothingIsPlacedInAHeldRackAndTheRejectionSaysWhy(t *testing.T) {
|
||||
st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("b1", "h100-80g", 8), "r2"))
|
||||
st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{
|
||||
"rack:r2": {Domain: "rack:r2", Reason: "PDU A replacement"},
|
||||
}})
|
||||
st.Allocate("a1", inventory.Allocation{ShardID: "x", GPUIndices: []int{0, 1, 2, 3, 4, 5, 6, 7}})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatalf("placed on %s, in a held rack", res.Placement.Nodes[0].NodeID)
|
||||
}
|
||||
if !strings.Contains(res.Explain(), "b1: rack r2 is held: PDU A replacement") {
|
||||
t.Errorf("the rejection does not name the hold:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeldPDUHoldsEveryRackOnIt(t *testing.T) {
|
||||
st := store(at(node("a1", "h100-80g", 8), "r1", "p1", "p2"), at(node("b1", "h100-80g", 8), "r2", "p3"))
|
||||
st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{"pdu:p2": {Domain: "pdu:p2"}}})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil)
|
||||
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" {
|
||||
t.Fatalf("want b1, the node on no held PDU:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondReplicaGoesToAnotherRackBeforeAFullerNodeInTheSameOne(t *testing.T) {
|
||||
// Bin packing alone would choose a2, the partly used node beside the
|
||||
// first replica; a rack apart is worth an emptier node.
|
||||
st := store(
|
||||
at(node("a1", "h100-80g", 8), "r1"),
|
||||
at(node("a2", "h100-80g", 8), "r1"),
|
||||
at(node("b1", "h100-80g", 8), "r2"),
|
||||
)
|
||||
st.Allocate("a2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("not placed:\n%s", res.Explain())
|
||||
}
|
||||
if got := res.Placement.Nodes[0].NodeID; got != "b1" {
|
||||
t.Errorf("placed on %s, in the rack of the first replica; want b1 in another", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicasAreSpreadAcrossPDUsAsWellAsRacks(t *testing.T) {
|
||||
st := store(
|
||||
at(node("a1", "h100-80g", 8), "r1", "p1"),
|
||||
at(node("b1", "h100-80g", 8), "r2", "p1"),
|
||||
at(node("c1", "h100-80g", 8), "r3", "p2"),
|
||||
)
|
||||
st.Allocate("b1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
|
||||
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "c1" {
|
||||
t.Fatalf("want c1, on another rack and another PDU:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpreadingAcrossRacksNeverLeavesAReplicaUnplaced(t *testing.T) {
|
||||
st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("a2", "h100-80g", 8), "r1"))
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
|
||||
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "a2" {
|
||||
t.Fatalf("the rack's other node should still be used:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesWithoutALocationPlaceAsTheyAlwaysDid(t *testing.T) {
|
||||
// Bin packing picks the partly used node; nothing about racks changes it.
|
||||
st := store(node("n1", "h100-80g", 8), node("n2", "h100-80g", 8))
|
||||
st.Allocate("n2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1}})
|
||||
st.SetFacility(facility.Policy{GPUWatts: 700, Margin: 0.1})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
|
||||
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "n2" {
|
||||
t.Fatalf("want n2, the bin-packed choice:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaThatWouldTakeARackPastItsCapGoesToAnotherRack(t *testing.T) {
|
||||
// Rack r1: 8 cards at 700 W, four of them placed: 2.8 kW used of a
|
||||
// 4.5 kW limit (5 kW less 10%). Four more need 2.8 kW, which does not fit.
|
||||
st := store(
|
||||
withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0),
|
||||
withPowerLimit(at(node("b1", "h100-80g", 8), "r2"), 700, 0),
|
||||
)
|
||||
st.Allocate("a1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
|
||||
st.SetFacility(facility.Policy{Margin: 0.1, Budgets: map[facility.Domain]facility.Budget{
|
||||
"rack:r1": {CapWatts: 5000},
|
||||
}})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
|
||||
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" {
|
||||
t.Fatalf("want b1, in the rack with room:\n%s", res.Explain())
|
||||
}
|
||||
want := "a1: rack r1: 2.8 kW of a 4.5 kW limit (5.0 kW cap less 10%) in use, the shard needs ~2.8 kW"
|
||||
if !strings.Contains(res.Explain(), want) {
|
||||
t.Errorf("the rejection should give the figures, %q:\n%s", want, res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfilesDeclaredDrawIsWhatIsCounted(t *testing.T) {
|
||||
st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0))
|
||||
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 2000}}})
|
||||
|
||||
derived := onePerNode("tp4", 4) // 4 × 700 W: too much for 2 kW
|
||||
if res := New(st).Place([]Profile{derived}, nil); res.Placement != nil {
|
||||
t.Fatalf("2.8 kW was placed under a 2 kW cap")
|
||||
}
|
||||
declared := derived
|
||||
declared.PowerWatts = 1500
|
||||
res := New(st).Place([]Profile{declared}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("a profile declaring 1.5 kW was refused under a 2 kW cap:\n%s", res.Explain())
|
||||
}
|
||||
if res.Placement.Nodes[0].PowerW != 1500 {
|
||||
t.Errorf("the shard carries %v W to record, want the declared 1500", res.Placement.Nodes[0].PowerW)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatTheSensorsReadCountsWhenItIsMoreThanWasReserved(t *testing.T) {
|
||||
// Nothing placed, but the cards draw 400 W each: 3.2 kW measured, and
|
||||
// another 2.8 kW would pass a 5 kW cap only on the reservations.
|
||||
st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400))
|
||||
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}})
|
||||
|
||||
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
|
||||
if res.Placement != nil {
|
||||
t.Fatalf("placed against reservations alone, with 3.2 kW measured")
|
||||
}
|
||||
if !strings.Contains(res.Explain(), "3.2 kW of a 5.0 kW cap in use") {
|
||||
t.Errorf("the measured draw should be what is in use:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStaleReadingIsNotCounted(t *testing.T) {
|
||||
n := withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400)
|
||||
n.Load.At = time.Now().Add(-10 * time.Minute)
|
||||
st := store(n)
|
||||
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}})
|
||||
|
||||
if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement == nil {
|
||||
t.Fatalf("a ten-minute-old reading kept a replica out:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestADrawNobodyKnowsNeverBlocksPlacement(t *testing.T) {
|
||||
// A cap, but no card reports a limit and no figure is configured: the
|
||||
// shard's draw is unknown, which is not a reason to refuse it.
|
||||
st := store(at(node("a1", "h100-80g", 8), "r1"))
|
||||
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 100}}})
|
||||
|
||||
if res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil); res.Placement == nil {
|
||||
t.Fatalf("an unknown draw was refused:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheOtherEquipmentInARackCountsAgainstItsCap(t *testing.T) {
|
||||
st := store(at(node("a1", "h100-80g", 8), "r1"))
|
||||
st.SetFacility(facility.Policy{GPUWatts: 500, Budgets: map[facility.Domain]facility.Budget{
|
||||
"rack:r1": {CapWatts: 5000, OtherWatts: 3500},
|
||||
}})
|
||||
|
||||
if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement != nil {
|
||||
t.Fatal("2 kW was placed where switches already take 3.5 of 5")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMultiNodeReplicasShardsAreCountedTogetherAgainstTheirRack(t *testing.T) {
|
||||
// Each node alone fits r1's budget; two together do not, so the second
|
||||
// shard goes to the rack with room — on the same fabric, as always.
|
||||
st := store(
|
||||
withPowerLimit(at(node("a1", "h100-80g", 8, "ib"), "r1"), 500, 0),
|
||||
withPowerLimit(at(node("a2", "h100-80g", 8, "ib"), "r1"), 500, 0),
|
||||
withPowerLimit(at(node("b1", "h100-80g", 8, "ib"), "r2"), 500, 0),
|
||||
)
|
||||
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}})
|
||||
|
||||
res := New(st).Place([]Profile{{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("not placed:\n%s", res.Explain())
|
||||
}
|
||||
racks := map[string]int{}
|
||||
for _, a := range res.Placement.Nodes {
|
||||
n, _ := st.Get(a.NodeID)
|
||||
racks[n.Location().Rack]++
|
||||
}
|
||||
if racks["r1"] != 1 || racks["r2"] != 1 {
|
||||
t.Errorf("shards by rack %v: two in r1 draw 8 kW against a 6 kW cap", racks)
|
||||
}
|
||||
if !strings.Contains(res.Explain(), "with this replica's other shards") {
|
||||
t.Errorf("the node passed over should say why:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileKeptWithinARackNeverSpansTwo(t *testing.T) {
|
||||
// The emptiest-first order would take a1 and b1; the profile wants one
|
||||
// rack, and only r2 has two nodes.
|
||||
st := store(
|
||||
at(node("a1", "h100-80g", 8, "ib"), "r1"),
|
||||
at(node("b1", "h100-80g", 8, "ib"), "r2"),
|
||||
at(node("b2", "h100-80g", 8, "ib"), "r2"),
|
||||
node("x1", "h100-80g", 8, "ib"),
|
||||
)
|
||||
p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack}
|
||||
|
||||
res := New(st).Place([]Profile{p}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("not placed:\n%s", res.Explain())
|
||||
}
|
||||
for _, a := range res.Placement.Nodes {
|
||||
if a.NodeID != "b1" && a.NodeID != "b2" {
|
||||
t.Errorf("a shard on %s, outside rack r2", a.NodeID)
|
||||
}
|
||||
}
|
||||
explain := res.Explain()
|
||||
if !strings.Contains(explain, "x1: has no llmgw.io/rack label, and the profile keeps its nodes within one rack") {
|
||||
t.Errorf("the unlabelled node should be named:\n%s", explain)
|
||||
}
|
||||
if !strings.Contains(explain, "needs 2 nodes within one rack, rack r1 has 1") {
|
||||
t.Errorf("the rack too small should be named:\n%s", explain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProfileKeptWithinARackIsNotPlacedAcrossRacks(t *testing.T) {
|
||||
st := store(at(node("a1", "h100-80g", 8, "ib"), "r1"), at(node("b1", "h100-80g", 8, "ib"), "r2"))
|
||||
p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack}
|
||||
if res := New(st).Place([]Profile{p}, nil); res.Placement != nil {
|
||||
t.Fatal("a replica kept within one rack was placed across two")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoCapIsEverExceeded fills random clusters until nothing more fits, and
|
||||
// checks every capped domain stayed within its limit — and that a cluster
|
||||
// whose draw nobody knows is filled as if it had no caps at all.
|
||||
func TestNoCapIsEverExceeded(t *testing.T) {
|
||||
rng := rand.New(rand.NewPCG(3, 4))
|
||||
for iter := range 200 {
|
||||
st := inventory.NewStore()
|
||||
known := rng.IntN(4) != 0
|
||||
cards := 0
|
||||
for i := range 2 + rng.IntN(6) {
|
||||
n := at(node(fmt.Sprintf("n%d", i), "h100-80g", 1+rng.IntN(8), "ib"),
|
||||
fmt.Sprintf("r%d", rng.IntN(3)), fmt.Sprintf("p%d", rng.IntN(2)))
|
||||
if known {
|
||||
n = withPowerLimit(n, float64(300+rng.IntN(500)), 0)
|
||||
}
|
||||
cards += len(n.GPUs)
|
||||
st.Upsert(n)
|
||||
}
|
||||
pol := facility.Policy{Margin: float64(rng.IntN(20)) / 100, Budgets: map[facility.Domain]facility.Budget{}}
|
||||
for _, d := range []facility.Domain{"rack:r0", "rack:r1", "rack:r2", "pdu:p0", "pdu:p1"} {
|
||||
if rng.IntN(2) == 0 {
|
||||
pol.Budgets[d] = facility.Budget{CapWatts: float64(1000 + rng.IntN(8000))}
|
||||
}
|
||||
}
|
||||
st.SetFacility(pol)
|
||||
|
||||
for k := range 60 {
|
||||
p := Profile{Name: "gen", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1 + rng.IntN(2)}
|
||||
res := New(st).Place([]Profile{p}, nil)
|
||||
if res.Placement == nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range res.Placement.Nodes {
|
||||
st.Allocate(a.NodeID, inventory.Allocation{ShardID: fmt.Sprintf("s%d-%d", k, a.Rank), GPUIndices: a.GPUIndices})
|
||||
}
|
||||
}
|
||||
for d, u := range st.PowerUse(time.Now()) {
|
||||
if u.Limit > 0 && u.Used > u.Limit+1e-6 {
|
||||
t.Fatalf("iteration %d: %s uses %.0f W of a %.0f W limit", iter, d, u.Used, u.Limit)
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
free := 0
|
||||
for _, n := range st.Nodes() {
|
||||
free += len(st.FreeGPUs(n.ID))
|
||||
}
|
||||
if free != 0 {
|
||||
t.Fatalf("iteration %d: %d of %d cards left free with no draw known", iter, free, cards)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,8 +16,10 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
@@ -45,6 +47,13 @@ type Profile struct {
|
||||
// Source is the model's weights, as the engine would be given it. Used to
|
||||
// prefer nodes that already hold them; empty asks for no preference.
|
||||
Source string
|
||||
// PowerWatts is what a replica of this profile draws, split evenly over
|
||||
// its nodes. Zero derives it from the cards' power limits and the cores.
|
||||
PowerWatts float64
|
||||
// KeepWithin keeps a multi-node replica's nodes inside one domain of
|
||||
// this level, rack or row, where the fabric or the cabling needs them
|
||||
// close. Empty is no constraint.
|
||||
KeepWithin facility.Level
|
||||
}
|
||||
|
||||
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
|
||||
@@ -81,6 +90,9 @@ type NodeAssignment struct {
|
||||
GPUIndices []int
|
||||
// HCAs are the adapters of the chosen fabric only.
|
||||
HCAs []string
|
||||
// PowerW is the draw the profile declared for this shard, to be recorded
|
||||
// with its allocation; zero when the profile declared none.
|
||||
PowerW float64
|
||||
}
|
||||
|
||||
// Placement is a decision: which profile, which fabric, which nodes.
|
||||
@@ -139,6 +151,9 @@ func (r Result) Explain() string {
|
||||
// Scheduler places replicas onto an inventory.
|
||||
type Scheduler struct {
|
||||
inv *inventory.Store
|
||||
// now is the clock power readings are judged fresh by; nil is the wall
|
||||
// clock.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// New builds a scheduler over an inventory.
|
||||
@@ -148,6 +163,11 @@ func New(inv *inventory.Store) *Scheduler { return &Scheduler{inv: inv} }
|
||||
//
|
||||
// avoid lists nodes already hosting this model, so replicas of one model spread
|
||||
// across failure domains instead of stacking on one machine.
|
||||
//
|
||||
// The rejections of the profile that was placed are kept with the placement,
|
||||
// not only those of the profiles before it: a replica placed in the second
|
||||
// rack because the first was held or at its power cap is placed somewhere
|
||||
// nobody expected, and explain-placement is where the operator asks why.
|
||||
func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
|
||||
var res Result
|
||||
|
||||
@@ -168,6 +188,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
|
||||
if p.Nodes == 1 {
|
||||
if pl, rej := s.placeSingleNode(p, avoid); pl != nil {
|
||||
res.Placement = pl
|
||||
res.Rejections = append(res.Rejections, rej...)
|
||||
return res
|
||||
} else {
|
||||
res.Rejections = append(res.Rejections, rej...)
|
||||
@@ -176,6 +197,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
|
||||
}
|
||||
if pl, rej := s.placeMultiNode(p, avoid); pl != nil {
|
||||
res.Placement = pl
|
||||
res.Rejections = append(res.Rejections, rej...)
|
||||
return res
|
||||
} else {
|
||||
res.Rejections = append(res.Rejections, rej...)
|
||||
@@ -212,17 +234,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
|
||||
len(s.inv.Allocations(candidates[j].node.ID))
|
||||
})
|
||||
preferNodesHoldingWeights(p.Source, candidates)
|
||||
s.spreadAcrossDomains(candidates, avoid)
|
||||
return &Placement{
|
||||
Profile: p,
|
||||
Pool: "none/cpu",
|
||||
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}},
|
||||
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0, PowerW: p.declaredShardWatts()}},
|
||||
}, rejections
|
||||
}
|
||||
|
||||
// Last, so it takes precedence: moving tens of gigabytes costs more than an
|
||||
// imperfectly packed node, and with isolated fabrics it may cost a
|
||||
// cross-fabric copy over the management network.
|
||||
// After packing, so it takes precedence: moving tens of gigabytes costs
|
||||
// more than an imperfectly packed node, and with isolated fabrics it may
|
||||
// cost a cross-fabric copy over the management network.
|
||||
preferNodesHoldingWeights(p.Source, candidates)
|
||||
// And racks over weights: a second replica in the rack of the first is
|
||||
// one breaker away from none, which no copy saved is worth.
|
||||
s.spreadAcrossDomains(candidates, avoid)
|
||||
|
||||
best := candidates[0]
|
||||
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode, nil)
|
||||
@@ -237,7 +263,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
|
||||
return &Placement{
|
||||
Profile: p,
|
||||
Pool: "none/" + p.classLabel(),
|
||||
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}},
|
||||
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus, PowerW: p.declaredShardWatts()}},
|
||||
}, rejections
|
||||
}
|
||||
|
||||
@@ -273,8 +299,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
if group.key != "" && group.split {
|
||||
pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel()
|
||||
}
|
||||
if pl := s.placeOnFabric(p, fabric, pool, group.nodes, &rejections); pl != nil {
|
||||
return pl, rejections
|
||||
for _, within := range withinGroups(p, pool, group.nodes, &rejections) {
|
||||
if pl := s.placeOnFabric(p, fabric, pool, within, &rejections); pl != nil {
|
||||
return pl, rejections
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -325,8 +353,10 @@ func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejec
|
||||
})
|
||||
for _, d := range keys {
|
||||
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
|
||||
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
|
||||
return pl
|
||||
for _, within := range withinGroups(p, pool, by[d], rejections) {
|
||||
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, within, rejections); pl != nil {
|
||||
return pl
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -416,8 +446,22 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
|
||||
// together, a degraded cable last.
|
||||
candidates = byLeaf(candidates, hints(s.inv))
|
||||
|
||||
// Each node passed the power budgets on its own; together, two shards in
|
||||
// one rack may not. Nodes whose shard would take a domain past its cap
|
||||
// with the shards already chosen are passed over for the next.
|
||||
budget := s.newBudget()
|
||||
assignments := make([]NodeAssignment, 0, p.Nodes)
|
||||
for rank, c := range candidates[:p.Nodes] {
|
||||
for _, c := range candidates {
|
||||
if len(assignments) == p.Nodes {
|
||||
break
|
||||
}
|
||||
if why := budget.check(c.node, s.shardWatts(p, c)); why != "" {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
Profile: p.Name, Pool: pool, Node: c.node.ID,
|
||||
Reason: why + ", with this replica's other shards",
|
||||
})
|
||||
continue
|
||||
}
|
||||
gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric))
|
||||
if gpus == nil {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
@@ -426,14 +470,23 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
|
||||
})
|
||||
return nil
|
||||
}
|
||||
budget.add(c.node, s.shardWatts(p, c))
|
||||
assignments = append(assignments, NodeAssignment{
|
||||
NodeID: c.node.ID,
|
||||
Rank: rank,
|
||||
Rank: len(assignments),
|
||||
GPUIndices: gpus,
|
||||
// Only this fabric's adapters, never the node's others.
|
||||
HCAs: c.node.HCAsFor(fabric),
|
||||
HCAs: c.node.HCAsFor(fabric),
|
||||
PowerW: p.declaredShardWatts(),
|
||||
})
|
||||
}
|
||||
if len(assignments) < p.Nodes {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
Profile: p.Name, Pool: pool,
|
||||
Reason: fmt.Sprintf("needs %d nodes, only %d fit within the power budgets together", p.Nodes, len(assignments)),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}
|
||||
}
|
||||
|
||||
@@ -508,6 +561,8 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
}
|
||||
pool += "/" + p.classLabel()
|
||||
|
||||
pol := s.inv.Facility()
|
||||
budget := s.newBudget()
|
||||
var out []candidate
|
||||
for _, n := range s.inv.Nodes() {
|
||||
reject := func(reason string) {
|
||||
@@ -518,6 +573,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
reject("node is " + n.State.String())
|
||||
continue
|
||||
}
|
||||
// A domain held for maintenance takes nothing new, as a cordon of
|
||||
// every node in it would, and says which hold and why.
|
||||
if h, held := pol.HeldIn(n.Domains()); held {
|
||||
reject(heldReason(h))
|
||||
continue
|
||||
}
|
||||
if slices.Contains(avoid, n.ID) {
|
||||
reject("already hosts this model; spreading across failure domains")
|
||||
continue
|
||||
@@ -608,7 +669,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
|
||||
if p.CPUOnly() {
|
||||
// Every ready node can run it; there is no card to be free.
|
||||
out = append(out, candidate{node: n})
|
||||
c := candidate{node: n}
|
||||
if why := budget.check(n, s.shardWatts(p, c)); why != "" {
|
||||
reject(why)
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -624,7 +690,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
|
||||
reject(fmt.Sprintf("has %d free %s GPUs, needs %d", len(free), p.GPUClass, p.GPUsPerNode))
|
||||
continue
|
||||
}
|
||||
out = append(out, candidate{node: n, free: free})
|
||||
c := candidate{node: n, free: free}
|
||||
if why := budget.check(n, s.shardWatts(p, c)); why != "" {
|
||||
reject(why)
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user