Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -0,0 +1,176 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
// Location is where the node is, from its labels.
|
||||
func (n Node) Location() facility.Location { return facility.LocationOf(n.Labels) }
|
||||
|
||||
// Domains are the failure domains the node is in, largest first.
|
||||
func (n Node) Domains() []facility.Domain { return n.Location().Domains() }
|
||||
|
||||
// mergeLabels lays the operator's labels over the agent's. A copy, so
|
||||
// neither is written through.
|
||||
func mergeLabels(agent, operator map[string]string) map[string]string {
|
||||
if len(operator) == 0 {
|
||||
return maps.Clone(agent)
|
||||
}
|
||||
out := make(map[string]string, len(agent)+len(operator))
|
||||
maps.Copy(out, agent)
|
||||
maps.Copy(out, operator)
|
||||
return out
|
||||
}
|
||||
|
||||
// SetFacility puts the facility's holds and power budgets in force for
|
||||
// every placement from now on.
|
||||
func (s *Store) SetFacility(p facility.Policy) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.facility = p.Clone()
|
||||
}
|
||||
|
||||
// Facility is the policy in force; a copy, so a caller may change it.
|
||||
func (s *Store) Facility() facility.Policy {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.facility.Clone()
|
||||
}
|
||||
|
||||
// SetOperatorLabels replaces the labels set through the control plane for a
|
||||
// node, which may not have registered yet: they are laid over its agent's
|
||||
// when it does.
|
||||
func (s *Store) SetOperatorLabels(id string, labels map[string]string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.operatorLabels == nil {
|
||||
s.operatorLabels = map[string]map[string]string{}
|
||||
}
|
||||
if len(labels) == 0 {
|
||||
delete(s.operatorLabels, id)
|
||||
} else {
|
||||
s.operatorLabels[id] = maps.Clone(labels)
|
||||
}
|
||||
if n, ok := s.nodes[id]; ok {
|
||||
n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[id])
|
||||
}
|
||||
}
|
||||
|
||||
// OperatorLabels are the labels set through the control plane for a node.
|
||||
func (s *Store) OperatorLabels(id string) map[string]string {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return maps.Clone(s.operatorLabels[id])
|
||||
}
|
||||
|
||||
// PowerUse is one domain's draw as placement counts it.
|
||||
type PowerUse struct {
|
||||
Domain facility.Domain
|
||||
// Nodes are the domain's nodes, sorted.
|
||||
Nodes []string
|
||||
// Reserved is what the replicas placed in the domain are estimated to
|
||||
// draw, with each node's own draw and the domain's other equipment.
|
||||
Reserved float64
|
||||
// Measured is what the nodes' sensors read, with the same additions,
|
||||
// for the nodes that read anything; the others count what is reserved.
|
||||
Measured float64
|
||||
// Used is what placement counts against the cap: per node, the larger
|
||||
// of reserved and measured, so an estimate that was too low is
|
||||
// corrected by the room it stops offering rather than by a breaker.
|
||||
Used float64
|
||||
// Limit is the cap less the margin; zero is no cap.
|
||||
Limit float64
|
||||
Budget facility.Budget
|
||||
}
|
||||
|
||||
// PowerUse is every domain any node is in, or that has a budget, by domain.
|
||||
func (s *Store) PowerUse(now time.Time) map[facility.Domain]PowerUse {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
pol := s.facility
|
||||
out := map[facility.Domain]PowerUse{}
|
||||
for d, b := range pol.Budgets {
|
||||
out[d] = PowerUse{Domain: d, Budget: b, Limit: pol.Limit(d),
|
||||
Reserved: b.OtherWatts, Measured: b.OtherWatts, Used: b.OtherWatts}
|
||||
}
|
||||
ids := slices.Sorted(maps.Keys(s.nodes))
|
||||
for _, id := range ids {
|
||||
n := s.nodes[id]
|
||||
reserved := 0.0
|
||||
for _, a := range s.allocs[id] {
|
||||
reserved += AllocationWatts(*n, a, pol)
|
||||
}
|
||||
measured, known := MeasuredWatts(*n, now, pol.Fresh())
|
||||
used := reserved
|
||||
if known && measured > used {
|
||||
used = measured
|
||||
}
|
||||
if !known {
|
||||
measured = reserved
|
||||
}
|
||||
for _, d := range n.Domains() {
|
||||
u, ok := out[d]
|
||||
if !ok {
|
||||
u = PowerUse{Domain: d}
|
||||
}
|
||||
u.Nodes = append(u.Nodes, id)
|
||||
u.Reserved += pol.NodeWatts + reserved
|
||||
u.Measured += pol.NodeWatts + measured
|
||||
u.Used += pol.NodeWatts + used
|
||||
out[d] = u
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AllocationWatts is what one shard is estimated to draw: what its profile
|
||||
// declared, or else each of its cards at the power limit the card reports
|
||||
// (the policy's figure for a card that reports none) and each of its cores
|
||||
// at the policy's figure.
|
||||
func AllocationWatts(n Node, a Allocation, pol facility.Policy) float64 {
|
||||
if a.PowerW > 0 {
|
||||
return a.PowerW
|
||||
}
|
||||
w := float64(a.CPUCores) * pol.CPUCoreWatts
|
||||
for _, i := range a.GPUIndices {
|
||||
w += GPUWatts(n, i, pol)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// GPUWatts is one card's estimated draw: the power limit it reports, else
|
||||
// the policy's figure for a card.
|
||||
func GPUWatts(n Node, index int, pol facility.Policy) float64 {
|
||||
if n.Load != nil {
|
||||
for _, g := range n.Load.GPUs {
|
||||
if g.Index == index && g.PowerLimitW > 0 {
|
||||
return g.PowerLimitW
|
||||
}
|
||||
}
|
||||
}
|
||||
return pol.GPUWatts
|
||||
}
|
||||
|
||||
// MeasuredWatts is what the node's sensors read at its last heartbeat: its
|
||||
// cards and its CPU packages. Known is false when the reading is older than
|
||||
// fresh, or the node has no power sensor at all, which a VM never has.
|
||||
func MeasuredWatts(n Node, now time.Time, fresh time.Duration) (w float64, known bool) {
|
||||
if n.Load == nil || now.Sub(n.Load.At) > fresh {
|
||||
return 0, false
|
||||
}
|
||||
for _, g := range n.Load.GPUs {
|
||||
if g.PowerW > 0 {
|
||||
w += g.PowerW
|
||||
known = true
|
||||
}
|
||||
}
|
||||
if n.Load.CPUPowerW > 0 {
|
||||
w += n.Load.CPUPowerW
|
||||
known = true
|
||||
}
|
||||
return w, known
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
func TestOperatorLabelsWinOverTheAgentsAndSurviveItsHeartbeats(t *testing.T) {
|
||||
s := NewStore()
|
||||
// Set before the node registers: labelling a rack's machines comes first.
|
||||
s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"})
|
||||
s.Upsert(Node{ID: "n1", Labels: map[string]string{"llmgw.io/rack": "r1", "role": "gpu"}})
|
||||
|
||||
n, _ := s.Get("n1")
|
||||
if n.Labels["llmgw.io/rack"] != "r2" || n.Labels["role"] != "gpu" {
|
||||
t.Fatalf("labels %v: the operator's rack over the agent's, the agent's role kept", n.Labels)
|
||||
}
|
||||
// A heartbeat hands back the node it read.
|
||||
s.Upsert(n)
|
||||
s.Upsert(n)
|
||||
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r2" || n.AgentLabels["llmgw.io/rack"] != "r1" {
|
||||
t.Fatalf("after heartbeats: labels %v, agent's %v", n.Labels, n.AgentLabels)
|
||||
}
|
||||
|
||||
s.SetOperatorLabels("n1", nil)
|
||||
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r1" {
|
||||
t.Errorf("the operator's label removed, rack is %q; want the agent's r1 back", n.Labels["llmgw.io/rack"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOperatorLabelOnANodeWithNoAgentLabelsIsNotTakenForTheAgents(t *testing.T) {
|
||||
s := NewStore()
|
||||
s.Upsert(Node{ID: "n1"})
|
||||
s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"})
|
||||
n, _ := s.Get("n1")
|
||||
s.Upsert(n)
|
||||
s.SetOperatorLabels("n1", nil)
|
||||
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "" {
|
||||
t.Errorf("removed, but the node still carries rack %q", n.Labels["llmgw.io/rack"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestADomainsPowerIsTheLargerOfReservedAndMeasuredNodeByNode(t *testing.T) {
|
||||
s := NewStore()
|
||||
gpus := []discovery.GPU{{Index: 0}, {Index: 1}}
|
||||
rack := map[string]string{"llmgw.io/rack": "r1"}
|
||||
now := time.Now()
|
||||
// a: two cards placed at a 700 W limit, drawing 300 each: 1400 reserved.
|
||||
s.Upsert(Node{ID: "a", Labels: rack, GPUs: gpus})
|
||||
s.Upsert(Node{ID: "b", Labels: rack, GPUs: gpus})
|
||||
setLoad(s, "a", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerLimitW: 700, PowerW: 300}, {Index: 1, PowerLimitW: 700, PowerW: 300}}})
|
||||
// b: nothing placed, but drawing 500 W.
|
||||
setLoad(s, "b", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerW: 250}, {Index: 1, PowerW: 250}}})
|
||||
s.Allocate("a", Allocation{ShardID: "x", GPUIndices: []int{0, 1}})
|
||||
s.SetFacility(facility.Policy{NodeWatts: 100, Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000, OtherWatts: 50}}})
|
||||
|
||||
u := s.PowerUse(now)["rack:r1"]
|
||||
if u.Reserved != 50+100+1400+100+0 {
|
||||
t.Errorf("reserved %v", u.Reserved)
|
||||
}
|
||||
if u.Measured != 50+100+600+100+500 {
|
||||
t.Errorf("measured %v", u.Measured)
|
||||
}
|
||||
if u.Used != 50+100+1400+100+500 {
|
||||
t.Errorf("used %v: a's reservation and b's reading", u.Used)
|
||||
}
|
||||
if len(u.Nodes) != 2 || u.Limit != 5000 {
|
||||
t.Errorf("nodes %v, limit %v", u.Nodes, u.Limit)
|
||||
}
|
||||
}
|
||||
|
||||
func setLoad(s *Store, id string, l *Load) {
|
||||
n, _ := s.Get(id)
|
||||
n.Load = l
|
||||
s.Upsert(n)
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
// NodeState is where a node stands.
|
||||
@@ -71,10 +72,17 @@ type FabricAttachment struct {
|
||||
|
||||
// Node is one machine in the cluster.
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Hostname string `json:"hostname"`
|
||||
MgmtAddr string `json:"mgmt_addr"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Hostname string `json:"hostname"`
|
||||
MgmtAddr string `json:"mgmt_addr"`
|
||||
// Labels are what placement matches: the agent's own, from agent.yaml,
|
||||
// with any an operator set through the control plane over them.
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
// AgentLabels are the labels the agent reported, kept apart so an
|
||||
// operator's label removed brings the agent's value back. Nil on a node
|
||||
// built anywhere but a registration; Upsert then takes Labels as the
|
||||
// agent's.
|
||||
AgentLabels map[string]string `json:"agent_labels,omitempty"`
|
||||
|
||||
// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
|
||||
// it, which is treated as no constraint — the same as every node before
|
||||
@@ -461,6 +469,10 @@ type Allocation struct {
|
||||
// CPUCores is the cores this shard was given to compute on, for a shard
|
||||
// with no GPU; zero for one that shares whatever is free.
|
||||
CPUCores int
|
||||
// PowerW is the draw this shard's profile declared; zero derives it from
|
||||
// the node's cards and the cores, which is what most profiles leave to
|
||||
// placement.
|
||||
PowerW float64
|
||||
}
|
||||
|
||||
// Store holds the cluster inventory. It is safe for concurrent use.
|
||||
@@ -473,6 +485,11 @@ type Store struct {
|
||||
smChanges []SMChange
|
||||
// scans are the latest ibnetdiscover of each physical fabric, by key.
|
||||
scans map[string]FabricScan
|
||||
// facility is the holds and power budgets placement reads.
|
||||
facility facility.Policy
|
||||
// operatorLabels are labels set through the control plane, by node,
|
||||
// laid over what each agent reports.
|
||||
operatorLabels map[string]map[string]string
|
||||
}
|
||||
|
||||
// NewStore returns an empty inventory.
|
||||
@@ -494,6 +511,7 @@ func (s *Store) Clone() *Store {
|
||||
for id, list := range s.allocs {
|
||||
c.allocs[id] = slices.Clone(list)
|
||||
}
|
||||
c.facility = s.facility.Clone()
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -564,6 +582,17 @@ func (s *Store) Upsert(n Node) {
|
||||
n.State = prev.State
|
||||
}
|
||||
}
|
||||
// A registration carries the agent's labels in Labels; a heartbeat hands
|
||||
// back the node it read, whose Labels already have the operator's in
|
||||
// them, and its AgentLabels set. Never nil after this, so the second case
|
||||
// is always told from the first.
|
||||
if n.AgentLabels == nil {
|
||||
n.AgentLabels = maps.Clone(n.Labels)
|
||||
if n.AgentLabels == nil {
|
||||
n.AgentLabels = map[string]string{}
|
||||
}
|
||||
}
|
||||
n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[n.ID])
|
||||
n.LastSeen = time.Now()
|
||||
s.nodes[n.ID] = &n
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user