Facility-aware placement: racks, PDUs, holds and power caps

Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:17:49 +02:00
1 parent d41104a452
commit 0d3eb82b9b
42 files changed
+3510 -63

No files matched your search

+176
View File
@@ -0,0 +1,176 @@
package inventory
import (
"maps"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
// Location is where the node is, from its labels.
func (n Node) Location() facility.Location { return facility.LocationOf(n.Labels) }
// Domains are the failure domains the node is in, largest first.
func (n Node) Domains() []facility.Domain { return n.Location().Domains() }
// mergeLabels lays the operator's labels over the agent's. A copy, so
// neither is written through.
func mergeLabels(agent, operator map[string]string) map[string]string {
if len(operator) == 0 {
return maps.Clone(agent)
}
out := make(map[string]string, len(agent)+len(operator))
maps.Copy(out, agent)
maps.Copy(out, operator)
return out
}
// SetFacility puts the facility's holds and power budgets in force for
// every placement from now on.
func (s *Store) SetFacility(p facility.Policy) {
s.mu.Lock()
defer s.mu.Unlock()
s.facility = p.Clone()
}
// Facility is the policy in force; a copy, so a caller may change it.
func (s *Store) Facility() facility.Policy {
s.mu.RLock()
defer s.mu.RUnlock()
return s.facility.Clone()
}
// SetOperatorLabels replaces the labels set through the control plane for a
// node, which may not have registered yet: they are laid over its agent's
// when it does.
func (s *Store) SetOperatorLabels(id string, labels map[string]string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.operatorLabels == nil {
s.operatorLabels = map[string]map[string]string{}
}
if len(labels) == 0 {
delete(s.operatorLabels, id)
} else {
s.operatorLabels[id] = maps.Clone(labels)
}
if n, ok := s.nodes[id]; ok {
n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[id])
}
}
// OperatorLabels are the labels set through the control plane for a node.
func (s *Store) OperatorLabels(id string) map[string]string {
s.mu.RLock()
defer s.mu.RUnlock()
return maps.Clone(s.operatorLabels[id])
}
// PowerUse is one domain's draw as placement counts it.
type PowerUse struct {
Domain facility.Domain
// Nodes are the domain's nodes, sorted.
Nodes []string
// Reserved is what the replicas placed in the domain are estimated to
// draw, with each node's own draw and the domain's other equipment.
Reserved float64
// Measured is what the nodes' sensors read, with the same additions,
// for the nodes that read anything; the others count what is reserved.
Measured float64
// Used is what placement counts against the cap: per node, the larger
// of reserved and measured, so an estimate that was too low is
// corrected by the room it stops offering rather than by a breaker.
Used float64
// Limit is the cap less the margin; zero is no cap.
Limit float64
Budget facility.Budget
}
// PowerUse is every domain any node is in, or that has a budget, by domain.
func (s *Store) PowerUse(now time.Time) map[facility.Domain]PowerUse {
s.mu.RLock()
defer s.mu.RUnlock()
pol := s.facility
out := map[facility.Domain]PowerUse{}
for d, b := range pol.Budgets {
out[d] = PowerUse{Domain: d, Budget: b, Limit: pol.Limit(d),
Reserved: b.OtherWatts, Measured: b.OtherWatts, Used: b.OtherWatts}
}
ids := slices.Sorted(maps.Keys(s.nodes))
for _, id := range ids {
n := s.nodes[id]
reserved := 0.0
for _, a := range s.allocs[id] {
reserved += AllocationWatts(*n, a, pol)
}
measured, known := MeasuredWatts(*n, now, pol.Fresh())
used := reserved
if known && measured > used {
used = measured
}
if !known {
measured = reserved
}
for _, d := range n.Domains() {
u, ok := out[d]
if !ok {
u = PowerUse{Domain: d}
}
u.Nodes = append(u.Nodes, id)
u.Reserved += pol.NodeWatts + reserved
u.Measured += pol.NodeWatts + measured
u.Used += pol.NodeWatts + used
out[d] = u
}
}
return out
}
// AllocationWatts is what one shard is estimated to draw: what its profile
// declared, or else each of its cards at the power limit the card reports
// (the policy's figure for a card that reports none) and each of its cores
// at the policy's figure.
func AllocationWatts(n Node, a Allocation, pol facility.Policy) float64 {
if a.PowerW > 0 {
return a.PowerW
}
w := float64(a.CPUCores) * pol.CPUCoreWatts
for _, i := range a.GPUIndices {
w += GPUWatts(n, i, pol)
}
return w
}
// GPUWatts is one card's estimated draw: the power limit it reports, else
// the policy's figure for a card.
func GPUWatts(n Node, index int, pol facility.Policy) float64 {
if n.Load != nil {
for _, g := range n.Load.GPUs {
if g.Index == index && g.PowerLimitW > 0 {
return g.PowerLimitW
}
}
}
return pol.GPUWatts
}
// MeasuredWatts is what the node's sensors read at its last heartbeat: its
// cards and its CPU packages. Known is false when the reading is older than
// fresh, or the node has no power sensor at all, which a VM never has.
func MeasuredWatts(n Node, now time.Time, fresh time.Duration) (w float64, known bool) {
if n.Load == nil || now.Sub(n.Load.At) > fresh {
return 0, false
}
for _, g := range n.Load.GPUs {
if g.PowerW > 0 {
w += g.PowerW
known = true
}
}
if n.Load.CPUPowerW > 0 {
w += n.Load.CPUPowerW
known = true
}
return w, known
}
@@ -0,0 +1,79 @@
package inventory
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
func TestOperatorLabelsWinOverTheAgentsAndSurviveItsHeartbeats(t *testing.T) {
s := NewStore()
// Set before the node registers: labelling a rack's machines comes first.
s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"})
s.Upsert(Node{ID: "n1", Labels: map[string]string{"llmgw.io/rack": "r1", "role": "gpu"}})
n, _ := s.Get("n1")
if n.Labels["llmgw.io/rack"] != "r2" || n.Labels["role"] != "gpu" {
t.Fatalf("labels %v: the operator's rack over the agent's, the agent's role kept", n.Labels)
}
// A heartbeat hands back the node it read.
s.Upsert(n)
s.Upsert(n)
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r2" || n.AgentLabels["llmgw.io/rack"] != "r1" {
t.Fatalf("after heartbeats: labels %v, agent's %v", n.Labels, n.AgentLabels)
}
s.SetOperatorLabels("n1", nil)
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r1" {
t.Errorf("the operator's label removed, rack is %q; want the agent's r1 back", n.Labels["llmgw.io/rack"])
}
}
func TestAnOperatorLabelOnANodeWithNoAgentLabelsIsNotTakenForTheAgents(t *testing.T) {
s := NewStore()
s.Upsert(Node{ID: "n1"})
s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"})
n, _ := s.Get("n1")
s.Upsert(n)
s.SetOperatorLabels("n1", nil)
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "" {
t.Errorf("removed, but the node still carries rack %q", n.Labels["llmgw.io/rack"])
}
}
func TestADomainsPowerIsTheLargerOfReservedAndMeasuredNodeByNode(t *testing.T) {
s := NewStore()
gpus := []discovery.GPU{{Index: 0}, {Index: 1}}
rack := map[string]string{"llmgw.io/rack": "r1"}
now := time.Now()
// a: two cards placed at a 700 W limit, drawing 300 each: 1400 reserved.
s.Upsert(Node{ID: "a", Labels: rack, GPUs: gpus})
s.Upsert(Node{ID: "b", Labels: rack, GPUs: gpus})
setLoad(s, "a", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerLimitW: 700, PowerW: 300}, {Index: 1, PowerLimitW: 700, PowerW: 300}}})
// b: nothing placed, but drawing 500 W.
setLoad(s, "b", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerW: 250}, {Index: 1, PowerW: 250}}})
s.Allocate("a", Allocation{ShardID: "x", GPUIndices: []int{0, 1}})
s.SetFacility(facility.Policy{NodeWatts: 100, Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000, OtherWatts: 50}}})
u := s.PowerUse(now)["rack:r1"]
if u.Reserved != 50+100+1400+100+0 {
t.Errorf("reserved %v", u.Reserved)
}
if u.Measured != 50+100+600+100+500 {
t.Errorf("measured %v", u.Measured)
}
if u.Used != 50+100+1400+100+500 {
t.Errorf("used %v: a's reservation and b's reading", u.Used)
}
if len(u.Nodes) != 2 || u.Limit != 5000 {
t.Errorf("nodes %v, limit %v", u.Nodes, u.Limit)
}
}
func setLoad(s *Store, id string, l *Load) {
n, _ := s.Get(id)
n.Load = l
s.Upsert(n)
}
+33 -4
View File
@@ -11,6 +11,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
// NodeState is where a node stands.
@@ -71,10 +72,17 @@ type FabricAttachment struct {
// Node is one machine in the cluster.
type Node struct {
ID string `json:"id"`
Hostname string `json:"hostname"`
MgmtAddr string `json:"mgmt_addr"`
Labels map[string]string `json:"labels,omitempty"`
ID string `json:"id"`
Hostname string `json:"hostname"`
MgmtAddr string `json:"mgmt_addr"`
// Labels are what placement matches: the agent's own, from agent.yaml,
// with any an operator set through the control plane over them.
Labels map[string]string `json:"labels,omitempty"`
// AgentLabels are the labels the agent reported, kept apart so an
// operator's label removed brings the agent's value back. Nil on a node
// built anywhere but a registration; Upsert then takes Labels as the
// agent's.
AgentLabels map[string]string `json:"agent_labels,omitempty"`
// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
// it, which is treated as no constraint — the same as every node before
@@ -461,6 +469,10 @@ type Allocation struct {
// CPUCores is the cores this shard was given to compute on, for a shard
// with no GPU; zero for one that shares whatever is free.
CPUCores int
// PowerW is the draw this shard's profile declared; zero derives it from
// the node's cards and the cores, which is what most profiles leave to
// placement.
PowerW float64
}
// Store holds the cluster inventory. It is safe for concurrent use.
@@ -473,6 +485,11 @@ type Store struct {
smChanges []SMChange
// scans are the latest ibnetdiscover of each physical fabric, by key.
scans map[string]FabricScan
// facility is the holds and power budgets placement reads.
facility facility.Policy
// operatorLabels are labels set through the control plane, by node,
// laid over what each agent reports.
operatorLabels map[string]map[string]string
}
// NewStore returns an empty inventory.
@@ -494,6 +511,7 @@ func (s *Store) Clone() *Store {
for id, list := range s.allocs {
c.allocs[id] = slices.Clone(list)
}
c.facility = s.facility.Clone()
return c
}
@@ -564,6 +582,17 @@ func (s *Store) Upsert(n Node) {
n.State = prev.State
}
}
// A registration carries the agent's labels in Labels; a heartbeat hands
// back the node it read, whose Labels already have the operator's in
// them, and its AgentLabels set. Never nil after this, so the second case
// is always told from the first.
if n.AgentLabels == nil {
n.AgentLabels = maps.Clone(n.Labels)
if n.AgentLabels == nil {
n.AgentLabels = map[string]string{}
}
}
n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[n.ID])
n.LastSeen = time.Now()
s.nodes[n.ID] = &n
}