Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -0,0 +1,268 @@
|
||||
// Package facility is where nodes are in the building, and what the building
|
||||
// allows: the racks, rows, rooms and power distribution units a node belongs
|
||||
// to, which of them an operator has taken out of service, and how much power
|
||||
// each may draw.
|
||||
//
|
||||
// Placement reads it to keep replicas of one model off a single rack or PDU,
|
||||
// to keep a replica's shards inside one domain when its profile asks, to place
|
||||
// nothing in a domain held for maintenance, and to refuse a replica that would
|
||||
// take a domain past its power cap. Everything here is optional: a node with
|
||||
// no location labels and a cluster with no facility configuration place
|
||||
// exactly as they did before.
|
||||
package facility
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Level is a kind of failure domain, from the largest to the smallest.
|
||||
type Level string
|
||||
|
||||
const (
|
||||
Site Level = "site"
|
||||
Room Level = "room"
|
||||
Row Level = "row"
|
||||
Rack Level = "rack"
|
||||
// PDU is a power distribution unit. A node may be on several, one per
|
||||
// feed of a dual-fed rack, so its label takes a comma separated list.
|
||||
PDU Level = "pdu"
|
||||
)
|
||||
|
||||
// Levels are every level, the building's tree first and power after it.
|
||||
var Levels = []Level{Site, Room, Row, Rack, PDU}
|
||||
|
||||
// LabelPrefix starts the node labels that say where a node is.
|
||||
const LabelPrefix = "llmgw.io/"
|
||||
|
||||
// Label is the node label a level is read from: llmgw.io/rack.
|
||||
func (l Level) Label() string { return LabelPrefix + string(l) }
|
||||
|
||||
// ParseLevel reads a level's name, as a profile or a command gives it.
|
||||
func ParseLevel(s string) (Level, error) {
|
||||
l := Level(strings.ToLower(strings.TrimSpace(s)))
|
||||
if slices.Contains(Levels, l) {
|
||||
return l, nil
|
||||
}
|
||||
return "", fmt.Errorf("%q is not a facility level: use site, room, row, rack or pdu", s)
|
||||
}
|
||||
|
||||
// Domain names one failure domain as level:name, rack:r12-07. Names are taken
|
||||
// as the labels give them, so a rack is one domain across the whole cluster:
|
||||
// two rooms that both have a rack r1 must label them apart.
|
||||
type Domain string
|
||||
|
||||
// Of builds a domain from its level and name.
|
||||
func Of(l Level, name string) Domain { return Domain(string(l) + ":" + name) }
|
||||
|
||||
// Level is the domain's level.
|
||||
func (d Domain) Level() Level {
|
||||
l, _, _ := strings.Cut(string(d), ":")
|
||||
return Level(l)
|
||||
}
|
||||
|
||||
// Name is the domain's name within its level.
|
||||
func (d Domain) Name() string {
|
||||
_, n, _ := strings.Cut(string(d), ":")
|
||||
return n
|
||||
}
|
||||
|
||||
// String is level:name; human text says "rack r12-07".
|
||||
func (d Domain) String() string { return string(d) }
|
||||
|
||||
// Human is the domain as a sentence says it: rack r12-07.
|
||||
func (d Domain) Human() string { return string(d.Level()) + " " + d.Name() }
|
||||
|
||||
// ParseDomain reads level:name, the way commands and configuration name a
|
||||
// domain.
|
||||
func ParseDomain(s string) (Domain, error) {
|
||||
l, n, ok := strings.Cut(strings.TrimSpace(s), ":")
|
||||
if !ok || strings.TrimSpace(n) == "" {
|
||||
return "", fmt.Errorf("%q is not a domain: name one as level:name, like rack:r12-07", s)
|
||||
}
|
||||
lv, err := ParseLevel(l)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return Of(lv, strings.TrimSpace(n)), nil
|
||||
}
|
||||
|
||||
// Location is where a node is, read from its labels. Empty fields are not
|
||||
// known, which places as it always did.
|
||||
type Location struct {
|
||||
Site string `json:"site,omitempty"`
|
||||
Room string `json:"room,omitempty"`
|
||||
Row string `json:"row,omitempty"`
|
||||
Rack string `json:"rack,omitempty"`
|
||||
PDUs []string `json:"pdus,omitempty"`
|
||||
}
|
||||
|
||||
// LocationOf reads a node's labels.
|
||||
func LocationOf(labels map[string]string) Location {
|
||||
get := func(l Level) string { return strings.TrimSpace(labels[l.Label()]) }
|
||||
loc := Location{Site: get(Site), Room: get(Room), Row: get(Row), Rack: get(Rack)}
|
||||
for _, p := range strings.Split(get(PDU), ",") {
|
||||
if p = strings.TrimSpace(p); p != "" && !slices.Contains(loc.PDUs, p) {
|
||||
loc.PDUs = append(loc.PDUs, p)
|
||||
}
|
||||
}
|
||||
return loc
|
||||
}
|
||||
|
||||
// Empty reports whether nothing is known about where the node is.
|
||||
func (l Location) Empty() bool {
|
||||
return l.Site == "" && l.Room == "" && l.Row == "" && l.Rack == "" && len(l.PDUs) == 0
|
||||
}
|
||||
|
||||
// Domains are every domain the location is in, largest first.
|
||||
func (l Location) Domains() []Domain {
|
||||
var out []Domain
|
||||
add := func(lv Level, name string) {
|
||||
if name != "" {
|
||||
out = append(out, Of(lv, name))
|
||||
}
|
||||
}
|
||||
add(Site, l.Site)
|
||||
add(Room, l.Room)
|
||||
add(Row, l.Row)
|
||||
add(Rack, l.Rack)
|
||||
for _, p := range l.PDUs {
|
||||
add(PDU, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// At is the domain at one level, or empty. For PDUs it is the first.
|
||||
func (l Location) At(lv Level) Domain {
|
||||
for _, d := range l.Domains() {
|
||||
if d.Level() == lv {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Short is the location in a few characters for a table: the rack, else the
|
||||
// row, room or site, with the PDUs after a slash.
|
||||
func (l Location) Short() string {
|
||||
s := ""
|
||||
switch {
|
||||
case l.Rack != "":
|
||||
s = l.Rack
|
||||
case l.Row != "":
|
||||
s = "row " + l.Row
|
||||
case l.Room != "":
|
||||
s = l.Room
|
||||
case l.Site != "":
|
||||
s = l.Site
|
||||
}
|
||||
if len(l.PDUs) > 0 {
|
||||
if s != "" {
|
||||
s += " "
|
||||
}
|
||||
s += "pdu " + strings.Join(l.PDUs, ",")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Hold takes a domain out of placement: maintenance, a cooling fault, work on
|
||||
// a PDU. What runs there keeps running unless Drain is set.
|
||||
type Hold struct {
|
||||
Domain Domain `json:"domain"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Drain bool `json:"drain,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Budget is a domain's power cap.
|
||||
type Budget struct {
|
||||
// CapWatts is what the domain may draw. For a dual-fed rack it is what
|
||||
// one feed can carry alone, since that is what the rack must survive.
|
||||
CapWatts float64
|
||||
// OtherWatts is drawn in the domain by things llmgw does not place on:
|
||||
// switches, storage, nodes outside the cluster.
|
||||
OtherWatts float64
|
||||
}
|
||||
|
||||
// Policy is the facility's rules as placement reads them. The zero value
|
||||
// holds nothing and caps nothing.
|
||||
type Policy struct {
|
||||
// Holds are the domains taken out of placement.
|
||||
Holds map[Domain]Hold
|
||||
// Budgets are the domains with a power cap.
|
||||
Budgets map[Domain]Budget
|
||||
// Margin is the share of every cap kept free, 0.1 for ten percent.
|
||||
Margin float64
|
||||
// GPUWatts is a card's draw when its own power limit is not known.
|
||||
GPUWatts float64
|
||||
// CPUCoreWatts is what a core reserved by a CPU replica draws.
|
||||
CPUCoreWatts float64
|
||||
// NodeWatts is what a node draws beyond the GPUs and the cores given to
|
||||
// replicas: its idle draw, fans, memory and disks. Counted for every node
|
||||
// in a capped domain, busy or not.
|
||||
NodeWatts float64
|
||||
// MeasuredFresh is how old a node's power reading may be and still be
|
||||
// counted. Zero is two minutes.
|
||||
MeasuredFresh time.Duration
|
||||
}
|
||||
|
||||
// Clone is a copy whose maps are its own.
|
||||
func (p Policy) Clone() Policy {
|
||||
c := p
|
||||
if p.Holds != nil {
|
||||
c.Holds = make(map[Domain]Hold, len(p.Holds))
|
||||
for k, v := range p.Holds {
|
||||
c.Holds[k] = v
|
||||
}
|
||||
}
|
||||
if p.Budgets != nil {
|
||||
c.Budgets = make(map[Domain]Budget, len(p.Budgets))
|
||||
for k, v := range p.Budgets {
|
||||
c.Budgets[k] = v
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// HeldIn is the first hold among a node's domains, largest first.
|
||||
func (p Policy) HeldIn(domains []Domain) (Hold, bool) {
|
||||
for _, d := range domains {
|
||||
if h, ok := p.Holds[d]; ok {
|
||||
return h, true
|
||||
}
|
||||
}
|
||||
return Hold{}, false
|
||||
}
|
||||
|
||||
// Limit is the most a domain may draw, its cap less the margin; zero with
|
||||
// no cap.
|
||||
func (p Policy) Limit(d Domain) float64 {
|
||||
b, ok := p.Budgets[d]
|
||||
if !ok || b.CapWatts <= 0 {
|
||||
return 0
|
||||
}
|
||||
m := p.Margin
|
||||
if m < 0 || m >= 1 {
|
||||
m = 0
|
||||
}
|
||||
return b.CapWatts * (1 - m)
|
||||
}
|
||||
|
||||
// Fresh is how old a power reading may be.
|
||||
func (p Policy) Fresh() time.Duration {
|
||||
if p.MeasuredFresh > 0 {
|
||||
return p.MeasuredFresh
|
||||
}
|
||||
return 2 * time.Minute
|
||||
}
|
||||
|
||||
// KW renders watts as a sentence about a rack would.
|
||||
func KW(w float64) string {
|
||||
if w >= 1000 {
|
||||
return fmt.Sprintf("%.1f kW", w/1000)
|
||||
}
|
||||
return fmt.Sprintf("%.0f W", w)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package facility
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestALocationIsReadFromTheWellKnownLabels(t *testing.T) {
|
||||
loc := LocationOf(map[string]string{
|
||||
"llmgw.io/site": "sto1", "llmgw.io/room": "hall-b", "llmgw.io/row": "12",
|
||||
"llmgw.io/rack": "r12-07", "llmgw.io/pdu": "r12-07-a, r12-07-b,r12-07-a", "role": "gpu",
|
||||
})
|
||||
want := []Domain{"site:sto1", "room:hall-b", "row:12", "rack:r12-07", "pdu:r12-07-a", "pdu:r12-07-b"}
|
||||
if got := loc.Domains(); !slices.Equal(got, want) {
|
||||
t.Errorf("domains %v, want %v", got, want)
|
||||
}
|
||||
if loc.Short() != "r12-07 pdu r12-07-a,r12-07-b" {
|
||||
t.Errorf("short %q", loc.Short())
|
||||
}
|
||||
if loc.At(Rack) != "rack:r12-07" || loc.At(PDU) != "pdu:r12-07-a" {
|
||||
t.Errorf("at rack %q, at pdu %q", loc.At(Rack), loc.At(PDU))
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithoutLabelsIsNowhere(t *testing.T) {
|
||||
loc := LocationOf(map[string]string{"role": "gpu"})
|
||||
if !loc.Empty() || len(loc.Domains()) != 0 || loc.Short() != "" {
|
||||
t.Errorf("an unlabelled node is somewhere: %+v", loc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADomainIsNamedAsLevelColonName(t *testing.T) {
|
||||
d, err := ParseDomain(" Rack:r12-07 ")
|
||||
if err != nil || d != "rack:r12-07" || d.Human() != "rack r12-07" {
|
||||
t.Errorf("got %q, %v", d, err)
|
||||
}
|
||||
for _, bad := range []string{"r12-07", "rack:", "shelf:3"} {
|
||||
if _, err := ParseDomain(bad); err == nil {
|
||||
t.Errorf("%q was accepted as a domain", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLimitIsTheCapLessTheMargin(t *testing.T) {
|
||||
p := Policy{Margin: 0.1, Budgets: map[Domain]Budget{"rack:r1": {CapWatts: 16000}}}
|
||||
if got := p.Limit("rack:r1"); got != 14400 {
|
||||
t.Errorf("limit %v, want 14400", got)
|
||||
}
|
||||
if p.Limit("rack:r2") != 0 {
|
||||
t.Error("a domain with no budget has a limit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFirstHoldAmongANodesDomainsIsFound(t *testing.T) {
|
||||
p := Policy{Holds: map[Domain]Hold{"pdu:p2": {Domain: "pdu:p2", Reason: "breaker"}}}
|
||||
if h, ok := p.HeldIn([]Domain{"rack:r1", "pdu:p1", "pdu:p2"}); !ok || h.Reason != "breaker" {
|
||||
t.Errorf("got %+v, %v", h, ok)
|
||||
}
|
||||
if _, ok := p.HeldIn([]Domain{"rack:r1"}); ok {
|
||||
t.Error("an unheld rack is held")
|
||||
}
|
||||
c := p.Clone()
|
||||
delete(c.Holds, "pdu:p2")
|
||||
if _, ok := p.Holds["pdu:p2"]; !ok {
|
||||
t.Error("changing a clone changed the original")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user