llmgw/internal/control/facility/facility.go

// Package facility is where nodes are in the building, and what the building
// allows: the racks, rows, rooms and power distribution units a node belongs
// to, which of them an operator has taken out of service, and how much power
// each may draw.
//
// Placement reads it to keep replicas of one model off a single rack or PDU,
// to keep a replica's shards inside one domain when its profile asks, to place
// nothing in a domain held for maintenance, and to refuse a replica that would
// take a domain past its power cap. Everything here is optional: a node with
// no location labels and a cluster with no facility configuration place
// exactly as they did before.
package facility

import (
	"fmt"
	"slices"
	"strings"
	"time"
)

// Level is a kind of failure domain, from the largest to the smallest.
type Level string

const (
	Site Level = "site"
	Room Level = "room"
	Row  Level = "row"
	Rack Level = "rack"
	// PDU is a power distribution unit. A node may be on several, one per
	// feed of a dual-fed rack, so its label takes a comma separated list.
	PDU Level = "pdu"
)

// Levels are every level, the building's tree first and power after it.
var Levels = []Level{Site, Room, Row, Rack, PDU}

// LabelPrefix starts the node labels that say where a node is.
const LabelPrefix = "llmgw.io/"

// Label is the node label a level is read from: llmgw.io/rack.
func (l Level) Label() string { return LabelPrefix + string(l) }

// ParseLevel reads a level's name, as a profile or a command gives it.
func ParseLevel(s string) (Level, error) {
	l := Level(strings.ToLower(strings.TrimSpace(s)))
	if slices.Contains(Levels, l) {
		return l, nil
	}
	return "", fmt.Errorf("%q is not a facility level: use site, room, row, rack or pdu", s)
}

// Domain names one failure domain as level:name, rack:r12-07. Names are taken
// as the labels give them, so a rack is one domain across the whole cluster:
// two rooms that both have a rack r1 must label them apart.
type Domain string

// Of builds a domain from its level and name.
func Of(l Level, name string) Domain { return Domain(string(l) + ":" + name) }

// Level is the domain's level.
func (d Domain) Level() Level {
	l, _, _ := strings.Cut(string(d), ":")
	return Level(l)
}

// Name is the domain's name within its level.
func (d Domain) Name() string {
	_, n, _ := strings.Cut(string(d), ":")
	return n
}

// String is level:name; human text says "rack r12-07".
func (d Domain) String() string { return string(d) }

// Human is the domain as a sentence says it: rack r12-07.
func (d Domain) Human() string { return string(d.Level()) + " " + d.Name() }

// ParseDomain reads level:name, the way commands and configuration name a
// domain.
func ParseDomain(s string) (Domain, error) {
	l, n, ok := strings.Cut(strings.TrimSpace(s), ":")
	if !ok || strings.TrimSpace(n) == "" {
		return "", fmt.Errorf("%q is not a domain: name one as level:name, like rack:r12-07", s)
	}
	lv, err := ParseLevel(l)
	if err != nil {
		return "", err
	}
	return Of(lv, strings.TrimSpace(n)), nil
}

// Location is where a node is, read from its labels. Empty fields are not
// known, which places as it always did.
type Location struct {
	Site string   `json:"site,omitempty"`
	Room string   `json:"room,omitempty"`
	Row  string   `json:"row,omitempty"`
	Rack string   `json:"rack,omitempty"`
	PDUs []string `json:"pdus,omitempty"`
}

// LocationOf reads a node's labels.
func LocationOf(labels map[string]string) Location {
	get := func(l Level) string { return strings.TrimSpace(labels[l.Label()]) }
	loc := Location{Site: get(Site), Room: get(Room), Row: get(Row), Rack: get(Rack)}
	for _, p := range strings.Split(get(PDU), ",") {
		if p = strings.TrimSpace(p); p != "" && !slices.Contains(loc.PDUs, p) {
			loc.PDUs = append(loc.PDUs, p)
		}
	}
	return loc
}

// Empty reports whether nothing is known about where the node is.
func (l Location) Empty() bool {
	return l.Site == "" && l.Room == "" && l.Row == "" && l.Rack == "" && len(l.PDUs) == 0
}

// Domains are every domain the location is in, largest first.
func (l Location) Domains() []Domain {
	var out []Domain
	add := func(lv Level, name string) {
		if name != "" {
			out = append(out, Of(lv, name))
		}
	}
	add(Site, l.Site)
	add(Room, l.Room)
	add(Row, l.Row)
	add(Rack, l.Rack)
	for _, p := range l.PDUs {
		add(PDU, p)
	}
	return out
}

// At is the domain at one level, or empty. For PDUs it is the first.
func (l Location) At(lv Level) Domain {
	for _, d := range l.Domains() {
		if d.Level() == lv {
			return d
		}
	}
	return ""
}

// Short is the location in a few characters for a table: the rack, else the
// row, room or site, with the PDUs after a slash.
func (l Location) Short() string {
	s := ""
	switch {
	case l.Rack != "":
		s = l.Rack
	case l.Row != "":
		s = "row " + l.Row
	case l.Room != "":
		s = l.Room
	case l.Site != "":
		s = l.Site
	}
	if len(l.PDUs) > 0 {
		if s != "" {
			s += " "
		}
		s += "pdu " + strings.Join(l.PDUs, ",")
	}
	return s
}

// Hold takes a domain out of placement: maintenance, a cooling fault, work on
// a PDU. What runs there keeps running unless Drain is set.
type Hold struct {
	Domain Domain    `json:"domain"`
	Reason string    `json:"reason,omitempty"`
	Drain  bool      `json:"drain,omitempty"`
	By     string    `json:"by,omitempty"`
	At     time.Time `json:"at"`
}

// Budget is a domain's power cap.
type Budget struct {
	// CapWatts is what the domain may draw. For a dual-fed rack it is what
	// one feed can carry alone, since that is what the rack must survive.
	CapWatts float64
	// OtherWatts is drawn in the domain by things llmgw does not place on:
	// switches, storage, nodes outside the cluster.
	OtherWatts float64
}

// Policy is the facility's rules as placement reads them. The zero value
// holds nothing and caps nothing.
type Policy struct {
	// Holds are the domains taken out of placement.
	Holds map[Domain]Hold
	// Budgets are the domains with a power cap.
	Budgets map[Domain]Budget
	// Margin is the share of every cap kept free, 0.1 for ten percent.
	Margin float64
	// GPUWatts is a card's draw when its own power limit is not known.
	GPUWatts float64
	// CPUCoreWatts is what a core reserved by a CPU replica draws.
	CPUCoreWatts float64
	// NodeWatts is what a node draws beyond the GPUs and the cores given to
	// replicas: its idle draw, fans, memory and disks. Counted for every node
	// in a capped domain, busy or not.
	NodeWatts float64
	// MeasuredFresh is how old a node's power reading may be and still be
	// counted. Zero is two minutes.
	MeasuredFresh time.Duration
}

// Clone is a copy whose maps are its own.
func (p Policy) Clone() Policy {
	c := p
	if p.Holds != nil {
		c.Holds = make(map[Domain]Hold, len(p.Holds))
		for k, v := range p.Holds {
			c.Holds[k] = v
		}
	}
	if p.Budgets != nil {
		c.Budgets = make(map[Domain]Budget, len(p.Budgets))
		for k, v := range p.Budgets {
			c.Budgets[k] = v
		}
	}
	return c
}

// HeldIn is the first hold among a node's domains, largest first.
func (p Policy) HeldIn(domains []Domain) (Hold, bool) {
	for _, d := range domains {
		if h, ok := p.Holds[d]; ok {
			return h, true
		}
	}
	return Hold{}, false
}

// Limit is the most a domain may draw, its cap less the margin; zero with
// no cap.
func (p Policy) Limit(d Domain) float64 {
	b, ok := p.Budgets[d]
	if !ok || b.CapWatts <= 0 {
		return 0
	}
	m := p.Margin
	if m < 0 || m >= 1 {
		m = 0
	}
	return b.CapWatts * (1 - m)
}

// Fresh is how old a power reading may be.
func (p Policy) Fresh() time.Duration {
	if p.MeasuredFresh > 0 {
		return p.MeasuredFresh
	}
	return 2 * time.Minute
}

// KW renders watts as a sentence about a rack would.
func KW(w float64) string {
	if w >= 1000 {
		return fmt.Sprintf("%.1f kW", w/1000)
	}
	return fmt.Sprintf("%.0f W", w)
}